Exemple #1
0
def whoosh_index(repo_location, full_index):
    #log = whoosh_index.get_logger()
    from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon
    index_location = config['index_dir']
    WhooshIndexingDaemon(index_location=index_location,
                         repo_location=repo_location, sa=get_session())\
                         .run(full_index=full_index)
Exemple #2
0
def reset_user_password(user_email):
    from rhodecode.lib import auth

    log = get_logger(reset_user_password)
    DBS = get_session()

    try:
        try:
            user = User.get_by_email(user_email)
            new_passwd = auth.PasswordGenerator().gen_password(8,
                             auth.PasswordGenerator.ALPHABETS_BIG_SMALL)
            if user:
                user.password = auth.get_crypt_password(new_passwd)
                user.api_key = auth.generate_api_key(user.username)
                DBS.add(user)
                DBS.commit()
                log.info('change password for %s' % user_email)
            if new_passwd is None:
                raise Exception('unable to generate new password')
        except:
            log.error(traceback.format_exc())
            DBS.rollback()

        run_task(send_email, user_email,
                 'Your new password',
                 'Your new RhodeCode password:%s' % (new_passwd))
        log.info('send new password mail to %s' % user_email)

    except:
        log.error('Failed to update user password')
        log.error(traceback.format_exc())

    return True
Exemple #3
0
def send_password_link(user_email):
    from rhodecode.model.notification import EmailNotificationModel

    log = get_logger(send_password_link)
    DBS = get_session()

    try:
        user = User.get_by_email(user_email)
        if user:
            log.debug('password reset user found %s' % user)
            link = url('reset_password_confirmation', key=user.api_key,
                       qualified=True)
            reg_type = EmailNotificationModel.TYPE_PASSWORD_RESET
            body = EmailNotificationModel().get_email_tmpl(reg_type,
                                                **{'user':user.short_contact,
                                                   'reset_url':link})
            log.debug('sending email')
            run_task(send_email, user_email,
                     _("password reset link"), body)
            log.info('send new password mail to %s' % user_email)
        else:
            log.debug("password reset email %s not found" % user_email)
    except:
        log.error(traceback.format_exc())
        return False

    return True
Exemple #4
0
def whoosh_index(repo_location, full_index):
    from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon

    log = get_logger(whoosh_index)
    DBS = get_session()

    index_location = config["index_dir"]
    WhooshIndexingDaemon(index_location=index_location, repo_location=repo_location, sa=DBS).run(full_index=full_index)
Exemple #5
0
def whoosh_index(repo_location, full_index):
    from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon
    log = get_logger(whoosh_index)
    DBS = get_session()

    index_location = config['index_dir']
    WhooshIndexingDaemon(index_location=index_location,
                         repo_location=repo_location, sa=DBS)\
                         .run(full_index=full_index)
Exemple #6
0
def send_email(recipients, subject, body='', html_body=''):
    """
    Sends an email with defined parameters from the .ini files.

    :param recipients: list of recipients, it this is empty the defined email
        address from field 'email_to' is used instead
    :param subject: subject of the mail
    :param body: body of the mail
    :param html_body: html version of body
    """
    log = get_logger(send_email)
    DBS = get_session()

    email_config = config
    subject = "%s %s" % (email_config.get('email_prefix', ''), subject)
    if not recipients:
        # if recipients are not defined we send to email_config + all admins
        admins = [
            u.email for u in User.query().filter(User.admin == True).all()
        ]
        recipients = [email_config.get('email_to')] + admins

    mail_from = email_config.get('app_email_from', 'RhodeCode')
    user = email_config.get('smtp_username')
    passwd = email_config.get('smtp_password')
    mail_server = email_config.get('smtp_server')
    mail_port = email_config.get('smtp_port')
    tls = str2bool(email_config.get('smtp_use_tls'))
    ssl = str2bool(email_config.get('smtp_use_ssl'))
    debug = str2bool(email_config.get('debug'))
    smtp_auth = email_config.get('smtp_auth')

    if not mail_server:
        log.error("SMTP mail server not configured - cannot send mail")
        return False

    try:
        m = SmtpMailer(mail_from,
                       user,
                       passwd,
                       mail_server,
                       smtp_auth,
                       mail_port,
                       ssl,
                       tls,
                       debug=debug)
        m.send(recipients, subject, body, html_body)
    except:
        log.error('Mail sending failed')
        log.error(traceback.format_exc())
        return False
    return True
Exemple #7
0
def create_repo_fork(form_data, cur_user):
    from rhodecode.model.repo import RepoModel

    try:
        log = create_repo_fork.get_logger()
    except:
        log = logging.getLogger(__name__)

    repo_model = RepoModel(get_session())
    repo_model.create(form_data, cur_user, just_db=True, fork=True)
    repo_name = form_data['repo_name']
    repos_path = get_repos_path()
    repo_path = os.path.join(repos_path, repo_name)
    repo_fork_path = os.path.join(repos_path, form_data['fork_name'])
    alias = form_data['repo_type']

    log.info('creating repo fork %s as %s', repo_name, repo_path)
    backend = get_backend(alias)
    backend(str(repo_fork_path), create=True, src_url=str(repo_path))
Exemple #8
0
def send_email(recipients, subject, body):
    """
    Sends an email with defined parameters from the .ini files.

    :param recipients: list of recipients, it this is empty the defined email
        address from field 'email_to' is used instead
    :param subject: subject of the mail
    :param body: body of the mail
    """
    try:
        log = send_email.get_logger()
    except:
        log = logging.getLogger(__name__)
    
    sa = get_session()
    email_config = config

    if not recipients:
        # if recipients are not defined we send to email_config + all admins
        admins = [
            u.email for u in sa.query(User).filter(User.admin==True).all()
        ]
        recipients = [email_config.get('email_to')] + admins

    mail_from = email_config.get('app_email_from')
    user = email_config.get('smtp_username')
    passwd = email_config.get('smtp_password')
    mail_server = email_config.get('smtp_server')
    mail_port = email_config.get('smtp_port')
    tls = str2bool(email_config.get('smtp_use_tls'))
    ssl = str2bool(email_config.get('smtp_use_ssl'))
    debug = str2bool(config.get('debug'))
    smtp_auth = email_config.get('smtp_auth')

    try:
        m = SmtpMailer(mail_from, user, passwd, mail_server,smtp_auth,
                       mail_port, ssl, tls, debug=debug)
        m.send(recipients, subject, body)
    except:
        log.error('Mail sending failed')
        log.error(traceback.format_exc())
        return False
    return True
Exemple #9
0
def send_email(recipients, subject, body="", html_body=""):
    """
    Sends an email with defined parameters from the .ini files.

    :param recipients: list of recipients, it this is empty the defined email
        address from field 'email_to' is used instead
    :param subject: subject of the mail
    :param body: body of the mail
    :param html_body: html version of body
    """
    log = get_logger(send_email)
    DBS = get_session()

    email_config = config
    subject = "%s %s" % (email_config.get("email_prefix", ""), subject)
    if not recipients:
        # if recipients are not defined we send to email_config + all admins
        admins = [u.email for u in User.query().filter(User.admin == True).all()]
        recipients = [email_config.get("email_to")] + admins

    mail_from = email_config.get("app_email_from", "RhodeCode")
    user = email_config.get("smtp_username")
    passwd = email_config.get("smtp_password")
    mail_server = email_config.get("smtp_server")
    mail_port = email_config.get("smtp_port")
    tls = str2bool(email_config.get("smtp_use_tls"))
    ssl = str2bool(email_config.get("smtp_use_ssl"))
    debug = str2bool(email_config.get("debug"))
    smtp_auth = email_config.get("smtp_auth")

    if not mail_server:
        log.error("SMTP mail server not configured - cannot send mail")
        return False

    try:
        m = SmtpMailer(mail_from, user, passwd, mail_server, smtp_auth, mail_port, ssl, tls, debug=debug)
        m.send(recipients, subject, body, html_body)
    except:
        log.error("Mail sending failed")
        log.error(traceback.format_exc())
        return False
    return True
Exemple #10
0
def create_repo_fork(form_data, cur_user):
    """
    Creates a fork of repository using interval VCS methods

    :param form_data:
    :param cur_user:
    """
    from rhodecode.model.repo import RepoModel

    log = get_logger(create_repo_fork)
    DBS = get_session()

    base_path = Repository.base_path()

    fork_repo = RepoModel(DBS).create(form_data, cur_user,
                                      just_db=True, fork=True)

    alias = form_data['repo_type']
    org_repo_name = form_data['org_path']
    fork_name = form_data['repo_name_full']
    update_after_clone = form_data['update_after_clone']
    source_repo_path = os.path.join(base_path, org_repo_name)
    destination_fork_path = os.path.join(base_path, fork_name)

    log.info('creating fork of %s as %s', source_repo_path,
             destination_fork_path)
    backend = get_backend(alias)
    backend(safe_str(destination_fork_path), create=True,
            src_url=safe_str(source_repo_path),
            update_after_clone=update_after_clone)
    log_create_repository(fork_repo.get_dict(), created_by=cur_user.username)

    action_logger(cur_user, 'user_forked_repo:%s' % fork_name,
                   org_repo_name, '', DBS)

    action_logger(cur_user, 'user_created_fork:%s' % fork_name,
                   fork_name, '', DBS)
    # finally commit at latest possible stage
    DBS.commit()
Exemple #11
0
def reset_user_password(user_email):
    try:
        log = reset_user_password.get_logger()
    except:
        log = logging.getLogger(__name__)

    from rhodecode.lib import auth

    try:
        try:
            sa = get_session()
            user = sa.query(User).filter(User.email == user_email).scalar()
            new_passwd = auth.PasswordGenerator().gen_password(8,
                             auth.PasswordGenerator.ALPHABETS_BIG_SMALL)
            if user:
                user.password = auth.get_crypt_password(new_passwd)
                user.api_key = auth.generate_api_key(user.username)
                sa.add(user)
                sa.commit()
                log.info('change password for %s', user_email)
            if new_passwd is None:
                raise Exception('unable to generate new password')

        except:
            log.error(traceback.format_exc())
            sa.rollback()

        run_task(send_email, user_email,
                 "Your new RhodeCode password",
                 'Your new RhodeCode password:%s' % (new_passwd))
        log.info('send new password mail to %s', user_email)

    except:
        log.error('Failed to update user password')
        log.error(traceback.format_exc())

    return True
Exemple #12
0
def send_password_link(user_email):
    try:
        log = reset_user_password.get_logger()
    except:
        log = logging.getLogger(__name__)

    from rhodecode.lib import auth

    try:
        sa = get_session()
        user = sa.query(User).filter(User.email == user_email).scalar()

        if user:
            link = url('reset_password_confirmation', key=user.api_key,
                       qualified=True)
            tmpl = """
Hello %s

We received a request to create a new password for your account.

You can generate it by clicking following URL:

%s

If you didn't request new password please ignore this email.
            """
            run_task(send_email, user_email,
                     "RhodeCode password reset link",
                     tmpl % (user.short_contact, link))
            log.info('send new password mail to %s', user_email)

    except:
        log.error('Failed to update user password')
        log.error(traceback.format_exc())
        return False

    return True
Exemple #13
0
def get_commits_stats(repo_name, ts_min_y, ts_max_y, recurse_limit=100):
    log = get_logger(get_commits_stats)
    DBS = get_session()
    lockkey = __get_lockkey('get_commits_stats', repo_name, ts_min_y, ts_max_y)
    lockkey_path = config['app_conf']['cache_dir']

    log.info('running task with lockkey %s' % lockkey)

    try:
        lock = l = DaemonLock(file_=jn(lockkey_path, lockkey))

        # for js data compatibility cleans the key for person from '
        akc = lambda k: person(k).replace('"', "")

        co_day_auth_aggr = {}
        commits_by_day_aggregate = {}
        repo = Repository.get_by_repo_name(repo_name)
        if repo is None:
            return True

        repo = repo.scm_instance
        repo_size = repo.count()
        # return if repo have no revisions
        if repo_size < 1:
            lock.release()
            return True

        skip_date_limit = True
        parse_limit = int(config['app_conf'].get('commit_parse_limit'))
        last_rev = None
        last_cs = None
        timegetter = itemgetter('time')

        dbrepo = DBS.query(Repository)\
            .filter(Repository.repo_name == repo_name).scalar()
        cur_stats = DBS.query(Statistics)\
            .filter(Statistics.repository == dbrepo).scalar()

        if cur_stats is not None:
            last_rev = cur_stats.stat_on_revision

        if last_rev == repo.get_changeset().revision and repo_size > 1:
            # pass silently without any work if we're not on first revision or
            # current state of parsing revision(from db marker) is the
            # last revision
            lock.release()
            return True

        if cur_stats:
            commits_by_day_aggregate = OrderedDict(
                json.loads(cur_stats.commit_activity_combined))
            co_day_auth_aggr = json.loads(cur_stats.commit_activity)

        log.debug('starting parsing %s' % parse_limit)
        lmktime = mktime

        last_rev = last_rev + 1 if last_rev >= 0 else 0
        log.debug('Getting revisions from %s to %s' %
                  (last_rev, last_rev + parse_limit))
        for cs in repo[last_rev:last_rev + parse_limit]:
            log.debug('parsing %s' % cs)
            last_cs = cs  # remember last parsed changeset
            k = lmktime([
                cs.date.timetuple()[0],
                cs.date.timetuple()[1],
                cs.date.timetuple()[2], 0, 0, 0, 0, 0, 0
            ])

            if akc(cs.author) in co_day_auth_aggr:
                try:
                    l = [
                        timegetter(x)
                        for x in co_day_auth_aggr[akc(cs.author)]['data']
                    ]
                    time_pos = l.index(k)
                except ValueError:
                    time_pos = None

                if time_pos >= 0 and time_pos is not None:

                    datadict = \
                        co_day_auth_aggr[akc(cs.author)]['data'][time_pos]

                    datadict["commits"] += 1
                    datadict["added"] += len(cs.added)
                    datadict["changed"] += len(cs.changed)
                    datadict["removed"] += len(cs.removed)

                else:
                    if k >= ts_min_y and k <= ts_max_y or skip_date_limit:

                        datadict = {
                            "time": k,
                            "commits": 1,
                            "added": len(cs.added),
                            "changed": len(cs.changed),
                            "removed": len(cs.removed),
                        }
                        co_day_auth_aggr[akc(cs.author)]['data']\
                            .append(datadict)

            else:
                if k >= ts_min_y and k <= ts_max_y or skip_date_limit:
                    co_day_auth_aggr[akc(cs.author)] = {
                        "label":
                        akc(cs.author),
                        "data": [{
                            "time": k,
                            "commits": 1,
                            "added": len(cs.added),
                            "changed": len(cs.changed),
                            "removed": len(cs.removed),
                        }],
                        "schema": ["commits"],
                    }

            #gather all data by day
            if k in commits_by_day_aggregate:
                commits_by_day_aggregate[k] += 1
            else:
                commits_by_day_aggregate[k] = 1

        overview_data = sorted(commits_by_day_aggregate.items(),
                               key=itemgetter(0))

        if not co_day_auth_aggr:
            co_day_auth_aggr[akc(repo.contact)] = {
                "label": akc(repo.contact),
                "data": [0, 1],
                "schema": ["commits"],
            }

        stats = cur_stats if cur_stats else Statistics()
        stats.commit_activity = json.dumps(co_day_auth_aggr)
        stats.commit_activity_combined = json.dumps(overview_data)

        log.debug('last revison %s' % last_rev)
        leftovers = len(repo.revisions[last_rev:])
        log.debug('revisions to parse %s' % leftovers)

        if last_rev == 0 or leftovers < parse_limit:
            log.debug('getting code trending stats')
            stats.languages = json.dumps(__get_codes_stats(repo_name))

        try:
            stats.repository = dbrepo
            stats.stat_on_revision = last_cs.revision if last_cs else 0
            DBS.add(stats)
            DBS.commit()
        except:
            log.error(traceback.format_exc())
            DBS.rollback()
            lock.release()
            return False

        # final release
        lock.release()

        # execute another task if celery is enabled
        if len(repo.revisions) > 1 and CELERY_ON and recurse_limit > 0:
            recurse_limit -= 1
            run_task(get_commits_stats, repo_name, ts_min_y, ts_max_y,
                     recurse_limit)
        if recurse_limit <= 0:
            log.debug('Breaking recursive mode due to reach of recurse limit')
        return True
    except LockHeld:
        log.info('LockHeld')
        return 'Task with key %s already running' % lockkey
Exemple #14
0
def create_repo_fork(form_data, cur_user):
    """
    Creates a fork of repository using interval VCS methods

    :param form_data:
    :param cur_user:
    """
    from rhodecode.model.repo import RepoModel
    from rhodecode.model.user import UserModel

    log = get_logger(create_repo_fork)
    DBS = get_session()

    base_path = Repository.base_path()
    cur_user = UserModel(DBS)._get_user(cur_user)

    fork_name = form_data['repo_name_full']
    repo_type = form_data['repo_type']
    description = form_data['description']
    owner = cur_user
    private = form_data['private']
    clone_uri = form_data.get('clone_uri')
    repos_group = form_data['repo_group']
    landing_rev = form_data['landing_rev']
    copy_fork_permissions = form_data.get('copy_permissions')
    fork_of = RepoModel(DBS)._get_repo(form_data.get('fork_parent_id'))

    fork_repo = RepoModel(DBS).create_repo(
        fork_name,
        repo_type,
        description,
        owner,
        private,
        clone_uri,
        repos_group,
        landing_rev,
        just_db=True,
        fork_of=fork_of,
        copy_fork_permissions=copy_fork_permissions)

    update_after_clone = form_data['update_after_clone']

    source_repo_path = os.path.join(base_path, fork_of.repo_name)
    destination_fork_path = os.path.join(base_path, fork_name)

    log.info('creating fork of %s as %s', source_repo_path,
             destination_fork_path)
    backend = get_backend(repo_type)

    if repo_type == 'git':
        r = backend(safe_str(destination_fork_path),
                    create=True,
                    src_url=safe_str(source_repo_path),
                    update_after_clone=update_after_clone,
                    bare=True)
        # add rhodecode hook into this repo
        ScmModel().install_git_hook(repo=r)
    elif repo_type == 'hg':
        r = backend(safe_str(destination_fork_path),
                    create=True,
                    src_url=safe_str(source_repo_path),
                    update_after_clone=update_after_clone)
    else:
        raise Exception('Unknown backend type %s' % repo_type)

    log_create_repository(fork_repo.get_dict(), created_by=cur_user.username)

    action_logger(cur_user, 'user_forked_repo:%s' % fork_name,
                  fork_of.repo_name, '', DBS)

    action_logger(cur_user, 'user_created_fork:%s' % fork_name, fork_name, '',
                  DBS)
    # finally commit at latest possible stage
    DBS.commit()
    fork_repo.update_changeset_cache()
Exemple #15
0
def get_repos_path():
    sa = get_session()
    q = sa.query(RhodeCodeUi).filter(RhodeCodeUi.ui_key == '/').one()
    return q.ui_value
Exemple #16
0
def create_repo_fork(form_data, cur_user):
    """
    Creates a fork of repository using interval VCS methods

    :param form_data:
    :param cur_user:
    """
    from rhodecode.model.repo import RepoModel
    from rhodecode.model.user import UserModel

    log = get_logger(create_repo_fork)
    DBS = get_session()

    base_path = Repository.base_path()
    cur_user = UserModel(DBS)._get_user(cur_user)

    fork_name = form_data["repo_name_full"]
    repo_type = form_data["repo_type"]
    description = form_data["description"]
    owner = cur_user
    private = form_data["private"]
    clone_uri = form_data.get("clone_uri")
    repos_group = form_data["repo_group"]
    landing_rev = form_data["landing_rev"]
    copy_fork_permissions = form_data.get("copy_permissions")
    fork_of = RepoModel(DBS)._get_repo(form_data.get("fork_parent_id"))

    fork_repo = RepoModel(DBS).create_repo(
        fork_name,
        repo_type,
        description,
        owner,
        private,
        clone_uri,
        repos_group,
        landing_rev,
        just_db=True,
        fork_of=fork_of,
        copy_fork_permissions=copy_fork_permissions,
    )

    update_after_clone = form_data["update_after_clone"]

    source_repo_path = os.path.join(base_path, fork_of.repo_name)
    destination_fork_path = os.path.join(base_path, fork_name)

    log.info("creating fork of %s as %s", source_repo_path, destination_fork_path)
    backend = get_backend(repo_type)

    if repo_type == "git":
        r = backend(
            safe_str(destination_fork_path),
            create=True,
            src_url=safe_str(source_repo_path),
            update_after_clone=update_after_clone,
            bare=True,
        )
        # add rhodecode hook into this repo
        ScmModel().install_git_hook(repo=r)
    elif repo_type == "hg":
        r = backend(
            safe_str(destination_fork_path),
            create=True,
            src_url=safe_str(source_repo_path),
            update_after_clone=update_after_clone,
        )
    else:
        raise Exception("Unknown backend type %s" % repo_type)

    log_create_repository(fork_repo.get_dict(), created_by=cur_user.username)

    action_logger(cur_user, "user_forked_repo:%s" % fork_name, fork_of.repo_name, "", DBS)

    action_logger(cur_user, "user_created_fork:%s" % fork_name, fork_name, "", DBS)
    # finally commit at latest possible stage
    DBS.commit()
    fork_repo.update_changeset_cache()
def create_repo_fork(form_data, cur_user):
    """
    Creates a fork of repository using internal VCS methods

    :param form_data:
    :param cur_user:
    """
    from rhodecode.model.repo import RepoModel
    from rhodecode.model.user import UserModel

    log = get_logger(create_repo_fork)
    DBS = get_session()

    cur_user = UserModel(DBS)._get_user(cur_user)
    owner = cur_user

    repo_name = form_data['repo_name']  # fork in this case
    repo_name_full = form_data['repo_name_full']
    repo_type = form_data['repo_type']
    description = form_data['description']
    private = form_data['private']
    clone_uri = form_data.get('clone_uri')
    repo_group = safe_int(form_data['repo_group'])
    landing_rev = form_data['landing_rev']
    copy_fork_permissions = form_data.get('copy_permissions')
    fork_id = safe_int(form_data.get('fork_parent_id'))

    try:
        fork_of = RepoModel(DBS)._get_repo(fork_id)
        RepoModel(DBS)._create_repo(
            repo_name=repo_name_full,
            repo_type=repo_type,
            description=description,
            owner=owner,
            private=private,
            clone_uri=clone_uri,
            repo_group=repo_group,
            landing_rev=landing_rev,
            fork_of=fork_of,
            copy_fork_permissions=copy_fork_permissions)
        action_logger(cur_user, 'user_forked_repo:%s' % repo_name_full,
                      fork_of.repo_name, '', DBS)
        DBS.commit()

        base_path = Repository.base_path()
        source_repo_path = os.path.join(base_path, fork_of.repo_name)

        # now create this repo on Filesystem
        RepoModel(DBS)._create_filesystem_repo(
            repo_name=repo_name,
            repo_type=repo_type,
            repo_group=RepoModel(DBS)._get_repo_group(repo_group),
            clone_uri=source_repo_path,
        )
        repo = Repository.get_by_repo_name(repo_name_full)
        log_create_repository(created_by=owner.username, **repo.get_dict())

        # update repo commit caches initially
        config = repo._config
        config.set('extensions', 'largefiles', '')
        repo.update_commit_cache(config=config)

        # set new created state
        repo.set_state(Repository.STATE_CREATED)
        DBS.commit()
    except Exception as e:
        log.warning(
            'Exception %s occurred when forking repository, '
            'doing cleanup...', e)
        # rollback things manually !
        repo = Repository.get_by_repo_name(repo_name_full)
        if repo:
            Repository.delete(repo.repo_id)
            DBS.commit()
            RepoModel(DBS)._delete_filesystem_repo(repo)
        raise

    # it's an odd fix to make celery fail task when exception occurs
    def on_failure(self, *args, **kwargs):
        pass

    return True
def create_repo(form_data, cur_user):
    from rhodecode.model.repo import RepoModel
    from rhodecode.model.user import UserModel
    from rhodecode.model.settings import SettingsModel

    log = get_logger(create_repo)
    DBS = get_session()

    cur_user = UserModel(DBS)._get_user(cur_user)
    owner = cur_user

    repo_name = form_data['repo_name']
    repo_name_full = form_data['repo_name_full']
    repo_type = form_data['repo_type']
    description = form_data['repo_description']
    private = form_data['repo_private']
    clone_uri = form_data.get('clone_uri')
    repo_group = safe_int(form_data['repo_group'])
    landing_rev = form_data['repo_landing_rev']
    copy_fork_permissions = form_data.get('copy_permissions')
    copy_group_permissions = form_data.get('repo_copy_permissions')
    fork_of = form_data.get('fork_parent_id')
    state = form_data.get('repo_state', Repository.STATE_PENDING)

    # repo creation defaults, private and repo_type are filled in form
    defs = SettingsModel().get_default_repo_settings(strip_prefix=True)
    enable_statistics = form_data.get('enable_statistics',
                                      defs.get('repo_enable_statistics'))
    enable_locking = form_data.get('enable_locking',
                                   defs.get('repo_enable_locking'))
    enable_downloads = form_data.get('enable_downloads',
                                     defs.get('repo_enable_downloads'))

    try:
        RepoModel(DBS)._create_repo(
            repo_name=repo_name_full,
            repo_type=repo_type,
            description=description,
            owner=owner,
            private=private,
            clone_uri=clone_uri,
            repo_group=repo_group,
            landing_rev=landing_rev,
            fork_of=fork_of,
            copy_fork_permissions=copy_fork_permissions,
            copy_group_permissions=copy_group_permissions,
            enable_statistics=enable_statistics,
            enable_locking=enable_locking,
            enable_downloads=enable_downloads,
            state=state)

        action_logger(cur_user, 'user_created_repo', repo_name_full, '', DBS)
        DBS.commit()

        # now create this repo on Filesystem
        RepoModel(DBS)._create_filesystem_repo(
            repo_name=repo_name,
            repo_type=repo_type,
            repo_group=RepoModel(DBS)._get_repo_group(repo_group),
            clone_uri=clone_uri,
        )
        repo = Repository.get_by_repo_name(repo_name_full)
        log_create_repository(created_by=owner.username, **repo.get_dict())

        # update repo commit caches initially
        repo.update_commit_cache()

        # set new created state
        repo.set_state(Repository.STATE_CREATED)
        DBS.commit()
    except Exception as e:
        log.warning(
            'Exception %s occurred when creating repository, '
            'doing cleanup...', e)
        # rollback things manually !
        repo = Repository.get_by_repo_name(repo_name_full)
        if repo:
            Repository.delete(repo.repo_id)
            DBS.commit()
            RepoModel(DBS)._delete_filesystem_repo(repo)
        raise

    # it's an odd fix to make celery fail task when exception occurs
    def on_failure(self, *args, **kwargs):
        pass

    return True
Exemple #19
0
def get_commits_stats(repo_name, ts_min_y, ts_max_y):
    log = get_logger(get_commits_stats)
    DBS = get_session()
    lockkey = __get_lockkey('get_commits_stats', repo_name, ts_min_y,
                            ts_max_y)
    lockkey_path = config['here']

    log.info('running task with lockkey %s' % lockkey)

    try:
        lock = l = DaemonLock(file_=jn(lockkey_path, lockkey))

        # for js data compatibility cleans the key for person from '
        akc = lambda k: person(k).replace('"', "")

        co_day_auth_aggr = {}
        commits_by_day_aggregate = {}
        repo = Repository.get_by_repo_name(repo_name)
        if repo is None:
            return True

        repo = repo.scm_instance
        repo_size = repo.count()
        # return if repo have no revisions
        if repo_size < 1:
            lock.release()
            return True

        skip_date_limit = True
        parse_limit = int(config['app_conf'].get('commit_parse_limit'))
        last_rev = None
        last_cs = None
        timegetter = itemgetter('time')

        dbrepo = DBS.query(Repository)\
            .filter(Repository.repo_name == repo_name).scalar()
        cur_stats = DBS.query(Statistics)\
            .filter(Statistics.repository == dbrepo).scalar()

        if cur_stats is not None:
            last_rev = cur_stats.stat_on_revision

        if last_rev == repo.get_changeset().revision and repo_size > 1:
            # pass silently without any work if we're not on first revision or
            # current state of parsing revision(from db marker) is the
            # last revision
            lock.release()
            return True

        if cur_stats:
            commits_by_day_aggregate = OrderedDict(json.loads(
                                        cur_stats.commit_activity_combined))
            co_day_auth_aggr = json.loads(cur_stats.commit_activity)

        log.debug('starting parsing %s' % parse_limit)
        lmktime = mktime

        last_rev = last_rev + 1 if last_rev >= 0 else 0
        log.debug('Getting revisions from %s to %s' % (
             last_rev, last_rev + parse_limit)
        )
        for cs in repo[last_rev:last_rev + parse_limit]:
            log.debug('parsing %s' % cs)
            last_cs = cs  # remember last parsed changeset
            k = lmktime([cs.date.timetuple()[0], cs.date.timetuple()[1],
                          cs.date.timetuple()[2], 0, 0, 0, 0, 0, 0])

            if akc(cs.author) in co_day_auth_aggr:
                try:
                    l = [timegetter(x) for x in
                         co_day_auth_aggr[akc(cs.author)]['data']]
                    time_pos = l.index(k)
                except ValueError:
                    time_pos = False

                if time_pos >= 0 and time_pos is not False:

                    datadict = \
                        co_day_auth_aggr[akc(cs.author)]['data'][time_pos]

                    datadict["commits"] += 1
                    datadict["added"] += len(cs.added)
                    datadict["changed"] += len(cs.changed)
                    datadict["removed"] += len(cs.removed)

                else:
                    if k >= ts_min_y and k <= ts_max_y or skip_date_limit:

                        datadict = {"time": k,
                                    "commits": 1,
                                    "added": len(cs.added),
                                    "changed": len(cs.changed),
                                    "removed": len(cs.removed),
                                   }
                        co_day_auth_aggr[akc(cs.author)]['data']\
                            .append(datadict)

            else:
                if k >= ts_min_y and k <= ts_max_y or skip_date_limit:
                    co_day_auth_aggr[akc(cs.author)] = {
                                        "label": akc(cs.author),
                                        "data": [{"time":k,
                                                 "commits":1,
                                                 "added":len(cs.added),
                                                 "changed":len(cs.changed),
                                                 "removed":len(cs.removed),
                                                 }],
                                        "schema": ["commits"],
                                        }

            #gather all data by day
            if k in commits_by_day_aggregate:
                commits_by_day_aggregate[k] += 1
            else:
                commits_by_day_aggregate[k] = 1

        overview_data = sorted(commits_by_day_aggregate.items(),
                               key=itemgetter(0))

        if not co_day_auth_aggr:
            co_day_auth_aggr[akc(repo.contact)] = {
                "label": akc(repo.contact),
                "data": [0, 1],
                "schema": ["commits"],
            }

        stats = cur_stats if cur_stats else Statistics()
        stats.commit_activity = json.dumps(co_day_auth_aggr)
        stats.commit_activity_combined = json.dumps(overview_data)

        log.debug('last revison %s' % last_rev)
        leftovers = len(repo.revisions[last_rev:])
        log.debug('revisions to parse %s' % leftovers)

        if last_rev == 0 or leftovers < parse_limit:
            log.debug('getting code trending stats')
            stats.languages = json.dumps(__get_codes_stats(repo_name))

        try:
            stats.repository = dbrepo
            stats.stat_on_revision = last_cs.revision if last_cs else 0
            DBS.add(stats)
            DBS.commit()
        except:
            log.error(traceback.format_exc())
            DBS.rollback()
            lock.release()
            return False

        # final release
        lock.release()

        # execute another task if celery is enabled
        if len(repo.revisions) > 1 and CELERY_ON:
            run_task(get_commits_stats, repo_name, ts_min_y, ts_max_y)
        return True
    except LockHeld:
        log.info('LockHeld')
        return 'Task with key %s already running' % lockkey
Exemple #20
0
def create_repo_fork(form_data, cur_user):
    """
    Creates a fork of repository using interval VCS methods

    :param form_data:
    :param cur_user:
    """
    from rhodecode.model.repo import RepoModel
    from rhodecode.model.user import UserModel

    log = get_logger(create_repo_fork)
    DBS = get_session()

    base_path = Repository.base_path()
    cur_user = UserModel(DBS)._get_user(cur_user)

    fork_name = form_data['repo_name_full']
    repo_type = form_data['repo_type']
    description = form_data['description']
    owner = cur_user
    private = form_data['private']
    clone_uri = form_data.get('clone_uri')
    repos_group = form_data['repo_group']
    landing_rev = form_data['landing_rev']
    copy_fork_permissions = form_data.get('copy_permissions')
    fork_of = RepoModel(DBS)._get_repo(form_data.get('fork_parent_id'))

    fork_repo = RepoModel(DBS).create_repo(
        fork_name, repo_type, description, owner, private, clone_uri,
        repos_group, landing_rev, just_db=True, fork_of=fork_of,
        copy_fork_permissions=copy_fork_permissions
    )

    update_after_clone = form_data['update_after_clone']

    source_repo_path = os.path.join(base_path, fork_of.repo_name)
    destination_fork_path = os.path.join(base_path, fork_name)

    log.info('creating fork of %s as %s', source_repo_path,
             destination_fork_path)
    backend = get_backend(repo_type)

    if repo_type == 'git':
        backend(safe_str(destination_fork_path), create=True,
                src_url=safe_str(source_repo_path),
                update_after_clone=update_after_clone,
                bare=True)
    elif repo_type == 'hg':
        backend(safe_str(destination_fork_path), create=True,
                src_url=safe_str(source_repo_path),
                update_after_clone=update_after_clone)
    else:
        raise Exception('Unknown backend type %s' % repo_type)

    log_create_repository(fork_repo.get_dict(), created_by=cur_user.username)

    action_logger(cur_user, 'user_forked_repo:%s' % fork_name,
                   fork_of.repo_name, '', DBS)

    action_logger(cur_user, 'user_created_fork:%s' % fork_name,
                   fork_name, '', DBS)
    # finally commit at latest possible stage
    DBS.commit()