def setUp(self): self.repo_path = mkdtemp() rmdir(self.repo_path) self.repo = get_repo(self.repo_path, VCS, create=True) self.commit(self.commits) self.project = Project.objects.create(slug='p1', vcs_type=VCS, repo_uri=self.repo_path)
def setUp(self): self.repo_path = mkdtemp() rmdir(self.repo_path) self.repo = get_repo(self.repo_path, VCS, create=True) self.commit(self.commits) self.project = Project.objects.create( slug='p1', vcs_type=VCS, repo_uri=self.repo_path )
def __get_codes_stats(repo_name): repos_path = get_repos_path() repo = get_repo(safe_str(os.path.join(repos_path, repo_name))) tip = repo.get_changeset() code_stats = {} def aggregate(cs): for f in cs[2]: ext = lower(f.extension) if ext in LANGUAGES_EXTENSIONS_MAP.keys() and not f.is_binary: if ext in code_stats: code_stats[ext] += 1 else: code_stats[ext] = 1 map(aggregate, tip.walk('/')) return code_stats or {}
def get_commits_stats(repo_name, ts_min_y, ts_max_y): try: log = get_commits_stats.get_logger() except: log = logging.getLogger(__name__) lockkey = __get_lockkey('get_commits_stats', repo_name, ts_min_y, ts_max_y) lockkey_path = config['here'] log.info('running task with lockkey %s', lockkey) try: lock = l = DaemonLock(file_=jn(lockkey_path, lockkey)) #for js data compatibilty cleans the key for person from ' akc = lambda k: person(k).replace('"', "") co_day_auth_aggr = {} commits_by_day_aggregate = {} repos_path = get_repos_path() repo = get_repo(safe_str(os.path.join(repos_path, repo_name))) repo_size = len(repo.revisions) #return if repo have no revisions if repo_size < 1: lock.release() return True skip_date_limit = True parse_limit = int(config['app_conf'].get('commit_parse_limit')) last_rev = 0 last_cs = None timegetter = itemgetter('time') sa = get_session() dbrepo = sa.query(Repository)\ .filter(Repository.repo_name == repo_name).scalar() cur_stats = sa.query(Statistics)\ .filter(Statistics.repository == dbrepo).scalar() if cur_stats is not None: last_rev = cur_stats.stat_on_revision if last_rev == repo.get_changeset().revision and repo_size > 1: #pass silently without any work if we're not on first revision or #current state of parsing revision(from db marker) is the #last revision lock.release() return True if cur_stats: commits_by_day_aggregate = OrderedDict(json.loads( cur_stats.commit_activity_combined)) co_day_auth_aggr = json.loads(cur_stats.commit_activity) log.debug('starting parsing %s', parse_limit) lmktime = mktime last_rev = last_rev + 1 if last_rev > 0 else last_rev for cs in repo[last_rev:last_rev + parse_limit]: last_cs = cs # remember last parsed changeset k = lmktime([cs.date.timetuple()[0], cs.date.timetuple()[1], cs.date.timetuple()[2], 0, 0, 0, 0, 0, 0]) if akc(cs.author) in co_day_auth_aggr: try: l = [timegetter(x) for x in co_day_auth_aggr[akc(cs.author)]['data']] time_pos = l.index(k) except ValueError: time_pos = False if time_pos >= 0 and time_pos is not False: datadict = \ co_day_auth_aggr[akc(cs.author)]['data'][time_pos] datadict["commits"] += 1 datadict["added"] += len(cs.added) datadict["changed"] += len(cs.changed) datadict["removed"] += len(cs.removed) else: if k >= ts_min_y and k <= ts_max_y or skip_date_limit: datadict = {"time": k, "commits": 1, "added": len(cs.added), "changed": len(cs.changed), "removed": len(cs.removed), } co_day_auth_aggr[akc(cs.author)]['data']\ .append(datadict) else: if k >= ts_min_y and k <= ts_max_y or skip_date_limit: co_day_auth_aggr[akc(cs.author)] = { "label": akc(cs.author), "data": [{"time":k, "commits":1, "added":len(cs.added), "changed":len(cs.changed), "removed":len(cs.removed), }], "schema": ["commits"], } #gather all data by day if k in commits_by_day_aggregate: commits_by_day_aggregate[k] += 1 else: commits_by_day_aggregate[k] = 1 overview_data = sorted(commits_by_day_aggregate.items(), key=itemgetter(0)) if not co_day_auth_aggr: co_day_auth_aggr[akc(repo.contact)] = { "label": akc(repo.contact), "data": [0, 1], "schema": ["commits"], } stats = cur_stats if cur_stats else Statistics() stats.commit_activity = json.dumps(co_day_auth_aggr) stats.commit_activity_combined = json.dumps(overview_data) log.debug('last revison %s', last_rev) leftovers = len(repo.revisions[last_rev:]) log.debug('revisions to parse %s', leftovers) if last_rev == 0 or leftovers < parse_limit: log.debug('getting code trending stats') stats.languages = json.dumps(__get_codes_stats(repo_name)) try: stats.repository = dbrepo stats.stat_on_revision = last_cs.revision if last_cs else 0 sa.add(stats) sa.commit() except: log.error(traceback.format_exc()) sa.rollback() lock.release() return False #final release lock.release() #execute another task if celery is enabled if len(repo.revisions) > 1 and CELERY_ON: run_task(get_commits_stats, repo_name, ts_min_y, ts_max_y) return True except LockHeld: log.info('LockHeld') return 'Task with key %s already running' % lockkey