def _handle(self, *args, **options): if len(args) != 2: raise CommandError("Need to specify (only) branch and changeset") (project, changeset) = args # get reference to repo rdm = RefDataManager() repos = filter(lambda x: x['name'] == project, rdm.get_all_repository_info()) if not repos: raise CommandError("No project found named '%s'" % project) repo = repos[0] # make sure all tasks are run synchronously / immediately settings.CELERY_ALWAYS_EAGER = True # get hg pushlog pushlog_url = '%s/json-pushes/?full=1&version=2' % repo['url'] # ingest this particular revision for this project process = HgPushlogProcess() # Use the actual push SHA, in case the changeset specified was a tag # or branch name (eg tip). HgPushlogProcess returns the full SHA. push_sha = process.run(pushlog_url, project, changeset=changeset)[:12] Builds4hJobsProcess().run(project_filter=project, revision_filter=push_sha, job_group_filter=options['filter_job_group']) PendingJobsProcess().run(project_filter=project, revision_filter=push_sha, job_group_filter=options['filter_job_group']) RunningJobsProcess().run(project_filter=project, revision_filter=push_sha, job_group_filter=options['filter_job_group'])
def _handle(self, *args, **options): if len(args) != 2: raise CommandError("Need to specify (only) branch and changeset") (project, changeset) = args # get reference to repo rdm = RefDataManager() repos = filter(lambda x: x['name'] == project, rdm.get_all_repository_info()) if not repos: raise CommandError("No project found named '%s'" % project) repo = repos[0] # make sure all tasks are run synchronously / immediately settings.CELERY_ALWAYS_EAGER = True # get hg pushlog pushlog_url = '%s/json-pushes/?full=1' % repo['url'] # ingest this particular revision for this project process = HgPushlogProcess() process.run(pushlog_url, project, changeset=changeset) self._process_all_objects_for_project(project) Builds4hJobsProcess().run(filter_to_project=project, filter_to_revision=changeset) PendingJobsProcess().run(filter_to_project=project, filter_to_revision=changeset) RunningJobsProcess().run(filter_to_project=project, filter_to_revision=changeset) self._process_all_objects_for_project(project)
def fetch_push_logs(): """ Run several fetch_hg_push_log subtasks, one per repository """ rdm = RefDataManager() try: repos = filter(lambda x: x['url'], rdm.get_all_repository_info()) # create a group of subtasks and apply them g = group(fetch_hg_push_log.si(repo['name'], repo['url']) for repo in repos if repo['dvcs_type'] == 'hg') g() finally: rdm.disconnect()
def fetch_push_logs(): """ Run several fetch_hg_push_log subtasks, one per repository """ rdm = RefDataManager() try: repos = filter(lambda x: x['url'], rdm.get_all_repository_info()) for repo in repos: if repo['dvcs_type'] == 'hg': fetch_hg_push_log.apply_async(args=(repo['name'], repo['url']), routing_key='pushlog') finally: rdm.disconnect()
def list(self, request): """ Retrieves a list of bugs from the bugs cache search -- Mandatory term of search """ search_term = request.QUERY_PARAMS.get("search", None) if not search_term: return Response({"message": "the 'search' parameter is mandatory"}, status=400) rdm = RefDataManager() try: suggested_bugs = rdm.get_bug_suggestions(search_term) finally: rdm.disconnect() return Response(suggested_bugs)
def handle(self, *args, **options): repositories = Repository.objects.filter(active_status='active') if options['repo_name']: repositories = repositories.filter(name=options['repo_name']) if options['codebase']: repositories = repositories.filter(codebase=options['codebase']) if options['group']: repositories = repositories.filter( repository_group__name=options['group']) repo_ids = repositories.values_list('id', flat=True) refdata = RefDataManager() for repo_id in repo_ids: refdata.update_repository_version(repo_id)
def fetch_push_logs(): """ Run several fetch_hg_push_log subtasks, one per repository """ rdm = RefDataManager() try: repos = filter(lambda x: x['url'], rdm.get_all_repository_info()) for repo in repos: if repo['dvcs_type'] == 'hg': fetch_hg_push_log.apply_async( args=(repo['name'], repo['url']), routing_key='pushlog' ) finally: rdm.disconnect()
def run(self): # this is the last day we fetched bugs from bugzilla bug_list = [] offset = 0 limit = 500 while True: # fetch the bugzilla service until we have an empty result paginated_url = "{0}&offset={1}&limit={2}".format( get_bz_source_url(), offset, limit ) response = self.extract(paginated_url) temp_bug_list = response.get('bugs', []) bug_list += temp_bug_list if len(temp_bug_list) < limit: break offset += limit if bug_list: for bug in bug_list: # drop the timezone indicator to avoid issues with mysql bug["last_change_time"] = bug["last_change_time"][0:19] with RefDataManager() as rdm: rdm.update_bugscache(bug_list)
def fetch_missing_push_logs(missing_pushlogs): """ Run several fetch_hg_push_log subtasks, one per repository """ rdm = RefDataManager() try: repos = filter(lambda x: x['url'], rdm.get_all_repository_info()) for repo in repos: if repo['dvcs_type'] == 'hg' and repo['name'] in missing_pushlogs: # we must get them one at a time, because if ANY are missing # from json-pushes, it'll return a 404 for the group. for resultset in missing_pushlogs[repo['name']]: fetch_missing_hg_push_logs.apply_async( args=(repo['name'], repo['url'], resultset), routing_key='pushlog') finally: rdm.disconnect()
def run(self): # this is the last day we fetched bugs from bugzilla last_fetched = cache.get("bz_last_fetched") curr_date = datetime.date.today() bug_list = [] if last_fetched: # if we have a last_fetched timestamp available # we don't need pagination. source_url = self._get_bz_source_url(last_fetched) response = self.extract(source_url) if response: bug_list = response.get("bugs", []) else: offset = 0 limit = 500 # fetch new pages no more than 30 times # this is a safe guard to not generate an infinite loop # in case something went wrong for i in range(1, 30 + 1): # fetch the bugzilla service until we have an empty result paginated_url = "{0}&offset={1}&limit={2}".format(self._get_bz_source_url(), offset, limit) response = self.extract(paginated_url) temp_bug_list = response.get("bugs", []) bug_list += temp_bug_list if len(temp_bug_list) < limit: break else: offset += limit if bug_list: # store the new date for one day cache.set("bz_last_fetched", curr_date, 60 * 60 * 24) rdm = RefDataManager() try: rdm.update_bugscache(bug_list) finally: rdm.disconnect()
def fetch_missing_push_logs(missing_pushlogs): """ Run several fetch_hg_push_log subtasks, one per repository """ rdm = RefDataManager() try: repos = filter(lambda x: x['url'], rdm.get_all_repository_info()) for repo in repos: if repo['dvcs_type'] == 'hg' and repo['name'] in missing_pushlogs: # we must get them one at a time, because if ANY are missing # from json-pushes, it'll return a 404 for the group. for resultset in missing_pushlogs[repo['name']]: fetch_missing_hg_push_logs.apply_async(args=( repo['name'], repo['url'], resultset ), routing_key='pushlog' ) finally: rdm.disconnect()
def list(self, request): """ Retrieves a list of bugs from the bugs cache search -- Mandatory term of search """ search_term = request.QUERY_PARAMS.get("search", None) if not search_term: return Response({"message": "the 'search' parameter is mandatory"}, status=400) with RefDataManager() as rdm: return Response(rdm.get_bug_suggestions(search_term))
def run(self): # this is the last day we fetched bugs from bugzilla bug_list = [] offset = 0 limit = 500 # fetch new pages no more than 30 times # this is a safe guard to not generate an infinite loop # in case something went wrong for i in range(1, 30 + 1): # fetch the bugzilla service until we have an empty result paginated_url = "{0}&offset={1}&limit={2}".format( get_bz_source_url(), offset, limit) response = self.extract(paginated_url) temp_bug_list = response.get('bugs', []) bug_list += temp_bug_list if len(temp_bug_list) < limit: break else: offset += limit if bug_list: for bug in bug_list: # drop the timezone indicator to avoid issues with mysql bug["last_change_time"] = bug["last_change_time"][0:19] rdm = RefDataManager() try: rdm.update_bugscache(bug_list) finally: rdm.disconnect()
def list(self, request): """ Retrieves a list of bugs from the bugs cache search -- Mandatory term of search status -- Optional filter on the status. Can be 'open' or 'closed'. Open by default """ search_term = request.QUERY_PARAMS.get("search", None) if not search_term: return Response({"message": "the 'search' parameter is mandatory"}, status=400) status = request.QUERY_PARAMS.get("status", "open") if not status in ("open", "closed"): return Response({"message": "status must be 'open' or 'closed'"}, status=400) open_only = True if status == "open" else False rdm = RefDataManager() try: suggested_bugs = rdm.get_bug_suggestions(search_term, open_only) finally: rdm.disconnect() return Response(suggested_bugs)
def list(self, request): with RefDataManager() as rdm: option_collection_hash = rdm.get_all_option_collections() ret = [] for (option_hash, val) in option_collection_hash.iteritems(): ret.append({ 'option_collection_hash': option_hash, 'options': [{ 'name': name } for name in val['opt'].split()] }) return Response(ret)
def run(self): # this is the last day we fetched bugs from bugzilla bug_list = [] offset = 0 limit = 500 # fetch new pages no more than 30 times # this is a safe guard to not generate an infinite loop # in case something went wrong for i in range(1, 30+1): # fetch the bugzilla service until we have an empty result paginated_url = "{0}&offset={1}&limit={2}".format( get_bz_source_url(), offset, limit ) response = self.extract(paginated_url) temp_bug_list = response.get('bugs', []) bug_list += temp_bug_list if len(temp_bug_list) < limit: break else: offset += limit if bug_list: for bug in bug_list: # drop the timezone indicator to avoid issues with mysql bug["last_change_time"] = bug["last_change_time"][0:19] rdm = RefDataManager() try: rdm.update_bugscache(bug_list) finally: rdm.disconnect()
def refdata(request): """returns a patched RefDataManager for testing purpose""" from treeherder.model.derived import RefDataManager from tests.conftest import add_test_procs_file refdata = RefDataManager() proc_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'refdata_test.json') add_test_procs_file(refdata.dhub, 'reference', proc_path) def fin(): refdata.disconnect() request.addfinalizer(fin) return refdata
def parse_log(project, job_id, result_set_id, check_errors=False): """ Call ArtifactBuilderCollection on the given job. """ pattern_obj = re.compile('\d+:\d+:\d+\s+') jm = JobsModel(project=project) rdm = RefDataManager() open_bugs_cache = {} closed_bugs_cache = {} status_publisher = JobStatusPublisher(settings.BROKER_URL) failure_publisher = JobFailurePublisher(settings.BROKER_URL) try: # return the resultset with the job id to identify if the UI wants # to fetch the whole thing. resultset = jm.get_result_set_by_id(result_set_id=result_set_id)[0] del(resultset["active_status"]) del(resultset["revision_hash"]) log_references = jm.get_log_references(job_id) # we may have many log references per job for log in log_references: # parse a log given its url artifact_bc = ArtifactBuilderCollection( log['url'], check_errors=check_errors, ) artifact_bc.parse() artifact_list = [] for name, artifact in artifact_bc.artifacts.items(): artifact_list.append((job_id, name, 'json', json.dumps(artifact))) if check_errors: # I'll try to begin with a full_text search on the entire row all_errors = artifact_bc.artifacts['Structured Log']['step_data']['all_errors'] open_bugs_suggestions = {} closed_bugs_suggestions = {} for err in all_errors: # remove timestamp clean_line = pattern_obj.sub('', err['line']) if clean_line not in open_bugs_cache: open_bugs_cache[clean_line] = rdm.get_suggested_bugs( clean_line) if clean_line not in closed_bugs_cache: closed_bugs_cache[clean_line] = rdm.get_suggested_bugs( clean_line, open_bugs=False) open_bugs_suggestions[ err['line'] ] = open_bugs_cache[clean_line] closed_bugs_suggestions[ err['line'] ] = closed_bugs_cache[clean_line] artifact_list.append((job_id, 'Open bugs', 'json', json.dumps(open_bugs_suggestions))) artifact_list.append((job_id, 'Closed bugs', 'json', json.dumps(closed_bugs_suggestions))) # store the artifacts generated jm.store_job_artifact(artifact_list) status_publisher.publish(job_id, resultset, project, 'processed') if check_errors: failure_publisher.publish(job_id, project) finally: rdm.disconnect() jm.disconnect() status_publisher.disconnect() failure_publisher.disconnect()