def run(self): # this is the last day we fetched bugs from bugzilla bug_list = [] offset = 0 limit = 500 # fetch new pages no more than 30 times # this is a safe guard to not generate an infinite loop # in case something went wrong for i in range(1, 30 + 1): # fetch the bugzilla service until we have an empty result paginated_url = "{0}&offset={1}&limit={2}".format( get_bz_source_url(), offset, limit) response = self.extract(paginated_url) temp_bug_list = response.get('bugs', []) bug_list += temp_bug_list if len(temp_bug_list) < limit: break else: offset += limit if bug_list: for bug in bug_list: # drop the timezone indicator to avoid issues with mysql bug["last_change_time"] = bug["last_change_time"][0:19] rdm = RefDataManager() try: rdm.update_bugscache(bug_list) finally: rdm.disconnect()
def fetch_push_logs(): """ Run several fetch_hg_push_log subtasks, one per repository """ rdm = RefDataManager() try: repos = filter(lambda x: x['url'], rdm.get_all_repository_info()) # create a group of subtasks and apply them g = group(fetch_hg_push_log.si(repo['name'], repo['url']) for repo in repos if repo['dvcs_type'] == 'hg') g() finally: rdm.disconnect()
def fetch_push_logs(): """ Run several fetch_hg_push_log subtasks, one per repository """ rdm = RefDataManager() try: repos = filter(lambda x: x['url'], rdm.get_all_repository_info()) for repo in repos: if repo['dvcs_type'] == 'hg': fetch_hg_push_log.apply_async(args=(repo['name'], repo['url']), routing_key='pushlog') finally: rdm.disconnect()
def list(self, request): """ Retrieves a list of bugs from the bugs cache search -- Mandatory term of search """ search_term = request.QUERY_PARAMS.get("search", None) if not search_term: return Response({"message": "the 'search' parameter is mandatory"}, status=400) rdm = RefDataManager() try: suggested_bugs = rdm.get_bug_suggestions(search_term) finally: rdm.disconnect() return Response(suggested_bugs)
def fetch_push_logs(): """ Run several fetch_hg_push_log subtasks, one per repository """ rdm = RefDataManager() try: repos = filter(lambda x: x['url'], rdm.get_all_repository_info()) for repo in repos: if repo['dvcs_type'] == 'hg': fetch_hg_push_log.apply_async( args=(repo['name'], repo['url']), routing_key='pushlog' ) finally: rdm.disconnect()
def fetch_missing_push_logs(missing_pushlogs): """ Run several fetch_hg_push_log subtasks, one per repository """ rdm = RefDataManager() try: repos = filter(lambda x: x['url'], rdm.get_all_repository_info()) for repo in repos: if repo['dvcs_type'] == 'hg' and repo['name'] in missing_pushlogs: # we must get them one at a time, because if ANY are missing # from json-pushes, it'll return a 404 for the group. for resultset in missing_pushlogs[repo['name']]: fetch_missing_hg_push_logs.apply_async( args=(repo['name'], repo['url'], resultset), routing_key='pushlog') finally: rdm.disconnect()
def run(self): # this is the last day we fetched bugs from bugzilla last_fetched = cache.get("bz_last_fetched") curr_date = datetime.date.today() bug_list = [] if last_fetched: # if we have a last_fetched timestamp available # we don't need pagination. source_url = self._get_bz_source_url(last_fetched) response = self.extract(source_url) if response: bug_list = response.get("bugs", []) else: offset = 0 limit = 500 # fetch new pages no more than 30 times # this is a safe guard to not generate an infinite loop # in case something went wrong for i in range(1, 30 + 1): # fetch the bugzilla service until we have an empty result paginated_url = "{0}&offset={1}&limit={2}".format(self._get_bz_source_url(), offset, limit) response = self.extract(paginated_url) temp_bug_list = response.get("bugs", []) bug_list += temp_bug_list if len(temp_bug_list) < limit: break else: offset += limit if bug_list: # store the new date for one day cache.set("bz_last_fetched", curr_date, 60 * 60 * 24) rdm = RefDataManager() try: rdm.update_bugscache(bug_list) finally: rdm.disconnect()
def fetch_missing_push_logs(missing_pushlogs): """ Run several fetch_hg_push_log subtasks, one per repository """ rdm = RefDataManager() try: repos = filter(lambda x: x['url'], rdm.get_all_repository_info()) for repo in repos: if repo['dvcs_type'] == 'hg' and repo['name'] in missing_pushlogs: # we must get them one at a time, because if ANY are missing # from json-pushes, it'll return a 404 for the group. for resultset in missing_pushlogs[repo['name']]: fetch_missing_hg_push_logs.apply_async(args=( repo['name'], repo['url'], resultset ), routing_key='pushlog' ) finally: rdm.disconnect()
def list(self, request): """ Retrieves a list of bugs from the bugs cache search -- Mandatory term of search status -- Optional filter on the status. Can be 'open' or 'closed'. Open by default """ search_term = request.QUERY_PARAMS.get("search", None) if not search_term: return Response({"message": "the 'search' parameter is mandatory"}, status=400) status = request.QUERY_PARAMS.get("status", "open") if not status in ("open", "closed"): return Response({"message": "status must be 'open' or 'closed'"}, status=400) open_only = True if status == "open" else False rdm = RefDataManager() try: suggested_bugs = rdm.get_bug_suggestions(search_term, open_only) finally: rdm.disconnect() return Response(suggested_bugs)
def run(self): # this is the last day we fetched bugs from bugzilla bug_list = [] offset = 0 limit = 500 # fetch new pages no more than 30 times # this is a safe guard to not generate an infinite loop # in case something went wrong for i in range(1, 30+1): # fetch the bugzilla service until we have an empty result paginated_url = "{0}&offset={1}&limit={2}".format( get_bz_source_url(), offset, limit ) response = self.extract(paginated_url) temp_bug_list = response.get('bugs', []) bug_list += temp_bug_list if len(temp_bug_list) < limit: break else: offset += limit if bug_list: for bug in bug_list: # drop the timezone indicator to avoid issues with mysql bug["last_change_time"] = bug["last_change_time"][0:19] rdm = RefDataManager() try: rdm.update_bugscache(bug_list) finally: rdm.disconnect()
def parse_log(project, job_id, result_set_id, check_errors=False): """ Call ArtifactBuilderCollection on the given job. """ pattern_obj = re.compile('\d+:\d+:\d+\s+') jm = JobsModel(project=project) rdm = RefDataManager() open_bugs_cache = {} closed_bugs_cache = {} status_publisher = JobStatusPublisher(settings.BROKER_URL) failure_publisher = JobFailurePublisher(settings.BROKER_URL) try: # return the resultset with the job id to identify if the UI wants # to fetch the whole thing. resultset = jm.get_result_set_by_id(result_set_id=result_set_id)[0] del(resultset["active_status"]) del(resultset["revision_hash"]) log_references = jm.get_log_references(job_id) # we may have many log references per job for log in log_references: # parse a log given its url artifact_bc = ArtifactBuilderCollection( log['url'], check_errors=check_errors, ) artifact_bc.parse() artifact_list = [] for name, artifact in artifact_bc.artifacts.items(): artifact_list.append((job_id, name, 'json', json.dumps(artifact))) if check_errors: # I'll try to begin with a full_text search on the entire row all_errors = artifact_bc.artifacts['Structured Log']['step_data']['all_errors'] open_bugs_suggestions = {} closed_bugs_suggestions = {} for err in all_errors: # remove timestamp clean_line = pattern_obj.sub('', err['line']) if clean_line not in open_bugs_cache: open_bugs_cache[clean_line] = rdm.get_suggested_bugs( clean_line) if clean_line not in closed_bugs_cache: closed_bugs_cache[clean_line] = rdm.get_suggested_bugs( clean_line, open_bugs=False) open_bugs_suggestions[ err['line'] ] = open_bugs_cache[clean_line] closed_bugs_suggestions[ err['line'] ] = closed_bugs_cache[clean_line] artifact_list.append((job_id, 'Open bugs', 'json', json.dumps(open_bugs_suggestions))) artifact_list.append((job_id, 'Closed bugs', 'json', json.dumps(closed_bugs_suggestions))) # store the artifacts generated jm.store_job_artifact(artifact_list) status_publisher.publish(job_id, resultset, project, 'processed') if check_errors: failure_publisher.publish(job_id, project) finally: rdm.disconnect() jm.disconnect() status_publisher.disconnect() failure_publisher.disconnect()