def generate_request_body(self): """ Create the data structure required by tbpl's submitBuildStar.php script It's used by both the bug_job_map endpoint and the job note endpoint. """ jm = JobsModel(self.project) try: buildapi_artifact = jm.get_job_artifact_list(0, 1, { 'job_id': set([("=", self.job_id)]), 'name': set([("=", "buildapi_complete")]) })[0] job_data = jm.get_job(self.job_id)[0] finally: jm.disconnect() note = "" if self.bug_id: note = "Bug {0}".format(self.bug_id) if self.classification_id: if note: note += " - " note += FailureClassification.objects.get( id=self.classification_id).name if self.note: if note: note += " - " note += self.note self.body = { "id": buildapi_artifact["blob"]["id"], "machinename": job_data["machine_name"], "starttime": int(job_data["start_timestamp"]), "note": note, "who": self.who }
def cycle_data(max_iterations=50, debug=False): projects = Repository.objects.all().values_list('name', flat=True) for project in projects: jm = JobsModel(project) sql_targets = {} if debug: print "Cycling Database: {0}".format(project) cycle_iterations = max_iterations while cycle_iterations > 0: sql_targets = jm.cycle_data(sql_targets) if debug: print "Iterations: {0}".format(str(cycle_iterations)) print "sql_targets" print sql_targets cycle_iterations -= 1 # No more items to delete if sql_targets['total_count'] == 0: cycle_iterations = 0 jm.disconnect()
def generate_request_body(self): """ Create the data structure that will be sent to ElasticSearch. """ jm = JobsModel(self.project) try: job_data = jm.get_job(self.job_id)[0] option_collection = jm.refdata_model.get_all_option_collections() revision_list = jm.get_resultset_revisions_list( job_data["result_set_id"]) buildapi_artifact = jm.get_job_artifact_list( 0, 1, { 'job_id': set([("=", self.job_id)]), 'name': set([("=", "buildapi")]) }) if buildapi_artifact: buildname = buildapi_artifact[0]["blob"]["buildername"] else: # OrangeFactor needs a buildname to be set or it skips the failure # classification, so we make one up for non-buildbot jobs. buildname = 'non-buildbot %s test %s' % ( job_data["platform"], job_data["job_type_name"]) finally: jm.disconnect() self.body = { "buildname": buildname, "machinename": job_data["machine_name"], "os": job_data["platform"], # I'm using the request time date here, as start time is not # available for pending jobs "date": datetime.fromtimestamp(int( job_data["submit_timestamp"])).strftime("%Y-%m-%d"), "type": job_data["job_type_name"], "buildtype": option_collection[job_data["option_collection_hash"]]["opt"], # Intentionally using strings for starttime, bug, timestamp for compatibility # with TBPL's legacy output format. "starttime": str(job_data["start_timestamp"]), "tree": self.project, "rev": revision_list[0]["revision"], "bug": str(self.bug_id), "who": self.who, "timestamp": str(self.submit_timestamp), "logfile": "00000000" }
def use_jobs_model(*args, **kwargs): project = kwargs["project"] jm = JobsModel(project) try: return model_func(*args, jm=jm, **kwargs) finally: jm.disconnect()
def populate_performance_series(project, series_type, series_data): jm = JobsModel(project) for t_range in settings.TREEHERDER_PERF_SERIES_TIME_RANGES: for signature in series_data: jm.store_performance_series(t_range['seconds'], series_type, signature, series_data[signature]) jm.disconnect()
def populate_performance_series(project, series_type, series_data): jm = JobsModel(project) for t_range in settings.TREEHERDER_PERF_SERIES_TIME_RANGES: for signature in series_data: jm.store_performance_series( t_range['seconds'], series_type, signature, series_data[signature] ) jm.disconnect()
def publish_to_pulse(project, ids, data_type): jm = JobsModel(project) # Get appropriate data for data_type # using the ids provided data = [] if data_type == 'result_set': data = jm.get_result_set_list_by_ids(ids) jm.disconnect()
def calculate_eta(sample_window_seconds=21600, debug=False): projects = Repository.objects.all().values_list('name', flat=True) for project in projects: jm = JobsModel(project) jm.calculate_eta(sample_window_seconds, debug) jm.disconnect()
def process_objects(limit=None): """ Process a number of objects from the objectstore and load them to the jobs store """ # default limit to 100 limit = limit or 100 for ds in Datasource.objects.all(): jm = JobsModel(ds.project) try: jm.process_objects(limit) finally: jm.disconnect()
def generate_request_body(self): """ Create the data structure required by tbpl's starcomment.php script """ jm = JobsModel(self.project) try: buildapi_artifact = jm.get_job_artifact_list( 0, 1, { 'job_id': set([("=", self.job_id)]), 'name': set([("=", "buildapi")]) })[0] job_data = jm.get_job(self.job_id)[0] option_collection = jm.refdata_model.get_all_option_collections() revision_list = jm.get_resultset_revisions_list( job_data["result_set_id"]) finally: jm.disconnect() self.body = { "buildname": buildapi_artifact["blob"]["buildername"], "machinename": job_data["machine_name"], "os": job_data["platform"], # I'm using the request time date here, as start time is not # available for pending jobs "date": datetime.fromtimestamp(int( job_data["submit_timestamp"])).strftime("%Y-%m-%d"), "type": job_data["job_type_name"], "buildtype": option_collection[job_data["option_collection_hash"]]["opt"], "starttime": int(job_data["start_timestamp"]), # "logfile": "", "tree": self.project, "rev": revision_list[0]["revision"], "comment": "Bug {0}".format(self.bug_id), "who": self.who, "timestamp": self.submit_timestamp, "logfile": "00000000" }
def unclassified_failure_count(projects=None): if not projects: projects = Repository.objects.all().values_list('name', flat=True) unclassified_failure_publisher = UnclassifiedFailureCountPublisher(settings.BROKER_URL) for project in projects: jm = JobsModel(project) count = jm.get_unclassified_failure_count() count_excluded = jm.get_unclassified_failure_count_excluded() unclassified_failure_publisher.publish(project, count, count_excluded) jm.disconnect() unclassified_failure_publisher.disconnect()
def publish_to_pulse(project, ids, data_type): # If we don't have a publisher (because of missing configs), then we can't # publish any pulse messages. This is okay, local installs etc. doesn't # need to publish on pulse, and requiring a pulse user is adding more # overhead to an already large development setup process. if not publisher: return jm = JobsModel(project) try: # Publish messages with new result-sets if data_type == 'result_set': # Get appropriate data for data_type # using the ids provided for entry in jm.get_result_set_list_by_ids(ids): repository = jm.refdata_model.get_repository_info(entry['repository_id']) entry['repository_url'] = repository['url'] # Don't expose these properties, they are internal, at least that's # what I think without documentation I have no clue... what any of # this is del entry['revisions'] # Not really internal, but too big del entry['repository_id'] # Set required properties entry['version'] = 1 entry['project'] = project # Property revision_hash should already be there, I suspect it is the # result-set identifier... # publish the data to pulse publisher.new_result_set( message = entry, revision_hash = entry['revision_hash'], project = project ) # Basically, I have no idea what context this runs and was inherently # unable to make kombu with or without pyamqp, etc. confirm-publish, # so we're stuck with this super ugly hack where we just close the # connection so that if the process context is destroyed then at least # messages will still get published... Well, assuming nothing goes # wrong, because we're not using confirm channels for publishing... publisher.connection.release() finally: jm.disconnect()
def get_context_data(self, **kwargs): assert "repository" in kwargs and "revision" in kwargs context = super(ResultsetStatusView, self).get_context_data(**kwargs) jobs_model = JobsModel(kwargs['repository']) try: resultset_list = jobs_model.get_revision_resultset_lookup( [kwargs['revision']]) if not resultset_list: raise Http404("No resultset found for revision {0}".format( kwargs['revision'])) result_set_id = resultset_list[kwargs['revision']]['id'] resultset_status_dict = jobs_model.get_resultset_status( result_set_id) context['resultset_status_dict'] = resultset_status_dict return context finally: jobs_model.disconnect()
def publish_to_pulse(project, ids, data_type): # If we don't have a publisher (because of missing configs), then we can't # publish any pulse messages. This is okay, local installs etc. doesn't # need to publish on pulse, and requiring a pulse user is adding more # overhead to an already large development setup process. if not publisher: return jm = JobsModel(project) try: # Publish messages with new result-sets if data_type == 'result_set': # Get appropriate data for data_type # using the ids provided for entry in jm.get_result_set_list_by_ids(ids): # Don't expose these properties, they are internal, at least that's # what I think without documentation I have no clue... what any of # this is del entry['revisions'] # Not really internal, but too big del entry['repository_id'] # Set required properties entry['version'] = 1 entry['project'] = project # Property revision_hash should already be there, I suspect it is the # result-set identifier... # publish the data to pulse publisher.new_result_set( message = entry, revision_hash = entry['revision_hash'], project = project ) # Basically, I have no idea what context this runs and was inherently # unable to make kombu with or without pyamqp, etc. confirm-publish, # so we're stuck with this super ugly hack where we just close the # connection so that if the process context is destroyed then at least # messages will still get published... Well, assuming nothing goes # wrong, because we're not using confirm channels for publishing... publisher.connection.release() finally: jm.disconnect()
def process_objects(limit=None, project=None): """ Process a number of objects from the objectstore and load them to the jobs store """ # default limit to 100 limit = limit or 100 if project: projects_to_process = [project] else: projects_to_process = Datasource.objects.values_list( 'project', flat=True).distinct() for project in projects_to_process: jm = JobsModel(project) try: jm.process_objects(limit) finally: jm.disconnect()
def use_jobs_model(*args, **kwargs): project = kwargs["project"] try: jm = JobsModel(project) return model_func(*args, jm=jm, **kwargs) except DatasetNotFoundError as e: return Response( {"message": "No project with name {0}".format(project)}, status=404, ) except ObjectNotFoundException as e: return Response({"message": unicode(e)}, status=404) except Exception as e: # pragma nocover msg = {"message": unicode(e)} if settings.DEBUG: import traceback msg["traceback"] = traceback.format_exc() return Response(msg, status=500) finally: jm.disconnect()
def generate_request_body(self): """ Create the data structure required by tbpl's starcomment.php script """ jm = JobsModel(self.project) try: buildapi_artifact = jm.get_job_artifact_list(0, 1, { 'job_id': set([("=", self.job_id)]), 'name': set([("=", "buildapi")]) })[0] job_data = jm.get_job(self.job_id)[0] option_collection = jm.refdata_model.get_all_option_collections() revision_list = jm.get_resultset_revisions_list(job_data["result_set_id"]) finally: jm.disconnect() self.body = { "buildname": buildapi_artifact["blob"]["buildername"], "machinename": job_data["machine_name"], "os": job_data["platform"], # I'm using the request time date here, as start time is not # available for pending jobs "date": datetime.fromtimestamp( int(job_data["submit_timestamp"])).strftime("%Y-%m-%d"), "type": job_data["job_type_name"], "buildtype": option_collection[ job_data["option_collection_hash"] ]["opt"], "starttime": int(job_data["start_timestamp"]), # "logfile": "", "tree": self.project, "rev": revision_list[0]["revision"], "comment": "Bug {0}".format(self.bug_id), "who": self.who, "timestamp": self.submit_timestamp, "logfile": "00000000" }
def handle(self, *args, **options): self.is_debug = options['debug'] if options['cycle_interval']: cycle_interval = datetime.timedelta(days=options['cycle_interval']) else: cycle_interval = settings.DATA_CYCLE_INTERVAL self.debug("cycle interval: {0}".format(cycle_interval)) projects = Datasource.objects\ .filter(contenttype='jobs')\ .values_list('project', flat=True) for project in projects: self.debug("Cycling Database: {0}".format(project)) jm = JobsModel(project) try: num_deleted = jm.cycle_data(cycle_interval, options['chunk_size'], options['sleep_time']) self.debug("Deleted {0} resultsets from {1}".format( num_deleted, project)) finally: jm.disconnect()
def parse_log(project, job_id, result_set_id, check_errors=False): """ Call ArtifactBuilderCollection on the given job. """ pattern_obj = re.compile('\d+:\d+:\d+\s+') jm = JobsModel(project=project) rdm = RefDataManager() open_bugs_cache = {} closed_bugs_cache = {} status_publisher = JobStatusPublisher(settings.BROKER_URL) failure_publisher = JobFailurePublisher(settings.BROKER_URL) try: # return the resultset with the job id to identify if the UI wants # to fetch the whole thing. resultset = jm.get_result_set_by_id(result_set_id=result_set_id)[0] del(resultset["active_status"]) del(resultset["revision_hash"]) log_references = jm.get_log_references(job_id) # we may have many log references per job for log in log_references: # parse a log given its url artifact_bc = ArtifactBuilderCollection( log['url'], check_errors=check_errors, ) artifact_bc.parse() artifact_list = [] for name, artifact in artifact_bc.artifacts.items(): artifact_list.append((job_id, name, 'json', json.dumps(artifact))) if check_errors: # I'll try to begin with a full_text search on the entire row all_errors = artifact_bc.artifacts['Structured Log']['step_data']['all_errors'] open_bugs_suggestions = {} closed_bugs_suggestions = {} for err in all_errors: # remove timestamp clean_line = pattern_obj.sub('', err['line']) if clean_line not in open_bugs_cache: open_bugs_cache[clean_line] = rdm.get_suggested_bugs( clean_line) if clean_line not in closed_bugs_cache: closed_bugs_cache[clean_line] = rdm.get_suggested_bugs( clean_line, open_bugs=False) open_bugs_suggestions[ err['line'] ] = open_bugs_cache[clean_line] closed_bugs_suggestions[ err['line'] ] = closed_bugs_cache[clean_line] artifact_list.append((job_id, 'Open bugs', 'json', json.dumps(open_bugs_suggestions))) artifact_list.append((job_id, 'Closed bugs', 'json', json.dumps(closed_bugs_suggestions))) # store the artifacts generated jm.store_job_artifact(artifact_list) status_publisher.publish(job_id, resultset, project, 'processed') if check_errors: failure_publisher.publish(job_id, project) finally: rdm.disconnect() jm.disconnect() status_publisher.disconnect() failure_publisher.disconnect()
def generate_request_body(self): """ Create the data structure required by tbpl's submitBugzillaComment.php script This is triggered by a new bug-job association. """ jm = JobsModel(self.project) try: job = jm.get_job(self.job_id)[0] failures_artifacts = jm.get_job_artifact_list(0, 1, { 'job_id': set([('=', job['id'])]), 'name': set([('=', 'Bug suggestions')]), }) error_lines = [] for artifact in failures_artifacts: # a bug suggestion aritfact looks like this: # [{ "search": "my-error-line", "bugs": ....}] error_lines += [line["search"] for line in artifact["blob"]] bug_job_map = jm.get_bug_job_map_detail(self.job_id, self.bug_id) revision_list = jm.get_resultset_revisions_list( job["result_set_id"] ) buildapi_info = jm.get_job_artifact_list(0, 1, { 'job_id': set([("=", self.job_id)]), 'name': set([("=", "buildapi")]) }) finally: jm.disconnect() who = bug_job_map["who"]\ .replace("@", "[at]")\ .replace(".", "[dot]") submit_date = datetime.fromtimestamp(bug_job_map["submit_timestamp"])\ .replace(microsecond=0)\ .isoformat() job_description = { 'repository': self.project, 'who': who, 'submit_timestamp': submit_date, 'log': "{0}{1}/logviewer.html#?repo={2}&job_id={3}".format( settings.SITE_URL, settings.UI_PREFIX, self.project, self.job_id ), 'machine': job["machine_name"], 'revision': revision_list[0]["revision"], } if buildapi_info: job_description['buildname'] = buildapi_info[0]["blob"]["buildername"] body_comment = '\n'.join( ["{0}: {1}".format(k, v) for k, v in job_description.items()]) body_comment += '\n\n' body_comment += '\n'.join(error_lines) self.body = { "id": self.bug_id, "comment": body_comment }