def run(self):
# this is the last day we fetched bugs from bugzilla
bug_list = []
offset = 0
limit = 500
# fetch new pages no more than 30 times
# this is a safe guard to not generate an infinite loop
# in case something went wrong
for i in range(1, 30 + 1):
# fetch the bugzilla service until we have an empty result
paginated_url = "{0}&offset={1}&limit={2}".format(
get_bz_source_url(), offset, limit)
response = self.extract(paginated_url)
temp_bug_list = response.get('bugs', [])
bug_list += temp_bug_list
if len(temp_bug_list) < limit:
break
else:
offset += limit
if bug_list:
for bug in bug_list:
# drop the timezone indicator to avoid issues with mysql
bug["last_change_time"] = bug["last_change_time"][0:19]
rdm = RefDataManager()
try:
rdm.update_bugscache(bug_list)
finally:
rdm.disconnect()
def fetch_push_logs():
"""
Run several fetch_hg_push_log subtasks, one per repository
"""
rdm = RefDataManager()
try:
repos = filter(lambda x: x['url'], rdm.get_all_repository_info())
# create a group of subtasks and apply them
g = group(fetch_hg_push_log.si(repo['name'], repo['url'])
for repo in repos if repo['dvcs_type'] == 'hg')
g()
finally:
rdm.disconnect()
def fetch_push_logs():
"""
Run several fetch_hg_push_log subtasks, one per repository
"""
rdm = RefDataManager()
try:
repos = filter(lambda x: x['url'], rdm.get_all_repository_info())
for repo in repos:
if repo['dvcs_type'] == 'hg':
fetch_hg_push_log.apply_async(args=(repo['name'], repo['url']),
routing_key='pushlog')
finally:
rdm.disconnect()
def list(self, request):
"""
Retrieves a list of bugs from the bugs cache
search -- Mandatory term of search
"""
search_term = request.QUERY_PARAMS.get("search", None)
if not search_term:
return Response({"message": "the 'search' parameter is mandatory"}, status=400)
rdm = RefDataManager()
try:
suggested_bugs = rdm.get_bug_suggestions(search_term)
finally:
rdm.disconnect()
return Response(suggested_bugs)
def fetch_push_logs():
"""
Run several fetch_hg_push_log subtasks, one per repository
"""
rdm = RefDataManager()
try:
repos = filter(lambda x: x['url'], rdm.get_all_repository_info())
for repo in repos:
if repo['dvcs_type'] == 'hg':
fetch_hg_push_log.apply_async(
args=(repo['name'], repo['url']),
routing_key='pushlog'
)
finally:
rdm.disconnect()
def list(self, request):
"""
Retrieves a list of bugs from the bugs cache
search -- Mandatory term of search
"""
search_term = request.QUERY_PARAMS.get("search", None)
if not search_term:
return Response({"message": "the 'search' parameter is mandatory"},
status=400)
rdm = RefDataManager()
try:
suggested_bugs = rdm.get_bug_suggestions(search_term)
finally:
rdm.disconnect()
return Response(suggested_bugs)
def fetch_missing_push_logs(missing_pushlogs):
"""
Run several fetch_hg_push_log subtasks, one per repository
"""
rdm = RefDataManager()
try:
repos = filter(lambda x: x['url'], rdm.get_all_repository_info())
for repo in repos:
if repo['dvcs_type'] == 'hg' and repo['name'] in missing_pushlogs:
# we must get them one at a time, because if ANY are missing
# from json-pushes, it'll return a 404 for the group.
for resultset in missing_pushlogs[repo['name']]:
fetch_missing_hg_push_logs.apply_async(
args=(repo['name'], repo['url'], resultset),
routing_key='pushlog')
finally:
rdm.disconnect()
def run(self):
# this is the last day we fetched bugs from bugzilla
last_fetched = cache.get("bz_last_fetched")
curr_date = datetime.date.today()
bug_list = []
if last_fetched:
# if we have a last_fetched timestamp available
# we don't need pagination.
source_url = self._get_bz_source_url(last_fetched)
response = self.extract(source_url)
if response:
bug_list = response.get("bugs", [])
else:
offset = 0
limit = 500
# fetch new pages no more than 30 times
# this is a safe guard to not generate an infinite loop
# in case something went wrong
for i in range(1, 30 + 1):
# fetch the bugzilla service until we have an empty result
paginated_url = "{0}&offset={1}&limit={2}".format(self._get_bz_source_url(), offset, limit)
response = self.extract(paginated_url)
temp_bug_list = response.get("bugs", [])
bug_list += temp_bug_list
if len(temp_bug_list) < limit:
break
else:
offset += limit
if bug_list:
# store the new date for one day
cache.set("bz_last_fetched", curr_date, 60 * 60 * 24)
rdm = RefDataManager()
try:
rdm.update_bugscache(bug_list)
finally:
rdm.disconnect()
def fetch_missing_push_logs(missing_pushlogs):
"""
Run several fetch_hg_push_log subtasks, one per repository
"""
rdm = RefDataManager()
try:
repos = filter(lambda x: x['url'], rdm.get_all_repository_info())
for repo in repos:
if repo['dvcs_type'] == 'hg' and repo['name'] in missing_pushlogs:
# we must get them one at a time, because if ANY are missing
# from json-pushes, it'll return a 404 for the group.
for resultset in missing_pushlogs[repo['name']]:
fetch_missing_hg_push_logs.apply_async(args=(
repo['name'],
repo['url'],
resultset
),
routing_key='pushlog'
)
finally:
rdm.disconnect()
def list(self, request):
"""
Retrieves a list of bugs from the bugs cache
search -- Mandatory term of search
status -- Optional filter on the status. Can be 'open' or 'closed'. Open by default
"""
search_term = request.QUERY_PARAMS.get("search", None)
if not search_term:
return Response({"message": "the 'search' parameter is mandatory"}, status=400)
status = request.QUERY_PARAMS.get("status", "open")
if not status in ("open", "closed"):
return Response({"message": "status must be 'open' or 'closed'"}, status=400)
open_only = True if status == "open" else False
rdm = RefDataManager()
try:
suggested_bugs = rdm.get_bug_suggestions(search_term, open_only)
finally:
rdm.disconnect()
return Response(suggested_bugs)
def run(self):
# this is the last day we fetched bugs from bugzilla
bug_list = []
offset = 0
limit = 500
# fetch new pages no more than 30 times
# this is a safe guard to not generate an infinite loop
# in case something went wrong
for i in range(1, 30+1):
# fetch the bugzilla service until we have an empty result
paginated_url = "{0}&offset={1}&limit={2}".format(
get_bz_source_url(),
offset,
limit
)
response = self.extract(paginated_url)
temp_bug_list = response.get('bugs', [])
bug_list += temp_bug_list
if len(temp_bug_list) < limit:
break
else:
offset += limit
if bug_list:
for bug in bug_list:
# drop the timezone indicator to avoid issues with mysql
bug["last_change_time"] = bug["last_change_time"][0:19]
rdm = RefDataManager()
try:
rdm.update_bugscache(bug_list)
finally:
rdm.disconnect()
def parse_log(project, job_id, result_set_id, check_errors=False):
"""
Call ArtifactBuilderCollection on the given job.
"""
pattern_obj = re.compile('\d+:\d+:\d+\s+')
jm = JobsModel(project=project)
rdm = RefDataManager()
open_bugs_cache = {}
closed_bugs_cache = {}
status_publisher = JobStatusPublisher(settings.BROKER_URL)
failure_publisher = JobFailurePublisher(settings.BROKER_URL)
try:
# return the resultset with the job id to identify if the UI wants
# to fetch the whole thing.
resultset = jm.get_result_set_by_id(result_set_id=result_set_id)[0]
del(resultset["active_status"])
del(resultset["revision_hash"])
log_references = jm.get_log_references(job_id)
# we may have many log references per job
for log in log_references:
# parse a log given its url
artifact_bc = ArtifactBuilderCollection(
log['url'],
check_errors=check_errors,
)
artifact_bc.parse()
artifact_list = []
for name, artifact in artifact_bc.artifacts.items():
artifact_list.append((job_id, name, 'json', json.dumps(artifact)))
if check_errors:
# I'll try to begin with a full_text search on the entire row
all_errors = artifact_bc.artifacts['Structured Log']['step_data']['all_errors']
open_bugs_suggestions = {}
closed_bugs_suggestions = {}
for err in all_errors:
# remove timestamp
clean_line = pattern_obj.sub('', err['line'])
if clean_line not in open_bugs_cache:
open_bugs_cache[clean_line] = rdm.get_suggested_bugs(
clean_line)
if clean_line not in closed_bugs_cache:
closed_bugs_cache[clean_line] = rdm.get_suggested_bugs(
clean_line, open_bugs=False)
open_bugs_suggestions[ err['line'] ] = open_bugs_cache[clean_line]
closed_bugs_suggestions[ err['line'] ] = closed_bugs_cache[clean_line]
artifact_list.append((job_id, 'Open bugs', 'json', json.dumps(open_bugs_suggestions)))
artifact_list.append((job_id, 'Closed bugs', 'json', json.dumps(closed_bugs_suggestions)))
# store the artifacts generated
jm.store_job_artifact(artifact_list)
status_publisher.publish(job_id, resultset, project, 'processed')
if check_errors:
failure_publisher.publish(job_id, project)
finally:
rdm.disconnect()
jm.disconnect()
status_publisher.disconnect()
failure_publisher.disconnect()
