def _sync_artifact_as_log(self, jobstep, job_name, build_no, artifact):
job = jobstep.job
logsource, created = get_or_create(LogSource, where={
'name': artifact['displayPath'],
'job': job,
'step': jobstep,
}, defaults={
'project': job.project,
'date_created': job.date_started,
})
url = '{base}/job/{job}/{build}/artifact/{artifact}'.format(
base=self.base_url, job=job_name,
build=build_no, artifact=artifact['relativePath'],
)
offset = 0
resp = requests.get(url, stream=True, timeout=15)
iterator = resp.iter_content()
for chunk in chunked(iterator, LOG_CHUNK_SIZE):
chunk_size = len(chunk)
chunk, _ = create_or_update(LogChunk, where={
'source': logsource,
'offset': offset,
}, values={
'job': job,
'project': job.project,
'size': chunk_size,
'text': chunk,
})
offset += chunk_size
publish_logchunk_update(chunk)
def _sync_log(self, jobstep, name, job_name, build_no):
job = jobstep.job
# TODO(dcramer): this doesnt handle concurrency
logsource, created = get_or_create(LogSource, where={
'name': name,
'job': job,
}, defaults={
'step': jobstep,
'project': jobstep.project,
'date_created': jobstep.date_started,
})
if created:
offset = 0
else:
offset = jobstep.data.get('log_offset', 0)
url = '{base}/job/{job}/{build}/logText/progressiveHtml/'.format(
base=self.base_url,
job=job_name,
build=build_no,
)
resp = requests.get(
url, params={'start': offset}, stream=True, timeout=15)
log_length = int(resp.headers['X-Text-Size'])
# When you request an offset that doesnt exist in the build log, Jenkins
# will instead return the entire log. Jenkins also seems to provide us
# with X-Text-Size which indicates the total size of the log
if offset > log_length:
return
iterator = resp.iter_content()
# XXX: requests doesnt seem to guarantee chunk_size, so we force it
# with our own helper
for chunk in chunked(iterator, LOG_CHUNK_SIZE):
chunk_size = len(chunk)
chunk, _ = create_or_update(LogChunk, where={
'source': logsource,
'offset': offset,
}, values={
'job': job,
'project': job.project,
'size': chunk_size,
'text': chunk,
})
offset += chunk_size
publish_logchunk_update(chunk)
# We **must** track the log offset externally as Jenkins embeds encoded
# links and we cant accurately predict the next `start` param.
jobstep.data['log_offset'] = log_length
db.session.add(jobstep)
# Jenkins will suggest to us that there is more data when the job has
# yet to complete
return True if resp.headers.get('X-More-Data') == 'true' else None
def _sync_log(self, jobstep, name, job_name, build_no):
job = jobstep.job
# TODO(dcramer): this doesnt handle concurrency
logsource, created = get_or_create(LogSource, where={
'name': name,
'job': job,
}, defaults={
'step': jobstep,
'project': jobstep.project,
'date_created': jobstep.date_started,
})
if created:
offset = 0
else:
offset = jobstep.data.get('log_offset', 0)
url = '{base}/job/{job}/{build}/logText/progressiveHtml/'.format(
base=self.base_url,
job=job_name,
build=build_no,
)
resp = requests.get(
url, params={'start': offset}, stream=True, timeout=15)
log_length = int(resp.headers['X-Text-Size'])
# When you request an offset that doesnt exist in the build log, Jenkins
# will instead return the entire log. Jenkins also seems to provide us
# with X-Text-Size which indicates the total size of the log
if offset > log_length:
return
iterator = resp.iter_content()
# XXX: requests doesnt seem to guarantee chunk_size, so we force it
# with our own helper
for chunk in chunked(iterator, LOG_CHUNK_SIZE):
chunk_size = len(chunk)
chunk, _ = create_or_update(LogChunk, where={
'source': logsource,
'offset': offset,
}, values={
'job': job,
'project': job.project,
'size': chunk_size,
'text': chunk,
})
offset += chunk_size
publish_logchunk_update(chunk)
# We **must** track the log offset externally as Jenkins embeds encoded
# links and we cant accurately predict the next `start` param.
jobstep.data['log_offset'] = log_length
db.session.add(jobstep)
# Jenkins will suggest to us that there is more data when the job has
# yet to complete
return True if resp.headers.get('X-More-Data') == 'true' else None
def _sync_artifact_as_log(self, jobstep, artifact):
job = jobstep.job
logsource, created = get_or_create(LogSource, where={
'name': artifact['displayPath'],
'job': job,
'step': jobstep,
}, defaults={
'project': job.project,
'date_created': job.date_started,
})
job_name = jobstep.data['job_name']
build_no = jobstep.data['build_no']
url = '{base}/job/{job}/{build}/artifact/{artifact}'.format(
base=self.base_url, job=job_name,
build=build_no, artifact=artifact['relativePath'],
)
offset = 0
resp = requests.get(url, stream=True, timeout=15)
iterator = resp.iter_content()
for chunk in chunked(iterator, LOG_CHUNK_SIZE):
chunk_size = len(chunk)
chunk, _ = create_or_update(LogChunk, where={
'source': logsource,
'offset': offset,
}, values={
'job': job,
'project': job.project,
'size': chunk_size,
'text': chunk,
})
offset += chunk_size
publish_logchunk_update(chunk)