def pubsub(instanceId=None):
jobs = None
output = ''
# We keep all jobs for 10 hours
redis_conn.zremrangebyscore(
'jobs:{}'.format(instanceId),
0,
time.time() - 36000
)
if instanceId is not None:
jobs = redis_conn.zrevrange(
'jobs:{}'.format(instanceId), 0, 0, withscores=True
)
if jobs:
for userId, job in jobs:
console = _read_console(job)
output += console\
.replace('\n', '<br />')\
.replace('#BEGIN#', '')\
.replace('#END#', '')
output = ansiconv.to_html(output)
# if '#END#' in console:
# Save job into the db auditlog
return Response('data: {}\n\n'.format(output),
mimetype='text/event-stream')
def pubsub(instanceId=None):
jobs = None
output = ''
# We keep all jobs for 10 hours
# get all jobs for an instance
jobs_key = 'jobs:{}'.format(instanceId)
jobs = redis_conn.hkeys(jobs_key)
# sort by jobid/time most recent first
jobs.sort(reverse=True)
# filter older jobs
old_jobs = filter(
lambda x: x < (time.time() - 3600), jobs)
# delete them
if old_jobs:
redis_conn.hdel(jobs_key, *old_jobs)
if instanceId is not None and jobs:
# timeStarted = jobs[0]
# timeStarted = jobs[0]
# userId = redis_conn.hget(
# jobs_key,
# timeStarted)
# for job in jobs:
job = jobs[0]
console = _read_console(job)
output += console\
.replace('\n', '<br />')\
.replace('#BEGIN#', '')\
.replace('#END#', '')
output = ansiconv.to_html(output)
# if '#END#' in console:
# Save job into the db auditlog
return Response('data: {}\n\n'.format(output),
mimetype='text/event-stream')
def pubsub(instanceId=None):
jobs = None
output = ''
# We keep all jobs for 10 hours
redis_conn.zremrangebyscore('jobs:{}'.format(instanceId), 0,
time.time() - 36000)
if instanceId is not None:
jobs = redis_conn.zrevrange('jobs:{}'.format(instanceId),
0,
0,
withscores=True)
if jobs:
for userId, job in jobs:
console = _read_console(job)
output += console\
.replace('\n', '<br />')\
.replace('#BEGIN#', '')\
.replace('#END#', '')
output = ansiconv.to_html(output)
# if '#END#' in console:
# Save job into the db auditlog
return Response('data: {}\n\n'.format(output),
mimetype='text/event-stream')
def start_task(message):
time.sleep(1)
project = Project.objects.get(id=message.content['project_id'])
deployment = Deployment.objects.get(id=message.content['deployment_id'])
deployment.output = ''
deployment.save()
engine = import_module(settings.SESSION_ENGINE)
SessionStore = engine.SessionStore
session = SessionStore(message.content['session_key'])
if backend.get_task_details(project, deployment.task.name) is None:
return
process = subprocess.Popen(
backend.build_command(project, deployment, session),
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
shell=True,
executable=getattr(settings, 'SHELL', '/bin/sh'),
)
while True:
nextline = process.stdout.readline()
if nextline == '' and process.poll() is not None:
break
Group("deployment-{}".format(deployment.id)).send({
"text":
json.dumps({
'status':
'pending',
'text':
str('<span class="output-line">{}</span>'.format(
ansiconv.to_html(nextline)))
}),
})
deployment.add_output(nextline)
sys.stdout.flush()
Deployment.objects.filter(pk=deployment.id).update(
status=deployment.SUCCESS if process.returncode ==
0 else deployment.FAILED)
Group("deployment-{}".format(deployment.id)).send({
"text":
json.dumps({
'status':
deployment.SUCCESS
if process.returncode == 0 else deployment.FAILED,
'text':
''
}),
})
deployment_finished.send(deployment, deployment_id=deployment.pk)
def base64_decode(raw_id_1, raw_id_2):
std_str = '<!DOCTYPE html><html lang="en"><head><meta charset="UTF-8"><link rel="stylesheet" href="https://cdn.jsdelivr.net/gh/tonsky/[email protected]/distr/fira_code.css"><style>.code-font-medium {font-family: \'Fira Code\', monospace;font-size: medium;}</style><style>'
try:
str_raw_html = base64.b64decode((''.join(open('static/datalogs/{}_{}.txt'.format(raw_id_1, raw_id_2), 'r').readlines())).encode()).decode().replace('\n', '<br>')
html_conv = ansiconv.to_html(str_raw_html).replace('ansi37', '')
str_html = '<body class="code-font-medium">{}</body></html>'.format(html_conv)
st_css = ansiconv.base_css()
return std_str + st_css + '</style></head>' + str_html
except Exception as ident:
return 'Error occurred. {}'.format(ident)
def __init__(self, title, details, parent = None):
super(self.__class__, self).__init__(parent, QtCore.Qt.WindowSystemMenuHint | QtCore.Qt.WindowTitleHint | QtCore.Qt.MSWindowsFixedSizeDialogHint)
self.setupUi(self) # This is defined in design.py file automatically
# It sets up layout and widgets that are defined
self.setWindowTitle("IEEE Testbank Tool - " + title)
# Pre-processing
details = str(details).replace("\n", "<br />\n")
self.detailsTxt.setText("<style>%s</style>%s" % (ansiconv.base_css(), ansiconv.to_html(details)))
def get_experiment(path):
logdir = APP.config['logdir']
complete_path = os.path.join(logdir, path)
if os.path.isfile(complete_path):
file_content = get_file(complete_path)
if path.endswith(".log"):
result = ansiconv.to_html(file_content)
elif path.endswith(".ini"):
result = html.escape(file_content)
else:
result = "Unknow file type: '{}'.".format(complete_path)
else:
result = "File '{}' does not exist.".format(complete_path)
return Response(result, mimetype='text/html', status=200)
def get_experiment(path):
logdir = APP.config['logdir']
complete_path = os.path.join(logdir, path)
if os.path.isfile(complete_path):
file_content = get_file(complete_path)
if path.endswith(".log"):
result = ansiconv.to_html(file_content)
elif path.endswith(".ini"):
result = file_content
else:
result = "Unknow file type: \"{}\".".format(complete_path)
else:
result = "File \"{}\" does not exist.".format(complete_path)
return Response(result, mimetype='text/html', status=200)
def to_html(self):
import ansiconv
output = """
<style>
{}
div.timelinetool {{
font-family: monospace;
white-space: pre-wrap;
padding: 10px;
}}
</style>
<div class="timelinetool ansi_fore ansi_back">{}</div>
""".format(ansiconv.base_css(), ansiconv.to_html(str(self)))
return output
def to_html(self):
import ansiconv
output = """
<style>
{}
div.timelinetool {{
font-family: monospace;
white-space: pre-wrap;
padding: 10px;
}}
</style>
<div class="timelinetool ansi_fore ansi_back">{}</div>
""".format(ansiconv.base_css(), ansiconv.to_html(str(self)))
return output
def get_experiment(path):
logdir = APP.config['logdir']
complete_path = os.path.join(logdir, path)
if os.path.isfile(complete_path):
file_content = get_file(complete_path)
if path.endswith(".log"):
result = ansiconv.to_html(html.escape(file_content))
elif path.endswith(".ini"):
lexer = IniLexer()
formatter = HtmlFormatter(linenos=True)
result = highlight(file_content, lexer, formatter)
else:
result = "Unknown file type: '{}'.".format(complete_path)
else:
result = "File '{}' does not exist.".format(complete_path)
return Response(result, mimetype='text/html', status=200)
def lastlog(self, lines=20, format='', html=True):
handler = self._loghandler
if format:
formatter = ColoredFormatter(format, datefmt=self.time_format)
else:
formatter = ColoredFormatter(
self.html_format if html else self.format,
datefmt=self.time_format)
rv = u'\n'.join([formatter.format(i) for i in handler.buffer[-lines:]])
if html:
rv = ansiconv.to_html(self.html_fix(rv))
else:
rv = ansiconv.to_plain(rv)
return rv
def get(self, user, repo, sha1):
fullname = "%s/%s" % (user, repo)
project = Project.get_by_id(fullname)
if project is None:
self.response.set_status(404)
template = paths.JINJA_ENVIRONMENT.get_template('404.html')
self.response.write(template.render(user=self.user,
baseurl="../../../",
message="Project %s not found" % fullname))
return
push = Push.get_by_id(sha1)
if push is None:
self.response.set_status(404)
template = paths.JINJA_ENVIRONMENT.get_template('404.html')
self.response.write(template.render(user=self.user,
baseurl="../../../",
message="Commit %s not found" % sha1))
return
repo = project.repo.get()
junit = JUnitTestResult.query(JUnitTestResult.push==push.key).fetch()
junit = [j for j in junit if j.valid]
logging.info("Found %d junit results for sha %s", len(junit), sha1)
junit = junit[0]
sim_job = junit.travis_job.get()
sim_logfiles = []
for logfile in sim_job.logfiles:
gcs_file = gcs.open(logfile)
sim_logfiles.append((logfile.split('/')[-1].split(".")[0], ansiconv.to_html(gcs_file.read())))
gcs_file.close()
template = paths.JINJA_ENVIRONMENT.get_template('build.html')
self.response.write(template.render(user=self.user,
project=project,
repo=repo,
push=push,
junit=junit,
sim_job=sim_job,
sim_logfiles=sim_logfiles))
return
def output_stream_generator(self):
if backend.get_task_details(self.project,
self.object.task.name) is None:
return
try:
process = subprocess.Popen(
backend.build_command(self.project, self.object,
self.request.session),
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
shell=True,
executable=getattr(settings, 'SHELL', '/bin/sh'),
)
all_output = ''
yield '<link rel="stylesheet" type="text/css" href="/static/css/console-style.css">'
while True:
nextline = process.stdout.readline()
if nextline == '' and process.poll() is not None:
break
all_output += nextline
nextline = '<span class="output-line">{}</span>'.format(
ansiconv.to_html(nextline))
yield nextline + ' ' * 1024
sys.stdout.flush()
self.object.status = self.object.SUCCESS if process.returncode == 0 else self.object.FAILED
yield '<span id="finished" style="display:none;">{}</span> {}'.format(
self.object.status, ' ' * 1024)
self.object.output = all_output
self.object.save()
deployment_finished.send(self.object, deployment_id=self.object.pk)
except Exception as e:
message = "An error occurred: " + e.message
yield '<span class="output-line">{}</span>'.format(
message) + ' ' * 1024
yield '<span id="finished" style="display:none;">failed</span> {}'.format(
'*1024')
def main(cfg):
# get working directory
wd = cfg.get(SETTINGS, 'repo_path')
# perform git add
cmd = cfg.get(SETTINGS, 'git_cmd_add').split(' ')
p = run(cmd, cwd=wd, stdout=PIPE)
print('*** git add done')
# perform git diff
cmd = cfg.get(SETTINGS, 'git_cmd_diff').split(' ')
p = run(cmd, cwd=wd, stdout=PIPE)
print('*** git diff done')
diffo = p.stdout.decode('utf-8')
# only commit and report if there is a diff output
if len(diffo) < DIFF_LEN_REQUIRED:
print('*** no changes detected, will stop here')
return False
else:
print('*** changes detected (%d bytes), will proceed' % len(diffo))
# perform git commit
cmd = cfg.get(SETTINGS, 'git_cmd_commit').split(' ')
p = run(cmd, cwd=wd, stdout=PIPE)
print('*** commited')
# perform redacting
if cfg.getboolean(SETTINGS, 'redact_enable'):
cre = re.compile(cfg.get(SETTINGS, 'redact_pattern'), flags=re.MULTILINE)
replace = cfg.get(SETTINGS, 'redact_replace')
print('*** perfoming redaction')
diffr = re.sub(cre, replace, diffo)
# remove diff and index lines
if cfg.getboolean(SETTINGS, 'redact_minimize'):
cre = re.compile(r'(diff\s--git\s.+\n|index\s.+\n|---\s.+\n)', flags=re.MULTILINE)
diffr = re.sub(cre, '', diffr)
# send report
report(cfg, ansiconv.to_html(diffr))
print('*** report done')
def output_stream_generator(self):
if get_task_details(self.project, self.object.task.name) is None:
return
try:
process = subprocess.Popen(
build_command(self.object, self.request.session),
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
shell=True,
executable=getattr(settings, 'SHELL', '/bin/sh'),
)
all_output = ''
yield '<link rel="stylesheet" type="text/css" href="/static/css/console-style.css">'
while True:
nextline = process.stdout.readline()
if nextline == '' and process.poll() is not None:
break
all_output += nextline
nextline = '<span class="output-line">{}</span>'.format(ansiconv.to_html(nextline))
yield nextline + ' '*1024
sys.stdout.flush()
self.object.status = self.object.SUCCESS if process.returncode == 0 else self.object.FAILED
yield '<span id="finished" style="display:none;">{}</span> {}'.format(self.object.status, ' '*1024)
self.object.output = all_output
self.object.save()
deployment_finished.send(self.object, deployment_id=self.object.pk)
except Exception as e:
message = "An error occurred: " + e.message
yield '<span class="output-line">{}</span>'.format(message) + ' '*1024
yield '<span id="finished" style="display:none;">failed</span> {}'.format('*1024')
def start_task(message):
time.sleep(1)
project = Project.objects.get(id=message.content['project_id'])
deployment = Deployment.objects.get(id=message.content['deployment_id'])
deployment.output = ''
deployment.save()
engine = import_module(settings.SESSION_ENGINE)
SessionStore = engine.SessionStore
session = SessionStore(message.content['session_key'])
if backend.get_task_details(project, deployment.task.name) is None:
return
process = subprocess.Popen(
backend.build_command(project, deployment, session),
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
stdin=subprocess.PIPE,
shell=True,
executable=getattr(settings, 'SHELL', '/bin/sh'),
close_fds=True
)
fd = process.stdout.fileno()
fl = fcntl.fcntl(fd, fcntl.F_GETFL)
fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)
while True:
try:
nextline = process.stdout.readline()
except IOError as e:
nextline = ''
if nextline == '' and process.poll() is not None:
break
#
# next_input = deployment.get_next_input()
# if next_input:
# process.stdin.write(next_input + '\n')
if nextline:
Group("deployment-{}".format(deployment.id)).send({
"text": json.dumps({
'status': 'pending',
'text': str('<span class="output-line">{}</span>'.format(ansiconv.to_html(nextline)))
}),
}, immediately=True)
deployment.add_output(nextline)
sys.stdout.flush()
Deployment.objects.filter(pk=deployment.id).update(
status=deployment.SUCCESS if process.returncode == 0 else deployment.FAILED
)
Group("deployment-{}".format(deployment.id)).send({
"text": json.dumps({
'status': deployment.SUCCESS if process.returncode == 0 else deployment.FAILED,
'text': ''
}),
}, immediately=True)
deployment_finished.send(deployment, deployment_id=deployment.pk)
def test_basic_to_html_replace_newline():
expected = '<span class="ansi32">Foo<br />\nBar</span>'
actual = ansiconv.to_html('\033[0;32mFoo\nBar', replace_newline=True)
assert actual == expected
def get_context_data(self, **kwargs):
context = super(DeploymentDetail, self).get_context_data(**kwargs)
context['deploy_output'] = ansiconv.to_html(self.object.output) if self.object.output else ''
return context
def test_to_html_with_cursor_up_command():
expected = '<span class="ansi34">Foo</span>'
actual = ansiconv.to_html('\033[0;30mFoo\n\033[A\033[0;34mFoo')
assert actual == expected
def test_to_html_with_multiple_color_changes():
expected = '<span class="ansi32">Foo</span><span class="ansi31">Bar</span>'
actual = ansiconv.to_html('\033[0;32mFoo\033[0;31mBar')
assert actual == expected
def test_basic_to_html():
expected = '<span class="ansi32">Foo</span>'
actual = ansiconv.to_html('\033[0;32mFoo')
assert actual == expected
def emit(self, record):
super().emit(record)
service.most_recent_line = ansiconv.to_html(
service.html_fix(html_formatter.format(record))) + '\n'
def get_formatted_output(self):
return ansiconv.to_html(self.output) if self.output else ''
def get_formatted_output(self):
return ansiconv.to_html(self.output) if self.output else ''
def log(id):
testroles_row = testroles.query.filter(testroles.id == id).order_by(testroles.id).first()
html = ansiconv.to_html(testroles_row.testrole_log)
css = ansiconv.base_css()
return render_template('logs.html', testroles_row=testroles_row, html=html, css=css)
def retrieve(self, request, *args, **kwargs):
job = self.get_object()
try:
target_format = request.accepted_renderer.format
if target_format in ('html', 'api', 'json'):
content_format = request.query_params.get(
'content_format', 'html')
content_encoding = request.query_params.get(
'content_encoding', None)
start_line = request.query_params.get('start_line', 0)
end_line = request.query_params.get('end_line', None)
dark_val = request.query_params.get('dark', '')
dark = bool(dark_val
and dark_val[0].lower() in ('1', 't', 'y'))
content_only = bool(target_format in ('api', 'json'))
dark_bg = (content_only and dark) or (not content_only and
(dark or not dark_val))
content, start, end, absolute_end = job.result_stdout_raw_limited(
start_line, end_line)
# Remove any ANSI escape sequences containing job event data.
content = re.sub(
r'\x1b\[K(?:[A-Za-z0-9+/=]+\x1b\[\d+D)+\x1b\[K', '',
content)
body = ansiconv.to_html(cgi.escape(content))
context = {
'title': get_view_name(self.__class__),
'body': mark_safe(body),
'dark': dark_bg,
'content_only': content_only,
}
data = render_to_string('api/stdout.html', context).strip()
if target_format == 'api':
return Response(mark_safe(data))
if target_format == 'json':
if content_encoding == 'base64' and content_format == 'ansi':
return Response({
'range': {
'start': start,
'end': end,
'absolute_end': absolute_end
},
'content':
b64encode(content.encode('utf-8'))
})
elif content_format == 'html':
return Response({
'range': {
'start': start,
'end': end,
'absolute_end': absolute_end
},
'content': body
})
return Response(data)
elif target_format == 'txt':
return Response(job.result_stdout)
elif target_format == 'ansi':
return Response(job.result_stdout_raw)
elif target_format in {'txt_download', 'ansi_download'}:
filename = '{type}_{pk}{suffix}.txt'.format(
type=camelcase_to_underscore(job.__class__.__name__),
pk=job.id,
suffix='.ansi' if target_format == 'ansi_download' else '')
content_fd = job.result_stdout_raw_handle(
enforce_max_bytes=False)
if target_format == 'txt_download':
content_fd = StdoutANSIFilter(content_fd)
response = HttpResponse(FileWrapper(content_fd),
content_type='text/plain')
response[
"Content-Disposition"] = 'attachment; filename="{}"'.format(
filename)
return response
else:
return super(JobStdout, self).retrieve(request, *args,
**kwargs)
except StdoutMaxBytesExceeded as e:
response_message = _(
"Standard Output too large to display ({text_size} bytes), "
"only download supported for sizes over {supported_size} bytes."
).format(text_size=e.total, supported_size=e.supported)
if request.accepted_renderer.format == 'json':
return Response({
'range': {
'start': 0,
'end': 1,
'absolute_end': 1
},
'content': response_message
})
else:
return Response(response_message)
def get_context_data(self, **kwargs):
context = super(DeploymentDetail, self).get_context_data(**kwargs)
context['deploy_output'] = ansiconv.to_html(
self.object.output) if self.object.output else ''
return context
