def END_CMD(command, ok, output, info, stats) :
#assert f.opened_task
#assert f.opened_subtask
#assert f.opened_cmd
if ok :
f.content.append( '<span class="command_ok">[OK]</span>' )
else:
f.content.append( '<span class="command_failure">[FAILURE]</span>' )
f.content.append(
'<div id="output%d" class="output">'
'<div class="output_header">OUTPUT:</div>'
'<div class="plain_text">%s</div>'
'</div>' % (f.id_output, deansi.deansi(output) )
)
if output.count('\n') > 10 :
f.content.append( '<script type="text/javascript">togglesize(\'output%d\');</script>' % f.id_output )
f.id_output += 1
if info :
f.content.append(
'<div id="info%d" class="info">'
'<div class="info_header">INFO:</div>\n'
'<div class="plain_text">%s</div></div>' % ( f.id_info, deansi.deansi(info) ) )
if info.count('\n') > 10 :
f.content.append( '<script type="text/javascript">togglesize(\'info%d\');</script>' % f.id_info )
f.id_info += 1
if stats :
f.content.append(
'<div class="stats_header">Statistics:</div>'
'<div class="stats">' +
'<br />'.join(['<b>%s:</b> %s'%(item) for item in stats.iteritems() ]) +
'</div>'
)
f.content.append( '</div>' )
f.opened_cmd = False
def _streaming_gen():
yield '<style>{}</style>'.format(deansi.styleSheet())
yield FOLLOW_LOG_JS
yield '<div class="ansi_terminal">'
buffer = []
for log in container.logs(stdout=True,
stderr=True,
stream=True,
follow=True):
char = str(log)
buffer.append(char)
if char == '\n':
yield deansi.deansi(''.join(buffer))
buffer = []
if buffer:
yield deansi.deansi(''.join(buffer))
yield '</div>'
def END_CMD(command, ok, output, info, stats):
#assert f.opened_task
#assert f.opened_subtask
#assert f.opened_cmd
if ok:
f.content.append('<span class="command_ok">[OK]</span>')
else:
f.content.append(
'<span class="command_failure">[FAILURE]</span>')
f.content.append('<div id="output%d" class="output">'
'<div class="output_header">OUTPUT:</div>'
'<div class="plain_text">%s</div>'
'</div>' %
(f.id_output, deansi.deansi(output)))
if output.count('\n') > 10:
f.content.append(
'<script type="text/javascript">togglesize(\'output%d\');</script>'
% f.id_output)
f.id_output += 1
if info:
f.content.append('<div id="info%d" class="info">'
'<div class="info_header">INFO:</div>\n'
'<div class="plain_text">%s</div></div>' %
(f.id_info, deansi.deansi(info)))
if info.count('\n') > 10:
f.content.append(
'<script type="text/javascript">togglesize(\'info%d\');</script>'
% f.id_info)
f.id_info += 1
if stats:
f.content.append('<div class="stats_header">Statistics:</div>'
'<div class="stats">' + '<br />'.join([
'<b>%s:</b> %s' % (item)
for item in stats.iteritems()
]) + '</div>')
f.content.append('</div>')
f.opened_cmd = False
def contentBlock(self, kind, content, id):
id = kind + id
return ('<div id="{id}" class="{kind}">\n'
'<div class="{kind}_header">{KIND}:</div>\n'
'<div class="plain_text">{content}</div>\n'
'</div>\n'
'{expander}'.format(
id=id,
kind=kind,
KIND=kind.upper(),
content=deansi.deansi(content),
expander="" if content.count("\n") <= 10 else
"<script type='text/javascript'>"
"togglesize('{0}');</script>\n".format(id),
))
def get_docker_image(self):
docker_image_name = self.context.repository.replace('/', '-')
output = []
docker_image = self.docker.images(docker_image_name)
if not docker_image or self.context.rebuild:
dockerfile = os.path.join(self.context.clone_path, self.spec.dockerfile)
build_options = {
'tag': docker_image_name,
'rm': True,
'stream': True,
'decode': True,
'nocache': self.context.nocache,
}
if not os.path.exists(dockerfile):
logger.warning(
'No Dockerfile: %s found for repo: %s, using simple runner image',
dockerfile,
self.context.repository
)
dockerfile_content = 'FROM messense/badwolf-test-runner:python\n'
fileobj = io.BytesIO(dockerfile_content.encode('utf-8'))
build_options['fileobj'] = fileobj
else:
build_options['dockerfile'] = self.spec.dockerfile
build_success = False
logger.info('Building Docker image %s', docker_image_name)
self.update_build_status('INPROGRESS', 'Building Docker image')
res = self.docker.build(self.context.clone_path, **build_options)
for log in res:
if 'errorDetail' in log:
msg = log['errorDetail']['message']
elif 'error' in log:
# Deprecated
# https://github.com/docker/docker/blob/master/pkg/jsonmessage/jsonmessage.go#L104
msg = log['error']
else:
msg = log['stream']
if 'Successfully built' in msg:
build_success = True
output.append(deansi.deansi(msg))
logger.info('`docker build` : %s', msg.strip())
if not build_success:
return None, ''.join(output)
return docker_image_name, ''.join(output)
def run(self):
start_time = time.time()
branch = self.context.source['branch']
context = {
'context': self.context,
'build_log_url': self.build_status.url,
'branch': branch['name'],
'scripts': self.spec.scripts,
'ansi_termcolor_style': deansi.styleSheet(),
}
self.update_build_status('INPROGRESS', 'Test in progress')
docker_image_name, build_output = self.get_docker_image()
context.update({
'build_logs': Markup(build_output),
'elapsed_time': int(time.time() - start_time),
})
if not docker_image_name:
self.update_build_status('FAILED',
'Build or get Docker image failed')
context['exit_code'] = -1
self.send_notifications(context)
return
exit_code, output = self.run_in_container(docker_image_name)
logger.debug('Docker run output: %s', output)
if exit_code == 0:
# Success
logger.info('Test succeed for repo: %s', self.context.repository)
self.update_build_status('SUCCESSFUL', '1 of 1 test succeed')
else:
# Failed
logger.info('Test failed for repo: %s, exit code: %s',
self.context.repository, exit_code)
if exit_code == 137:
self.update_build_status('FAILED', 'build cancelled')
else:
self.update_build_status('FAILED', '1 of 1 test failed')
context.update({
'logs': Markup(deansi.deansi(output)),
'exit_code': exit_code,
'elapsed_time': int(time.time() - start_time),
})
self.send_notifications(context)
return exit_code == 0
def contentBlock(self, kind, content, id) :
id = kind+id
return (
'<div id="{id}" class="{kind}">\n'
'<div class="{kind}_header">{KIND}:</div>\n'
'<div class="plain_text">{content}</div>\n'
'</div>\n'
'{expander}'
.format(
id = id,
kind = kind,
KIND = kind.upper(),
content = deansi.deansi(content),
expander = "" if content.count("\n") <= 10 else
"<script type='text/javascript'>"
"togglesize('{}');</script>\n".format(id),
)
)
def execute(scriptname):
import os
scripts = configScripts()
parameters = ns(request.form.items())
params_list = []
output_file = False
if 'parameters' in scripts[scriptname]:
for parm_name,parm_data in scripts[scriptname]['parameters'].items():
if parm_data.get('type', None) == 'FILE':
filename=os.path.join(configdb.upload_folder,session[parm_name])
parameters[parm_name] = filename
elif parm_data.get('type', None) == 'FILEDOWN':
session[parm_name]=next(tmpfile())
filename=os.path.join(configdb.download_folder,session[parm_name])
parameters[parm_name] = filename
output_file = parm_name
if not parameters.get(parm_name, None) and scripts[scriptname]['parameters'][parm_name].get('default',None):
parameters[parm_name] = scripts[scriptname]['parameters'][parm_name]['default']
script = scripts[scriptname].script
if type(script) is not list:
script = shlex.split(script)
commandline = [
piece.format(**parameters)
for piece in script
]
commandline = [cmd.replace('SOME_SRC',configdb.prefix) for cmd in commandline]
return_code=0
try:
output=subprocess.check_output(commandline,stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
output=e.output
return_code=e.returncode
try:
output_decoded=output.decode('utf-8')
except UnicodeDecodeError:
output_decoded=output.decode('latin-1')
return json.dumps(dict(
script_name=scriptname,
output_file=output_file,
return_code=return_code,
response=deansi.deansi(output_decoded),
commandline=commandline,
))
def get_docker_image(self):
docker_image_name = self.context.repository.replace('/', '-')
output = []
try:
docker_image = self.docker.images.get(docker_image_name)
except ImageNotFound:
docker_image = None
if not docker_image or self.context.rebuild:
build_options = {
'tag': docker_image_name,
'rm': True,
'forcerm': True,
'stream': True,
'decode': True,
'nocache': self.context.nocache
}
if self.spec.image:
from_image_name, from_image_tag = self.spec.image.split(':', 2)
logger.info('Pulling Docker image %s', self.spec.image)
self.docker.images.pull(from_image_name, tag=from_image_tag)
logger.info('Pulled Docker image %s', self.spec.image)
dockerfile_content = 'FROM {}\n'.format(self.spec.image)
fileobj = io.BytesIO(dockerfile_content.encode('utf-8'))
build_options['fileobj'] = fileobj
else:
dockerfile = os.path.join(self.context.clone_path,
self.spec.dockerfile)
if os.path.exists(dockerfile):
build_options['dockerfile'] = self.spec.dockerfile
else:
logger.warning(
'No Dockerfile: %s found for repo: %s, using simple runner image',
dockerfile, self.context.repository)
dockerfile_content = 'FROM messense/badwolf-test-runner:python\n'
fileobj = io.BytesIO(dockerfile_content.encode('utf-8'))
build_options['fileobj'] = fileobj
build_success = False
logger.info('Building Docker image %s', docker_image_name)
self.update_build_status('INPROGRESS', 'Building Docker image')
# Use low level API instead of high level API to get raw output
res = self.docker.api.build(self.context.clone_path,
**build_options)
for log in res:
if 'errorDetail' in log:
msg = log['errorDetail']['message']
elif 'error' in log:
# Deprecated
# https://github.com/docker/docker/blob/master/pkg/jsonmessage/jsonmessage.go
msg = log['error']
elif 'status' in log:
msg = log['status']
else:
msg = log.get('stream')
if not msg:
continue
if 'Successfully tagged' in msg:
build_success = True
output.append(deansi.deansi(msg))
logger.info('`docker build` : %s', msg.strip())
if not build_success:
return None, ''.join(output)
return docker_image_name, ''.join(output)
def sendMail(
sender,
to,
subject,
text=None,
html=None,
md=None,
ansi=None,
cc=[],
bcc=[],
replyto=[],
attachments = [],
template=None,
config=None,
stylesheets = [],
dump = None,
verbose=True
):
import smtplib
from email.mime.multipart import MIMEMultipart
from email.mime.base import MIMEBase
from email.mime.text import MIMEText
from email.encoders import encode_base64
from email.utils import formataddr, parseaddr
if not config:
from config import smtp
else:
import imp
smtp=imp.load_source('config',config).smtp
def formatAddress(address):
return formataddr(parseaddr(address))
def formatAddresses(addresses):
return ', '.join(formatAddress(a) for a in addresses)
# Headers
msg = MIMEMultipart()
msg['Subject'] = subject
msg['From'] = formatAddress(sender)
msg['To'] = formatAddresses(to)
if cc: msg['CC'] = formatAddresses(cc)
if bcc: msg['BCC'] = formatAddresses(bcc)
if replyto: msg['Reply-To'] = formatAddresses(replyto)
recipients = to + (cc if cc else []) + (bcc if bcc else [])
# Attachments
for filename in attachments:
step("Attaching {}...".format(filename))
part = MIMEBase('application', "octet-stream")
part.set_payload(open(filename, "rb").read())
encode_base64(part)
import os
part.add_header(
'Content-Disposition',
'attachment; filename="{}"'.format(
os.path.basename(filename.replace('"', ''))))
msg.attach(part)
# Content formatting
style=''
for stylesheet in stylesheets or []:
with open(stylesheet) as stylefile:
style+=stylefile.read()
if md:
step("Formating markdown input...")
import markdown
text = md # TODO: Format plain text
html = htmltemplate.format(
style = style,
body = markdown.markdown(md, output_format='html')
)
if ansi:
step("Formating ansi input...")
import deansi
text = ansi # TODO: Clean ansi sequences
html = htmltemplate.format(
style = deansi.styleSheet()+style,
body = "<div class='ansi_terminal'>"+deansi.deansi(ansi)+"</div>",
)
content = MIMEMultipart('alternative')
if text:
content.attach(MIMEText(text,'plain','utf8'))
if html:
step("Adapting html to mail clients...")
import premailer
html = premailer.transform(html)
content.attach(MIMEText(html,'html','utf8'))
import sys
#sys.stdout.write(html)
msg.attach(content)
if dump:
with open(dump,'w') as dumpfile:
dumpfile.write(msg.as_string())
success("Email dumped as {}".format(dump))
return
return
# Sending
step("Connecting to {host}:{port} as {user}...".format(**smtp))
server = smtplib.SMTP(smtp['host'], smtp['port'])
server.starttls()
server.login(smtp['user'], smtp['password'])
step("\tSending...")
server.sendmail(sender, recipients, msg.as_string())
success("\tMail sent")
server.quit()
def sendMail(sender,
to,
subject,
text=None,
html=None,
md=None,
ansi=None,
cc=[],
bcc=[],
replyto=[],
attachments=[],
template=None,
config=None,
stylesheets=[],
dump=None,
verbose=True):
from email.mime.multipart import MIMEMultipart
from email.mime.base import MIMEBase
from email.mime.text import MIMEText
from email.encoders import encode_base64
from email.utils import formataddr, parseaddr
def formatAddress(address):
return formataddr(parseaddr(address))
def formatAddresses(addresses):
return ', '.join(formatAddress(a) for a in addresses)
# Headers
msg = MIMEMultipart()
msg['Subject'] = subject
msg['From'] = formatAddress(sender)
msg['To'] = formatAddresses(to)
if cc: msg['CC'] = formatAddresses(cc)
if bcc: msg['BCC'] = formatAddresses(bcc)
if replyto: msg['Reply-To'] = formatAddresses(replyto)
recipients = to + (cc if cc else []) + (bcc if bcc else [])
# Attachments
for filename in attachments:
step("Attaching {}...".format(filename))
part = MIMEBase('application', "octet-stream")
part.set_payload(open(filename, "rb").read())
encode_base64(part)
import os
part.add_header(
'Content-Disposition', 'attachment; filename="{}"'.format(
os.path.basename(filename.replace('"', ''))))
msg.attach(part)
# Content formatting
style = ''
for stylesheet in stylesheets or []:
with open(stylesheet) as stylefile:
style += stylefile.read()
if md:
step("Formating markdown input...")
import markdown
text = md # TODO: Format plain text
html = htmltemplate.format(style=style,
body=markdown.markdown(
md, output_format='html'))
if ansi:
step("Formating ansi input...")
import deansi
text = ansi # TODO: Clean ansi sequences
html = htmltemplate.format(
style=deansi.styleSheet() + style,
body="<div class='ansi_terminal'>" + deansi.deansi(ansi) +
"</div>",
)
content = MIMEMultipart('alternative')
if text:
content.attach(MIMEText(text, 'plain', 'utf8'))
if html:
step("Adapting html to mail clients...")
import premailer
html = premailer.transform(html)
content.attach(MIMEText(html, 'html', 'utf8'))
import sys
#sys.stdout.write(html)
msg.attach(content)
if dump:
with open(dump, 'w') as dumpfile:
dumpfile.write(msg.as_string())
success("Email dumped as {}".format(dump))
return
# Sending
sendOverSmtp(config, sender, recipients, msg)