def extra_data_pretty(self):
if not self.extra_data:
return self.extra_data
try:
return format_html(json2html.convert(json=self.extra_data))
except Exception as e:
return repr(e)
def main():
args = parse_args()
if args.subcommand == "sub":
product_yaml = os.path.join(args.ssg_root, "products", args.product,
"product.yml")
env_yaml = ssg.environment.open_environment(args.build_config_yaml,
product_yaml)
try:
profile1 = ssg.build_yaml.Profile.from_yaml(
args.profile1, env_yaml)
profile2 = ssg.build_yaml.Profile.from_yaml(
args.profile2, env_yaml)
except jinja2.exceptions.TemplateNotFound as e:
print("Error: Profile {} could not be found.".format(str(e)))
exit(1)
subtracted_profile = profile1 - profile2
exclusive_rules = len(subtracted_profile.get_rule_selectors())
exclusive_vars = len(subtracted_profile.get_variable_selectors())
if exclusive_rules > 0:
print("{} rules were left after subtraction.".format(
exclusive_rules))
if exclusive_vars > 0:
print("{} variables were left after subtraction.".format(
exclusive_vars))
if exclusive_rules > 0 or exclusive_vars > 0:
profile1_basename = os.path.splitext(
os.path.basename(args.profile1))[0]
profile2_basename = os.path.splitext(
os.path.basename(args.profile2))[0]
subtracted_profile_filename = "{}_sub_{}.profile".format(
profile1_basename, profile2_basename)
print(
"Creating a new profile containing the exclusive selections: {}"
.format(subtracted_profile_filename))
subtracted_profile.title = profile1.title + " subtracted by " + profile2.title
subtracted_profile.dump_yaml(subtracted_profile_filename)
print("Profile {} was created successfully".format(
subtracted_profile_filename))
else:
print(
"Subtraction would produce an empty profile. No new profile was generated"
)
exit(0)
benchmark = ssg.build_profile.XCCDFBenchmark(args.benchmark, args.product)
ret = []
if args.profile:
ret.append(benchmark.show_profile_stats(args.profile, args))
else:
all_profile_elems = benchmark.tree.findall("./{%s}Profile" %
(ssg.constants.XCCDF11_NS))
ret = []
for elem in all_profile_elems:
profile = elem.get('id')
if profile is not None:
ret.append(benchmark.show_profile_stats(profile, args))
if args.format == "json":
print(json.dumps(ret, indent=4))
if args.format == "html":
from json2html import json2html
filtered_output = []
output_path = "./"
if args.output:
output_path = args.output
if not os.path.exists(output_path):
os.mkdir(output_path)
content_path = os.path.join(output_path, "content")
if not os.path.exists(content_path):
os.mkdir(content_path)
content_list = [
'rules', 'missing_stig_ids', 'missing_cis_refs',
'missing_hipaa_refs', 'missing_anssi_refs', 'missing_ospp_refs',
'missing_cui_refs', 'missing_ovals', 'missing_sces',
'missing_bash_fixes', 'missing_ansible_fixes',
'missing_ignition_fixes', 'missing_kubernetes_fixes',
'missing_puppet_fixes', 'missing_anaconda_fixes', 'missing_cces',
'ansible_parity', 'implemented_checks', 'implemented_fixes',
'missing_checks', 'missing_fixes'
]
link = """<a href="{}"><div style="height:100%;width:100%">{}</div></a>"""
for profile in ret:
bash_fixes_count = profile['rules_count'] - profile[
'missing_bash_fixes_count']
for content in content_list:
content_file = "{}_{}.txt".format(profile['profile_id'],
content)
content_filepath = os.path.join("content", content_file)
count = len(profile[content])
if count > 0:
if content == "ansible_parity":
#custom text link for ansible parity
count = link.format(
content_filepath, "{} out of {} ({}%)".format(
bash_fixes_count - count, bash_fixes_count,
int(((bash_fixes_count - count) /
bash_fixes_count) * 100)))
count_href_element = link.format(content_filepath, count)
profile['{}_count'.format(content)] = count_href_element
with open(os.path.join(content_path, content_file),
'w+') as f:
f.write('\n'.join(profile[content]))
else:
profile['{}_count'.format(content)] = count
del profile[content]
filtered_output.append(profile)
with open(os.path.join(output_path, "statistics.html"), 'w+') as f:
f.write(
json2html.convert(json=json.dumps(filtered_output),
escape=False))
elif args.format == "csv":
# we can assume ret has at least one element
# CSV header
print(",".join(ret[0].keys()))
for line in ret:
print(",".join([str(value) for value in line.values()]))
def get_extra_data_html(self):
if not self.extra_data:
return mark_safe('')
return mark_safe(json2html.convert(json=self.extra_data))
def form():
if request.method == 'POST':
pincode = request.form['pincode']
currentdate = str(request.form['currentdate'])
year = currentdate[0:4]
month = currentdate[5:7]
day = currentdate[8:]
print(pincode)
date = str(day + '-' + month + '-' + year)
# get json from COWIN API
url = 'https://cdn-api.co-vin.in/api/v2/appointment/sessions/public/calendarByPin?pincode=' + pincode + '&date=' + date
dat = requests.get(url)
dat_json = dat.json()
# if there are not centers print error
if not dat_json['centers']:
f = open(
os.getcwd() + '/flask-cowinapp/templates/' + 'report.html',
'w')
f.write('''
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>COWIN CENTER SEARCH</title>
</head>
<body>
<h2> ============ There are No Centers Available for Search results for
date
''')
f.write(currentdate + ' and Pincode ' + pincode)
f.write('============ </h2>')
f.write('''
<form action="/">
<input type='submit' action='index.html' value='Return to mainpage'>
</form>
</body>
</html>''')
f.close()
return render_template('report.html')
# if there are centers , print centers in table format
else:
html_data = json2html.convert(json=dat_json)
f = open(
os.getcwd() + '/flask-cowinapp/templates/' + 'report.html',
'w')
f.write('''============
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>COWIN CENTER SEARCH</title>
</head>
<body>
<h1></H>Search results for date {{date}} and pincode{{pincode}}</h1>
<table>
''')
f.write('''<h1>============Search results for date ''')
f.write(currentdate + ' and Pincode ' + pincode)
f.write('============ </h1>')
f.write('''
<form action="/">
<input type='submit' action='index.html' value='Return to mainpage'>
</form>''')
f.write(html_data)
f.write('''</table>
</body>
</html>''')
f.close()
return render_template('report.html')
else:
return render_template('index.html')
def __call__(self, reference, results, **kwargs):
# handle indent the same way python's json library does
if 'indent' in kwargs:
kwargs['indent'] = int(kwargs['indent'])
if 'ordered' in kwargs:
ordered = bool(kwargs.get('ordered', False))
del kwargs['ordered']
else:
ordered = False
# fetch a foreign key reference and append it as if it were part of the document.
if 'fetch' in kwargs:
fetch = kwargs['fetch'].split(',')
del kwargs['fetch']
else:
fetch = []
if 'bare_keys' in kwargs:
bare_keys = bool(kwargs.get('bare_keys', False))
del kwargs['bare_keys']
else:
bare_keys = False
# note this is a closure around the fetch parameter. Consider before refactoring out of the method.
def serialize(doc):
if isinstance(doc, document.Document):
ret = doc.json_repr(ordered=ordered, bare_keys=bare_keys)
for f in fetch:
if f in ret:
if isinstance(doc[f], list):
ret[f] = [d.json_repr(ordered=ordered, bare_keys=bare_keys) for d in doc[f]]
elif isinstance(doc[f], dict):
ret[f] = {k: v.json_repr(ordered=ordered, bare_keys=bare_keys) for k, v in doc[f].items()}
else:
ret[f] = doc[f].json_repr(ordered=ordered, bare_keys=bare_keys)
return ret
else:
return doc
result = mapjson(serialize, results) # make sure to serialize a full Document structure if we have one.
if not (isinstance(result, dict) or isinstance(result, list)):
result = {"_": result}
from json2html import json2html
rsp = StringIO()
rsp.write("""<!doctype html>
<html>
<head>
<!-- Latest compiled and minified CSS -->
<link rel="stylesheet" href="//maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap.min.css">
<!-- Optional theme -->
<link rel="stylesheet" href="//maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap-theme.min.css">
</head><body>
<!-- Latest compiled and minified JavaScript -->
<script src="//maxcdn.bootstrapcdn.com/bootstrap/3.3.7/js/bootstrap.min.js"></script>
""")
rsp.write(json2html.convert(json=result, table_attributes="class=\"table table-bordered table-hover\""))
rsp.write('</body></html>')
return 'text/html',rsp.getvalue()
else:
continue
t2Holder[goodRuns[run]]["STRIP"] = "BAD"
recorded = float(t2Holder[goodRuns[run]]["Lumi Recorded"].replace("pb",""))
bad_strip_recorded += recorded
if run not in d_:
d_[run] = 0
bad_any_recorded += recorded
except:
pass
print("The delivered and recorderd luminosity w/ track quality requirements = %.3f,%.3f, respectively" % (good_track_delivered,good_track_recorded))
t2 = json2html.convert( json = json.dumps(t2Holder) )
t2 = t2.replace("<td>GOOD","<td class=\"GOOD\">GOOD")
t2 = t2.replace("<td>BAD","<td class=\"BAD\">BAD")
t2 = t2.replace("<td>TODO","<td class=\"TODO\">TODO")
t2 = t2.replace("pb","pb<sup>-1</sup>")
import time
now = time.strftime("%c")
htmlHeader = """
<html>
<head>
<title>Certification Status</title>
<style type='text/css'>
body { font-family: "Candara", sans-serif; }
td.BAD { background-color: rgb(255,100,100); }
td.bah { background-color: rgb(255,180,80); }
def format_query_result_db(res_cursor, distict_generator = False, count = False, raw = False, html = False):
'''
Format the results from the result db (mongodb).
Parameters
----------
res_cursor : gridfs.grid_file.GridOutCursor or generator<object> or pymongo.cursor.Cursor
First if non_document and non_document_raw.
Second if distinct values wanted.
Thirst otherwise.
distict_generator : bool, optional (default is False)
Res is generator<object> created from the distinct(...) method of mongodb.
If generaor<dict>, convert each dict to json.
Otherwise just print.
count : bool, optional (default is False)
Only print count, not results
raw : bool, optional (default is False)
Print raw data from gridfs
Otherwise print json.
If `raw` will not be converted to html!
html : bool, optional (default is False)
Format as html.
Returns
-------
str
'''
from pymongo.errors import PyMongoError
from androlyze.ui.util import HtmlUtil
# if html enabled convert to table view if `json2html` is present
# otherwise use pygmentize
json_convert = lambda json : json
if html:
try:
from json2html import json2html
json_convert = lambda j : json2html.convert(json = j)
except ImportError:
from pygments import highlight
from pygments.formatters import HtmlFormatter
from pygments.lexers import get_lexer_by_name
json_convert = lambda json: highlight(json, get_lexer_by_name('json'), HtmlFormatter())
# collect results as list<str>
resl = []
def anl(text):
''' Append a newline '''
# dont format raw data as html
return '%s\n' % text if not html or raw else HtmlUtil.newline(HtmlUtil.prefy(text))
try:
# return count
if count:
cnt = 0
if is_pymongo_cursor(res_cursor):
cnt = res_cursor.count()
elif distict_generator:
cnt = len(list(res_cursor))
return '%d' % cnt
else:
if distict_generator:
for r in sorted(res_cursor):
if isinstance(r, dict):
r = dict2json(res_cursor)
resl.append(r)
elif isinstance(r, (str, unicode)):
resl.append(r)
else:
for i, res in enumerate(res_cursor, 1):
delimiter = '/* %d */' % i
text = HtmlUtil.newline(delimiter) if html else delimiter
if html: text = HtmlUtil.redify(text)
resl.append(text)
# return raw data
if raw:
# gridfs.grid_file.GridOut
for gridout_obj in res:
resl.append(gridout_obj)
# return json
else:
j = dict2json(res)
# convert json (if enabled)
j = json_convert(j)
resl.append(j)
# return result by joining single strings
return ''.join([anl(res_str) for res_str in resl])
except PyMongoError as e:
log.exception(e)
def alesco_data_pretty(self):
if not self.alesco_data:
return self.alesco_data
return format_html(json2html.convert(json=self.alesco_data, clubbing=False))
def shodanAnalyze():
# If home button is pressed then return to homepage
if request.method == 'POST' and request.values == 'To Shodan':
return redirect(url_for('shodan'))
if request.method == 'POST' and request.values == 'To Upload':
return redirect(url_for('uploadPage'))
print 'Going into shodanAnalyze'
# flashthismessage = "Analyze IP's that did not resolve"
form = ReusableForm(request.form)
print form.errors
# generalSRCInfo = []
detailedSRCInfo = []
# generalDSTInfo = []
detailedDSTInfo = []
analyzeThis = session['parse']
print 'analyzeThis == %s ' % analyzeThis
for key in analyzeThis:
print 'looping through key %s' % key
if isinstance(analyzeThis[key][3], (int, long, float)) == True:
try:
if ipaddress.ip_address(
analyzeThis[key][3]).is_private == True:
print 'Private IP-address. Not looking up'
return
else:
try:
# Lookup the host
host = api.host(analyzeThis[key][1])
# Store general info
# generalSRCInfo.append("""
# IP: {}
# Organization: {}
# Operating System: {}
# """.format(host['ip_str'], host.get('org', 'n/a'), host.get('os', 'n/a')))
# Store all banners
for item in host['data']:
detailedSRCInfo.append("""
Port: {}
""".format(item['port']))
# print 'general source %s' % generalSRCInfo
# print 'detailed source %s' % detailedSRCInfo
# analyzeThis[key][5] = generalSRCInfo
analyzeThis[key][5].append("Open ports on ")
analyzeThis[key][5].append(analyzeThis[key][3])
analyzeThis[key][5].append(detailedSRCInfo)
except:
analyzeThis[key][5].append("No open ports on ")
analyzeThis[key][5].append(analyzeThis[key][3])
print '%s can not be looked up' % analyzeThis[key][1]
except:
print 'Not an IP-address. Looking up anyway'
continue
else:
try:
# Lookup the host
host = api.host(analyzeThis[key][1])
# Store general info
# generalSRCInfo.append("""
# IP: {}
# Organization: {}
# Operating System: {}
# """.format(host['ip_str'], host.get('org', 'n/a'), host.get('os', 'n/a')))
# Store all banners
for item in host['data']:
detailedSRCInfo.append("""
Port: {}
""".format(item['port']))
# print 'general source %s' % generalSRCInfo
# print 'detailed source %s' % detailedSRCInfo
# analyzeThis[key][5] = generalSRCInfo
analyzeThis[key][5].append("Open ports on ")
analyzeThis[key][5].append(analyzeThis[key][3])
analyzeThis[key][5].append(detailedSRCInfo)
except:
analyzeThis[key][5].append("No open ports on ")
analyzeThis[key][5].append(analyzeThis[key][3])
print '%s can not be looked up' % analyzeThis[key][1]
if isinstance(analyzeThis[key][4], (int, long, float)) == True:
try:
if ipaddress.ip_address(
analyzeThis[key][4]).is_private == True:
print 'Private IP-address. Not looking up'
return
else:
try:
# Lookup the host
host = api.host(analyzeThis[key][1])
# Store general info
# generalSRCInfo.append("""
# IP: {}
# Organization: {}
# Operating System: {}
# """.format(host['ip_str'], host.get('org', 'n/a'), host.get('os', 'n/a')))
# Store all banners
for item in host['data']:
detailedSRCInfo.append("""
Port: {}
""".format(item['port']))
# print 'general source %s' % generalSRCInfo
# print 'detailed source %s' % detailedSRCInfo
# analyzeThis[key][5] = generalSRCInfo
analyzeThis[key][5].append("Open ports on ")
analyzeThis[key][5].append(analyzeThis[key][3])
analyzeThis[key][5].append(detailedSRCInfo)
except:
analyzeThis[key][5].append(
"Could not look up or no open ports on ")
analyzeThis[key][5].append(analyzeThis[key][3])
print '%s can not be looked up' % analyzeThis[key][1]
except:
print 'Not an IP-address. Looking up anyway'
continue
else:
try:
# Lookup the host
host = api.host(analyzeThis[key][2])
# Store general info
# generalDSTInfo.append("""
# IP: {}
# Organization: {}
# Operating System: {}
# """.format(host['ip_str'], host.get('org', 'n/a'), host.get('os', 'n/a')))
# Store all banners
for item in host['data']:
detailedDSTInfo.append("""
Port: {}
""".format(item['port']))
# print 'general destination %s' % generalDSTInfo
# print 'detailed destination %s' % detailedDSTInfo
# analyzeThis[key][6] = generalDSTInfo
analyzeThis[key][6].append("Open ports on ")
analyzeThis[key][6].append(analyzeThis[key][4])
analyzeThis[key][6].append(detailedDSTInfo)
except:
analyzeThis[key][6].append(
"Could not look up or no open ports on ")
analyzeThis[key][6].append(analyzeThis[key][4])
print '%s can not be looked up' % analyzeThis[key][2]
with open(os.path.join(UPLOAD_FOLDER) + 'shodan-json.json', 'w') as file:
json.dump(analyzeThis, file)
json_resp = json2html.convert(json=analyzeThis)
return render_template('shodan.html',
form=form,
flashthismessage='Result of Shodan lookup: ')
def _info_html(self, prefix):
info = getattr(self, '{}_info'.format(prefix))
return format_html(json2html.convert(json=info))
def serilize_Html(data: Any) -> str:
return json2html.convert(json=rapidjson.dumps(
data, ensure_ascii=False, sort_keys=True, datetime_mode=1))
def main():
args = parse_args()
if args.subcommand == "sub":
profile1 = ssg.build_yaml.Profile.from_yaml(args.profile1)
profile2 = ssg.build_yaml.Profile.from_yaml(args.profile2)
subtracted_profile = profile1 - profile2
exclusive_rules = len(subtracted_profile.get_rule_selectors())
if exclusive_rules > 0:
print("{} rules were left after subtraction.".format(
exclusive_rules))
profile1_basename = os.path.splitext(
os.path.basename(args.profile1))[0]
profile2_basename = os.path.splitext(
os.path.basename(args.profile2))[0]
subtracted_profile_filename = "{}_sub_{}.profile".format(
profile1_basename, profile2_basename)
print("Creating a new profile containing the exclusive rules: {}".
format(subtracted_profile_filename))
subtracted_profile.title = profile1.title + " subtracted by " + profile2.title
subtracted_profile.dump_yaml(subtracted_profile_filename)
print("Profile {} was created successfully".format(
subtracted_profile_filename))
else:
print(
"Subtraction would produce an empty profile. No new profile was generated"
)
exit(0)
benchmark = ssg.build_profile.XCCDFBenchmark(args.benchmark)
ret = []
if args.profile:
ret.append(benchmark.show_profile_stats(args.profile, args))
else:
all_profile_elems = benchmark.tree.findall("./{%s}Profile" %
(ssg.constants.XCCDF11_NS))
ret = []
for elem in all_profile_elems:
profile = elem.get('id')
if profile is not None:
ret.append(benchmark.show_profile_stats(profile, args))
if args.format == "json":
print(json.dumps(ret, indent=4))
if args.format == "html":
from json2html import json2html
filtered_output = []
output_path = "./"
if args.output:
output_path = args.output
if not os.path.exists(output_path):
os.mkdir(output_path)
content_path = os.path.join(output_path, "content")
if not os.path.exists(content_path):
os.mkdir(content_path)
content_list = [
'rules', 'missing_stig_ids', 'missing_ovals', 'missing_bash_fixes',
'missing_ansible_fixes', 'missing_puppet_fixes',
'missing_anaconda_fixes', 'missing_cces'
]
link = """<a href="{}"><div style="height:100%;width:100%">{}</div></a>"""
for profile in ret:
for content in content_list:
content_file = "{}_{}.txt".format(profile['profile_id'],
content)
content_filepath = os.path.join("content", content_file)
count = len(profile[content])
if count > 0:
count_href_element = link.format(content_filepath, count)
profile['{}_count'.format(content)] = count_href_element
with open(os.path.join(content_path, content_file),
'w+') as f:
f.write('\n'.join(profile[content]))
else:
profile['{}_count'.format(content)] = count
del profile[content]
filtered_output.append(profile)
with open(os.path.join(output_path, "statistics.html"), 'w+') as f:
f.write(
json2html.convert(json=json.dumps(filtered_output),
escape=False))
elif args.format == "csv":
# we can assume ret has at least one element
# CSV header
print(",".join(ret[0].keys()))
for line in ret:
print(",".join([str(value) for value in line.values()]))
def json_to_html_table(cls, filename: str):
with open(filename) as file:
data_dict = json.load(file)
html_table = json2html.convert(json=data_dict, escape=False)
return html_table
def send_alarm(self, alarm):
""" Send the alarm """
# Read the RedELK logo from file and base64 encode it
with open('redelk_white.png', 'rb') as logo_file:
redelk_logo_b64 = base64.b64encode(
logo_file.read()).decode('utf-8')
mail = f'''
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
<title>Alarm from RedELK</title>
<meta name="viewport" content="width=device-width, initial-scale=1.0"/>
<style type="text/css">
#normal {{
font-family: Tahoma, Geneva, sans-serif;
font-size: 16px;
line-height: 24px;
}}
</style>
</head>
<body style="margin: 0; padding: 0;">
<table align="center" cellpadding="0" cellspacing="0" width="800" style="border-collapse: collapse;" style="max-width:800px;">
<tr>
<td bgcolor="#212121" rowspan=2 width="120px" style="padding: 30px 30px 30px 30px; text-align:center;">
<img height="60px" src="data:image/png;base64,{redelk_logo_b64}" alt="img" />
</td>
<td bgcolor="#212121" height="30px" style="color: #FAFAFA; font-family: Arial, sans-serif; font-size: 24px; padding: 30px 30px 0px 10px;">
RedELK alarm: <em>{alarm["info"]["name"]}</em>
</td>
</tr>
<tr>
<td bgcolor="#212121" height="20px" style="color: #FAFAFA; font-family: Arial, sans-serif; font-size: 16px; line-height: 20px; padding: 20px 30px 30px 10px;">
Total hits: <em>{alarm["hits"]["total"]}</em>
</td>
</tr>
<tr>
<td colspan=2 style="color: #153643; font-family: Arial, sans-serif; font-size: 16px; line-height: 20px; padding: 0px 30px 0px 10px;">
<p>{alarm["info"]["description"]}</p>
</td>
</tr>
'''
subject = f'Alarm from {alarm["info"]["name"]} [{alarm["hits"]["total"]} hits]'
if len(alarm['groupby']) > 0:
mail += f'''
<tr>
<td colspan=2 style="color: #153643; font-family: Arial, sans-serif; font-size: 12px; line-height: 16px; padding: 0px 15px 0px 15px;">
<p>Please note that the items below have been grouped by: {pprint(alarm["groupby"])}</p>
</td>
</tr>
'''
try:
for hit in alarm['hits']['hits']:
index = 0
title = hit['_id']
while index < len(alarm['groupby']):
val = get_value(f'_source.{alarm["groupby"][index]}', hit)
if index == 0:
title = val
else:
title = f'{title} / {val}'
index += 1
mail += f'''
<tr>
<td bgcolor="#323232" colspan=2 style="color: #FAFAFA; font-family: Arial, sans-serif; font-size: 16px; line-height: 20px; padding: 10px 10px 10px 10px; text-align:center;">
<b>{title}</b>
</td>
</tr>
'''
row = 0
for field in alarm['fields']:
bgcolor = '#FAFAFA' if row % 2 == 0 else '#F1F1F1'
val = get_value(f'_source.{field}', hit)
value = json2html.convert(json=val)
mail += f'''
<tr bgcolor="{bgcolor}" style="color: #153643; font-family: Arial, sans-serif; font-size: 12px; line-height: 16px;">
<td style="padding: 10px 10px 10px 10px;"><b>{field}</b></td>
<td style="padding: 10px 10px 10px 10px; white-space:pre-wrap; word-wrap:break-word">{value}</td>
</tr>
'''
row += 1
mail += '<tr><td colspan=2 style="padding: 15px;"> </td></tr>'
mail += '</table>\n</body>\n</html>'
except Exception as error: # pylint: disable=broad-except
self.logger.error('Error sending email: %s', error)
self.logger.exception(error)
mail += '</body></html>\n'
self.send_mail(notifications['email']['to'], mail, subject)
def animalid_valid(animal_id):
animal_key = client.key(ANIMALS, int(animal_id))
animal = client.get(key=animal_key)
# No Animal with this animal_id exists
if animal is None:
# Failure 404 Not Found
return status_fail(404, ERR.NO_ANIMAL)
# Get an Animal
if request.method == "GET":
if "application/json" in request.accept_mimetypes:
# Success 200 OK
output = animal_output(animal, client)
return status_success(200, output=json.dumps(output))
elif "text/html" in request.accept_mimetypes:
# Success 200 OK
output = animal_output(animal, client)
conversion = json2html.convert(json=json.dumps(output))
return status_success(200, output=conversion, mime="text/html")
else:
# Failure 406 Not Acceptable
return status_fail(406, ERR.WRONG_MEDIA_REQUESTED)
# Edit an Animal
elif request.method == "PUT":
# Check media type
if "application/json" not in request.content_type:
# Failure 415 Unsupported Media Type
return status_fail(415, ERR.WRONG_MEDIA_RECEIVED)
if "application/json" not in request.accept_mimetypes:
# Failure 406 Not Acceptable
return status_fail(406, ERR.WRONG_MEDIA_REQUESTED)
# Check if request is missing any of the required attributes
content = request.get_json()
if ("name" not in content.keys() or "species" not in content.keys()
or "height" not in content.keys()):
# Failure 400 Bad Request
return status_fail(400, ERR.MISSING_ATTRIBUTE)
# Validate request
if not valid_alphanum(content["name"], 50):
# Failure 400 Bad Request
return status_fail(400, ERR.INVALID_NAME)
if not valid_alphanum(content["species"], 50):
# Failure 400 Bad Request
return status_fail(400, ERR.INVALID_SPECIES)
if not valid_int(content["height"], 25):
# Failure 400 Bad Request
return status_fail(400, ERR.INVALID_HEIGHT)
# Ensure that the name of an Animal is unique across all Animals
query = client.query(kind=ANIMALS)
results = list(query.fetch())
for a in results:
if a["name"] == content["name"]:
# Failure 403 Forbidden
return status_fail(403, ERR.NAME_EXISTS)
# Update Animal
animal = datastore.Entity(key=client.key(ANIMALS))
animal.update({
"name": content["name"],
"species": content["species"],
"height": content["height"]
})
client.put(animal)
# Success 303 See Other
output = animal_output(animal, client)
return status_success(303,
output=json.dumps(output),
location=output["self"])
# Edit an Animal
elif request.method == "PATCH":
# Check media type
if "application/json" not in request.content_type:
# Failure 415 Unsupported Media Type
return status_fail(415, ERR.WRONG_MEDIA_RECEIVED)
if "application/json" not in request.accept_mimetypes:
# Failure 406 Not Acceptable
return status_fail(406, ERR.WRONG_MEDIA_REQUESTED)
# Check if request contains any of the object attributes
content = request.get_json()
if "name" in content.keys():
# Validate name
if not valid_alphanum(content["name"], 50):
# Failure 400 Bad Request
return status_fail(400, ERR.INVALID_NAME)
# Ensure that the name of an Animals is unique across all Animals
query = client.query(kind=ANIMALS)
results = list(query.fetch())
for a in results:
if a["name"] == content["name"]:
# Failure 403 Forbidden
return status_fail(403, ERR.NAME_EXISTS)
animal.update({"name": content["name"]})
if "species" in content.keys():
# Validate species
if not valid_alphanum(content["species"], 50):
# Failure 400 Bad Request
return status_fail(400, ERR.INVALID_SPECIES)
animal.update({"species": content["species"]})
if "height" in content.keys():
# Validate height
if not valid_int(content["height"], 25):
# Failure 400 Bad Request
return status_fail(400, ERR.INVALID_HEIGHT)
animal.update({"height": content["height"]})
client.put(animal)
# Success 303 See Other
output = animal_output(animal, client)
return status_success(303,
output=json.dumps(output),
location=output["self"])
# Delete an Animal
elif request.method == "DELETE":
iceberg_id = animal["home"]
iceberg_key = client.key(ICEBERGS, int(iceberg_id))
iceberg = client.get(key=iceberg_key)
if iceberg:
if str(animal_id) in iceberg["inhabitants"]:
iceberg["inhabitants"].remove(str(animal_id))
client.put(iceberg)
# Success 204 No Content
client.delete(animal_key)
return status_success(204)
else:
# Failure 405 Method Not Allowed
animalid_invalid()
def get_extra_data_html(self):
if not self.extra_data:
return mark_safe('')
return mark_safe(json2html.convert(json=self.extra_data))
def generate_html(json_data):
'''Convert json to html'''
generated_html = json2html.convert(json=json_data)
return generated_html.encode('utf-8')
async def service_submission(request: web.Request):
reader = MultipartReader.from_response(request)
data = None
filedata = None
# Read multipart email
while True:
part = await reader.next() # pylint: disable=not-callable
if part is None:
break
if part.headers[hdrs.CONTENT_TYPE] == "application/json":
data = await part.json()
continue
if part.headers[hdrs.CONTENT_TYPE] == "application/zip":
filedata = await part.read(decode=True)
# Validate max file size
maxsize = 10 * 1024 * 1024 # 10MB
actualsize = len(filedata)
if actualsize > maxsize:
raise web.HTTPRequestEntityTooLarge(maxsize, actualsize)
filename = part.filename
continue
raise web.HTTPUnsupportedMediaType(
reason=
f"One part had an unexpected type: {part.headers[hdrs.CONTENT_TYPE]}"
)
# data (dict) and file (bytearray) have the necessary information to compose the email
support_email_address = request.app[APP_CONFIG_KEY]["smtp"]["sender"]
is_real_usage = any(env in os.environ.get("SWARM_STACK_NAME", "")
for env in ("production", "staging"))
db = get_storage(request.app)
user = await db.get_user({"id": request[RQT_USERID_KEY]})
user_email = user.get("email")
if not is_real_usage:
support_email_address = user_email
try:
# NOTE: temporarily internal import to avoid render_and_send_mail to be interpreted as handler
# TODO: Move outside when get_handlers_from_namespace is fixed
from .login.utils import render_and_send_mail
attachments = [("metadata.json", json.dumps(data, indent=4))]
if filedata:
attachments.append((filename, filedata))
# send email
await render_and_send_mail(
request,
to=support_email_address,
template=common_themed(EMAIL_TEMPLATE_NAME),
context={
"user":
user_email,
"data":
json2html.convert(json=json.dumps(data),
table_attributes='class="pure-table"'),
"subject":
"TEST: " * (not is_real_usage) + "New service submission",
},
attachments=attachments,
)
except Exception as exc:
log.exception("Error while sending the 'new service submission' mail.")
raise web.HTTPServiceUnavailable() from exc
raise web.HTTPNoContent(content_type="application/json")
ename = entity["name"]
# ignore mention of character names
if entity["mention"].lower() in series_characters:
print("ignored", entity["mention"])
continue
entities_per_episode[path.stem].append(ename)
if ename not in all_entities:
all_entities[ename] = entity
all_entities[ename]["episodes"] = [path.name]
all_entities[ename]["mentions"] = [entity["mention"]]
else:
all_entities[ename]["episodes"].append(path.name)
all_entities[ename]["mentions"].append(entity["mention"])
if "categories" in all_entities[ename]:
del all_entities[ename]["categories"]
if "mention" in all_entities[ename]:
del all_entities[ename]["mention"]
if "links" in all_entities[ename]:
del all_entities[ename]["links"]
#print(list(all_entities.values()))
with open(output_dir / "report.html", 'w') as f:
f.write(json2html.convert({"data": list(all_entities.values())}))
for episode in entities_per_episode:
with open(output_dir / (episode + "_report.html"), 'w') as f:
f.write(
json2html.convert({
"data":
[all_entities[e] for e in entities_per_episode[episode]]
}))
def format_json_to_html(json):
return json2html.convert(
json=json,
table_attributes="class=\"table table-bordered table-hover\"")
def ad_data_pretty(self):
if not self.ad_data:
return self.ad_data
return format_html(json2html.convert(json=self.ad_data))
jsondata = json.load(rawdata) # <- THIS WILL BE PASSED DOWN
rawdata.close()
os.chdir(pwd) # return to ~/cerebunitdata/docs/expdata/cells/GranuleCell
######################### CREATE THE HTML CONTAINER ##########################
structure = """
<html>
<head><title>DAngelo et al 1998 Data for Resting Membrane Voltage</title>
<body></body>
</html>
"""
soup = bs(structure, features="lxml")
title = soup.find("title")
body = soup.find("body")
###################### INSERT TITLE INTO HTML CONTAINER #######################
title.insert(0, title_content)
######################### CONVERT JSON DATA TO HTML ##########################
htmltable = json2html.convert(json=jsondata)
extrasoup = bs(htmltable, "html.parser")
###################### INSERT TABLE INTO HTML CONTAINER #######################
body.insert(0, extrasoup)
####################### FINALLY WRITE INTO HTML FILE #########################
savefile = pwd + os.sep + "rawhtmls" + os.sep + filename.replace(
".json", ".html")
html = open(savefile, "w", encoding="utf-8")
html.write(str(soup))
