def application(environ, start_response):
# translate ENVIRON variable about query string to dict
query = parse_qs(environ['QUERY_STRING'])
# As there can ben more than one value for a variable then
# a list is provied as a default for parse_qs
age = query.get('age', [''])[0]
hobbies = query.get('hobbies', [])
age = escape(age)
hobbies = [escape(hobby) for hobby in hobbies]
html = get_template('template.html')
body = html % {
'method': 'GET',
'checked-software': ('', 'checked')['software' in hobbies],
'checked-tunning': ('', 'checked')['tunning' in hobbies],
'age': age or 'Empty',
'hobbies': ', '.join(hobbies or ['No Hobbies?'])
}
status = '200 OK'
headers = [('Content-Type', 'text/html'),
('Content-Lenght', str(len(body)))]
start_response(status, headers)
return [bytes(body, 'utf-8')]
def main(request, response):
policyDeliveries = json.loads(request.GET.first('policyDeliveries', '[]'))
worker_type = request.GET.first('type', 'classic')
commonjs_url = '%s://%s:%s/common/security-features/resources/common.sub.js' % (
request.url_parts.scheme, request.url_parts.hostname,
request.url_parts.port)
if worker_type == 'classic':
import_line = 'importScripts("%s");' % commonjs_url
else:
import_line = 'import "%s";' % commonjs_url
maybe_additional_headers = {}
error = ''
for delivery in policyDeliveries:
if delivery['deliveryType'] == 'meta':
error = '<meta> cannot be used in WorkerGlobalScope'
elif delivery['deliveryType'] == 'http-rp':
if delivery['key'] == 'referrerPolicy':
maybe_additional_headers['Referrer-Policy'] = delivery['value']
else:
error = 'invalid delivery key for http-rp: %s' % delivery['key']
else:
error = 'invalid deliveryType: %s' % delivery['deliveryType']
handler = lambda: util.get_template('worker.js.template') % ({
'import': import_line,
'error': error
})
util.respond(request,
response,
payload_generator=handler,
content_type='text/javascript',
maybe_additional_headers=maybe_additional_headers)
def main(request, response):
policyDeliveries = json.loads(request.GET.first(b"policyDeliveries",
b"[]"))
maybe_additional_headers = {}
meta = u''
error = u''
for delivery in policyDeliveries:
if delivery[u'deliveryType'] == u'meta':
if delivery[u'key'] == u'referrerPolicy':
meta += u'<meta name="referrer" content="%s">' % delivery[
u'value']
else:
error = u'invalid delivery key'
elif delivery[u'deliveryType'] == u'http-rp':
if delivery[u'key'] == u'referrerPolicy':
maybe_additional_headers[
b'Referrer-Policy'] = isomorphic_encode(delivery[u'value'])
else:
error = u'invalid delivery key'
else:
error = u'invalid deliveryType'
handler = lambda: util.get_template(u"document.html.template"
) % ({
u"meta": meta,
u"error": error
})
util.respond(request,
response,
payload_generator=handler,
content_type=b"text/html",
maybe_additional_headers=maybe_additional_headers)
def generate_test_source_files(spec_directory, test_helper_filenames,
spec_json, target):
test_expansion_schema = spec_json['test_expansion_schema']
specification = spec_json['specification']
spec_json_js_template = util.get_template('spec_json.js.template')
util.write_file(
os.path.join(spec_directory, "generic", "spec_json.js"),
spec_json_js_template % {'spec_json': json.dumps(spec_json)})
util.write_file(
os.path.join(spec_directory, "generic", "debug-output.spec.src.json"),
json.dumps(spec_json, indent=2, separators=(',', ': ')))
# Choose a debug/release template depending on the target.
html_template = "test.%s.html.template" % target
artifact_order = test_expansion_schema.keys() + ['name']
artifact_order.remove('expansion')
# Create list of excluded tests.
exclusion_dict = {}
for excluded_pattern in spec_json['excluded_tests']:
excluded_expansion = \
expand_pattern(excluded_pattern, test_expansion_schema)
for excluded_selection in permute_expansion(excluded_expansion,
artifact_order):
excluded_selection['delivery_key'] = spec_json['delivery_key']
exclusion_dict[dump_test_parameters(excluded_selection)] = True
for spec in specification:
# Used to make entries with expansion="override" override preceding
# entries with the same |selection_path|.
output_dict = {}
for expansion_pattern in spec['test_expansion']:
expansion = expand_pattern(expansion_pattern,
test_expansion_schema)
for selection in permute_expansion(expansion, artifact_order):
selection['delivery_key'] = spec_json['delivery_key']
selection_path = spec_json['selection_pattern'] % selection
if selection_path in output_dict:
if expansion_pattern['expansion'] != 'override':
print(
"Error: %s's expansion is default but overrides %s"
% (selection['name'],
output_dict[selection_path]['name']))
sys.exit(1)
output_dict[selection_path] = copy.deepcopy(selection)
for selection_path in output_dict:
selection = output_dict[selection_path]
if dump_test_parameters(selection) in exclusion_dict:
print('Excluding selection:', selection_path)
continue
try:
generate_selection(spec_directory, test_helper_filenames,
spec_json, selection, spec, html_template)
except util.ShouldSkip:
continue
def handle_error(request, response, exception, code, msg):
"""
Render an error template with a code and message.
"""
logging.exception(exception)
# Try to get the current user
session_store = sessions.get_store(request=request)
session = session_store.get_session()
auth_id = session.get('auth_id')
user = None
if auth_id:
user = UserPrefs.get(auth_id)
if user:
logging.error('Currently logged in user is ' + user.name + ' (' +
user.email + ')')
# Render and error template
template = get_template('error.html')
response.status = code
response.out.write(
template.render({
'user': user,
'code': code,
'message': msg
}))
def flag_if_submitted_through_github(repo_username, repo_id, issue):
"""
Flags any issue that is submitted through github's UI, and not the Ionic site.
Adds a label, as well as a comment, to force the issue through the custom form.
@return: whether or not the issue was flagged (bool)
"""
if not issue:
return False
number = issue.get("number")
if not number:
return False
user = issue.get("user")
if not user:
return False
if not issue.get("body"):
return False
if is_valid_issue_opened_source(repo_username, repo_id, issue):
return False
context = {"issue": issue, "user": user}
msg = util.get_template("RESUBMIT_TEMPLATE", context)
github_api.create_issue_comment(repo_username, repo_id, number, msg)
return True
def flag_if_submitted_through_github(repo_username, repo_id, issue):
return False # temporarily disabling ionitron
"""
Flags any issue that is submitted through github's UI, and not the Ionic site.
Adds a label, as well as a comment, to force the issue through the custom form.
@return: whether or not the issue was flagged (bool)
"""
if not issue:
return False
number = issue.get('number')
if not number:
return False
user = issue.get('user')
if not user:
return False
if not issue.get('body'):
return False
if is_valid_issue_opened_source(repo_username, repo_id, issue):
return False
context = {
'issue': issue,
'user': user
}
msg = util.get_template('RESUBMIT_TEMPLATE', context)
github_api.create_issue_comment(repo_username, repo_id, number, msg)
return True
def main(request, response):
policyDeliveries = json.loads(request.GET.first("policyDeliveries", "[]"))
maybe_additional_headers = {}
meta = ''
error = ''
for delivery in policyDeliveries:
if delivery['deliveryType'] == 'meta':
if delivery['key'] == 'referrerPolicy':
meta += '<meta name="referrer" content="%s">' % delivery[
'value']
else:
error = 'invalid delivery key'
elif delivery['deliveryType'] == 'http-rp':
if delivery['key'] == 'referrerPolicy':
maybe_additional_headers['Referrer-Policy'] = delivery['value']
else:
error = 'invalid delivery key'
else:
error = 'invalid deliveryType'
handler = lambda: util.get_template("document.html.template"
) % ({
"meta": meta,
"error": error
})
util.respond(request,
response,
payload_generator=handler,
content_type="text/html",
maybe_additional_headers=maybe_additional_headers)
def main(request, response):
policyDeliveries = json.loads(request.GET.first('policyDeliveries', '[]'))
worker_type = request.GET.first('type', 'classic')
commonjs_url = '%s://%s:%s/common/security-features/resources/common.js' % (
request.url_parts.scheme, request.url_parts.hostname,
request.url_parts.port)
if worker_type == 'classic':
import_line = 'importScripts("%s");' % commonjs_url
else:
import_line = 'import "%s";' % commonjs_url
maybe_additional_headers = {}
error = ''
for delivery in policyDeliveries:
if delivery['deliveryType'] == 'meta':
error = '<meta> cannot be used in WorkerGlobalScope'
elif delivery['deliveryType'] == 'http-rp':
if delivery['key'] == 'referrerPolicy':
maybe_additional_headers['Referrer-Policy'] = delivery['value']
else:
error = 'invalid delivery key for http-rp: %s' % delivery['key']
else:
error = 'invalid deliveryType: %s' % delivery['deliveryType']
handler = lambda: util.get_template('worker.js.template') % ({
'import': import_line,
'error': error
})
util.respond(
request,
response,
payload_generator=handler,
content_type='text/javascript',
maybe_additional_headers=maybe_additional_headers)
def application(environ, start_response):
# PEP 3333 says: CONTENT_LENGTH may be empty or missing
try:
request_content_length = int(environ.get('CONTENT_LENGTH', 0))
except ValueError:
request_content_length = 0
# Request Body is passed inside wsgi.input environment variable
request_body = environ['wsgi.input'].read(request_content_length)
form = parse_qs(request_body.decode('utf-8'))
age = form.get('age', [''])[0]
hobbies = form.get('hobbies', [])
age = escape(age)
hobbies = [escape(hobby) for hobby in hobbies]
html = get_template('template.html')
body = html % {
'method': 'POST',
'checked-software': ('', 'checked')['software' in hobbies],
'checked-tunning': ('', 'checked')['tunning' in hobbies],
'age': age or 'Empty',
'hobbies': ', '.join(hobbies or ['No Hobbies?'])
}
status = '200 OK'
headers = [('Content-Type', 'text/html'),
('Content-Lenght', str(len(body)))]
start_response(status, headers)
return [bytes(body, 'utf-8')]
def generate_test_source_files(config, spec_json, target):
test_expansion_schema = spec_json['test_expansion_schema']
specification = spec_json['specification']
spec_json_js_template = util.get_template('spec_json.js.template')
generated_spec_json_filename = os.path.join(config.spec_directory,
"spec_json.js")
util.write_file(
generated_spec_json_filename,
spec_json_js_template % {'spec_json': json.dumps(spec_json)})
# Choose a debug/release template depending on the target.
html_template = "test.%s.html.template" % target
artifact_order = test_expansion_schema.keys() + ['name']
artifact_order.remove('expansion')
# Create list of excluded tests.
exclusion_dict = {}
for excluded_pattern in spec_json['excluded_tests']:
excluded_expansion = \
expand_pattern(excluded_pattern, test_expansion_schema)
for excluded_selection in permute_expansion(excluded_expansion,
artifact_order):
excluded_selection_path = config.selection_pattern % excluded_selection
exclusion_dict[excluded_selection_path] = True
for spec in specification:
# Used to make entries with expansion="override" override preceding
# entries with the same |selection_path|.
output_dict = {}
for expansion_pattern in spec['test_expansion']:
expansion = expand_pattern(expansion_pattern,
test_expansion_schema)
for selection in permute_expansion(expansion, artifact_order):
selection['delivery_key'] = spec_json['delivery_key']
selection_path = config.selection_pattern % selection
if not selection_path in exclusion_dict:
if selection_path in output_dict:
if expansion_pattern['expansion'] != 'override':
print(
"Error: %s's expansion is default but overrides %s"
% (selection['name'],
output_dict[selection_path]['name']))
sys.exit(1)
output_dict[selection_path] = copy.deepcopy(selection)
else:
print('Excluding selection:', selection_path)
for selection_path in output_dict:
selection = output_dict[selection_path]
generate_selection(config, selection, spec, html_template)
def close_old_issue(repo_username, repo_id, number, issue):
context = {
'issue': issue,
'user': issue.get('user')
}
comment = util.get_template('CLOSING_TEMPLATE', context)
github_api.close_issue(repo_username, repo_id, number, issue)
github_api.create_issue_comment(repo_username, repo_id, number, comment)
return {
'closed_old_issue': True
}
def generate_test_source_files(config, spec_json, target):
test_expansion_schema = spec_json['test_expansion_schema']
specification = spec_json['specification']
spec_json_js_template = util.get_template('spec_json.js.template')
generated_spec_json_filename = os.path.join(config.spec_directory, "spec_json.js")
util.write_file(generated_spec_json_filename,
spec_json_js_template % {'spec_json': json.dumps(spec_json)})
# Choose a debug/release template depending on the target.
html_template = "test.%s.html.template" % target
artifact_order = test_expansion_schema.keys() + ['name']
artifact_order.remove('expansion')
# Create list of excluded tests.
exclusion_dict = {}
for excluded_pattern in spec_json['excluded_tests']:
excluded_expansion = \
expand_pattern(excluded_pattern, test_expansion_schema)
for excluded_selection in permute_expansion(excluded_expansion,
artifact_order):
excluded_selection_path = config.selection_pattern % excluded_selection
exclusion_dict[excluded_selection_path] = True
for spec in specification:
# Used to make entries with expansion="override" override preceding
# entries with the same |selection_path|.
output_dict = {}
for expansion_pattern in spec['test_expansion']:
expansion = expand_pattern(expansion_pattern, test_expansion_schema)
for selection in permute_expansion(expansion, artifact_order):
selection_path = config.selection_pattern % selection
if not selection_path in exclusion_dict:
if selection_path in output_dict:
if expansion_pattern['expansion'] != 'override':
print("Error: %s's expansion is default but overrides %s" % (selection['name'], output_dict[selection_path]['name']))
sys.exit(1)
output_dict[selection_path] = copy.deepcopy(selection)
else:
print('Excluding selection:', selection_path)
for selection_path in output_dict:
selection = output_dict[selection_path]
generate_selection(config,
selection,
spec,
html_template)
def handle_error(request, response, exception, code, msg):
"""
Render an error template with a code and message.
"""
logging.exception(exception)
# Try to get the current user
session_store = sessions.get_store(request=request)
session = session_store.get_session()
auth_id = session.get("auth_id")
user = None
if auth_id:
user = UserPrefs.get(auth_id)
# Render and error template
template = get_template("error.html")
response.out.write(template.render({"user": user, "code": code, "message": msg}))
def gen_code(usr):
"""generate code to be emailed to user"""
email = usr.get("email")
if not is_user_exist(email):
raise EXCEPTION_USER_NOT_EXIST
part1, part2 = rand_code(), rand_code()
res = dbase.update("users", "where email=?", [email], {
"code": part1 + "-" + part2,
"tstamp": int(time.time())
})
if not res:
raise EXCEPTION_DATABASE
sendmail({
"to": email,
"subj": "logging in",
"text": get_template("email_verification") % (part1 + "-" + part2)
})
return part1
def main(request, response):
policyDeliveries = json.loads(request.GET.first(b'policyDeliveries', b'[]'))
worker_type = request.GET.first(b'type', b'classic')
commonjs_url = u'%s://%s:%s/common/security-features/resources/common.sub.js' % (
request.url_parts.scheme, request.url_parts.hostname,
request.url_parts.port)
if worker_type == b'classic':
import_line = u'importScripts("%s");' % commonjs_url
else:
import_line = u'import "%s";' % commonjs_url
maybe_additional_headers = {}
error = u''
for delivery in policyDeliveries:
if delivery[u'deliveryType'] == u'meta':
error = u'<meta> cannot be used in WorkerGlobalScope'
elif delivery[u'deliveryType'] == u'http-rp':
if delivery[u'key'] == u'referrerPolicy':
maybe_additional_headers[b'Referrer-Policy'] = isomorphic_encode(delivery[u'value'])
elif delivery[u'key'] == u'mixedContent' and delivery[u'value'] == u'opt-in':
maybe_additional_headers[b'Content-Security-Policy'] = b'block-all-mixed-content'
elif delivery[u'key'] == u'upgradeInsecureRequests' and delivery[u'value'] == u'upgrade':
maybe_additional_headers[b'Content-Security-Policy'] = b'upgrade-insecure-requests'
else:
error = u'invalid delivery key for http-rp: %s' % delivery[u'key']
else:
error = u'invalid deliveryType: %s' % delivery[u'deliveryType']
handler = lambda: util.get_template(u'worker.js.template') % ({
u'import': import_line,
u'error': error
})
util.respond(
request,
response,
payload_generator=handler,
content_type=b'text/javascript',
maybe_additional_headers=maybe_additional_headers)
def generate_selection(config, selection, spec, test_html_template_basename):
test_parameters = dump_test_parameters(selection)
# Adjust the template for the test invoking JS. Indent it to look nice.
indent = "\n" + " " * 8
test_parameters = test_parameters.replace("\n", indent)
selection['test_js'] = '''
%s(
%s,
document.querySelector("meta[name=assert]").content,
new SanityChecker()
).start();
''' % (config.test_case_name, test_parameters)
selection['spec_name'] = spec['name']
selection[
'test_page_title'] = config.test_page_title_template % spec['title']
selection['spec_description'] = spec['description']
selection['spec_specification_url'] = spec['specification_url']
selection['helper_js'] = config.helper_js
selection['sanity_checker_js'] = config.sanity_checker_js
selection['spec_json_js'] = config.spec_json_js
test_filename = get_test_filename(config, selection)
test_headers_filename = test_filename + ".headers"
test_directory = os.path.dirname(test_filename)
test_html_template = util.get_template(test_html_template_basename)
disclaimer_template = util.get_template('disclaimer.template')
html_template_filename = os.path.join(util.template_directory,
test_html_template_basename)
generated_disclaimer = disclaimer_template \
% {'generating_script_filename': os.path.relpath(sys.argv[0],
util.test_root_directory),
'html_template_filename': os.path.relpath(html_template_filename,
util.test_root_directory)}
# Adjust the template for the test invoking JS. Indent it to look nice.
selection['generated_disclaimer'] = generated_disclaimer.rstrip()
selection[
'test_description'] = config.test_description_template % selection
selection['test_description'] = \
selection['test_description'].rstrip().replace("\n", "\n" + " " * 33)
# Directory for the test files.
try:
os.makedirs(test_directory)
except:
pass
delivery = handle_deliveries([
util.PolicyDelivery(selection['delivery_type'],
selection['delivery_key'],
selection['delivery_value'])
])
if len(delivery['headers']) > 0:
with open(test_headers_filename, "w") as f:
for header in delivery['headers']:
f.write('%s: %s\n' % (header, delivery['headers'][header]))
selection['meta_delivery_method'] = delivery['meta']
# Obey the lint and pretty format.
if len(selection['meta_delivery_method']) > 0:
selection['meta_delivery_method'] = "\n " + \
selection['meta_delivery_method']
# Write out the generated HTML file.
util.write_file(test_filename, test_html_template % selection)
def generate_selection(config, selection, spec, test_html_template_basename):
# TODO: Refactor out this referrer-policy-specific part.
if 'referrer_policy' in spec:
# Oddball: it can be None, so in JS it's null.
selection['referrer_policy'] = spec['referrer_policy']
test_parameters = json.dumps(selection, indent=2, separators=(',', ':'))
# Adjust the template for the test invoking JS. Indent it to look nice.
indent = "\n" + " " * 8
test_parameters = test_parameters.replace("\n", indent)
selection['test_js'] = '''
%s(
%s,
document.querySelector("meta[name=assert]").content,
new SanityChecker()
).start();
''' % (config.test_case_name, test_parameters)
selection['spec_name'] = spec['name']
selection['test_page_title'] = config.test_page_title_template % spec['title']
selection['spec_description'] = spec['description']
selection['spec_specification_url'] = spec['specification_url']
selection['helper_js'] = config.helper_js
selection['sanity_checker_js'] = config.sanity_checker_js
selection['spec_json_js'] = config.spec_json_js
test_filename = os.path.join(config.spec_directory, config.test_file_path_pattern % selection)
test_headers_filename = test_filename + ".headers"
test_directory = os.path.dirname(test_filename)
test_html_template = util.get_template(test_html_template_basename)
disclaimer_template = util.get_template('disclaimer.template')
html_template_filename = os.path.join(util.template_directory,
test_html_template_basename)
generated_disclaimer = disclaimer_template \
% {'generating_script_filename': os.path.relpath(__file__,
util.test_root_directory),
'html_template_filename': os.path.relpath(html_template_filename,
util.test_root_directory)}
# Adjust the template for the test invoking JS. Indent it to look nice.
selection['generated_disclaimer'] = generated_disclaimer.rstrip()
selection['test_description'] = config.test_description_template % selection
selection['test_description'] = \
selection['test_description'].rstrip().replace("\n", "\n" + " " * 33)
# Directory for the test files.
try:
os.makedirs(test_directory)
except:
pass
delivery = config.handleDelivery(selection, spec)
if len(delivery['headers']) > 0:
with open(test_headers_filename, "w") as f:
for header in delivery['headers']:
f.write(header)
f.write('\n')
selection['meta_delivery_method'] = delivery['meta']
# Obey the lint and pretty format.
if len(selection['meta_delivery_method']) > 0:
selection['meta_delivery_method'] = "\n " + \
selection['meta_delivery_method']
# Write out the generated HTML file.
util.write_file(test_filename, test_html_template % selection)
def generate_selection(spec_json, config, selection, spec,
test_html_template_basename):
test_filename = get_test_filename(config, selection)
target_policy_delivery = util.PolicyDelivery(selection['delivery_type'],
selection['delivery_key'],
selection['delivery_value'])
del selection['delivery_type']
del selection['delivery_key']
del selection['delivery_value']
# Parse source context list and policy deliveries of source contexts.
# `util.ShouldSkip()` exceptions are raised if e.g. unsuppported
# combinations of source contexts and policy deliveries are used.
source_context_list_scheme = spec_json['source_context_list_schema'][
selection['source_context_list']]
selection['source_context_list'] = [
util.SourceContext.from_json(source_context, target_policy_delivery,
spec_json['source_context_schema'])
for source_context in source_context_list_scheme['sourceContextList']
]
# Check if the subresource is supported by the innermost source context.
innermost_source_context = selection['source_context_list'][-1]
supported_subresource = spec_json['source_context_schema'][
'supported_subresource'][innermost_source_context.source_context_type]
if supported_subresource != '*':
if selection['subresource'] not in supported_subresource:
raise util.ShouldSkip()
# Parse subresource policy deliveries.
selection[
'subresource_policy_deliveries'] = util.PolicyDelivery.list_from_json(
source_context_list_scheme['subresourcePolicyDeliveries'],
target_policy_delivery, spec_json['subresource_schema']
['supported_delivery_type'][selection['subresource']])
# We process the top source context below, and do not include it in
# `scenario` field in JavaScript.
top_source_context = selection['source_context_list'].pop(0)
assert (top_source_context.source_context_type == 'top')
# Adjust the template for the test invoking JS. Indent it to look nice.
indent = "\n" + " " * 8
selection['scenario'] = dump_test_parameters(selection).replace(
"\n", indent)
selection['spec_name'] = spec['name']
selection[
'test_page_title'] = config.test_page_title_template % spec['title']
selection['spec_description'] = spec['description']
selection['spec_specification_url'] = spec['specification_url']
selection['helper_js'] = config.helper_js
selection['sanity_checker_js'] = config.sanity_checker_js
selection['spec_json_js'] = config.spec_json_js
test_headers_filename = test_filename + ".headers"
test_directory = os.path.dirname(test_filename)
test_html_template = util.get_template(test_html_template_basename)
disclaimer_template = util.get_template('disclaimer.template')
html_template_filename = os.path.join(util.template_directory,
test_html_template_basename)
generated_disclaimer = disclaimer_template \
% {'generating_script_filename': os.path.relpath(sys.argv[0],
util.test_root_directory),
'html_template_filename': os.path.relpath(html_template_filename,
util.test_root_directory)}
# Adjust the template for the test invoking JS. Indent it to look nice.
selection['generated_disclaimer'] = generated_disclaimer.rstrip()
selection[
'test_description'] = config.test_description_template % selection
selection['test_description'] = \
selection['test_description'].rstrip().replace("\n", "\n" + " " * 33)
# Directory for the test files.
try:
os.makedirs(test_directory)
except:
pass
delivery = handle_deliveries(top_source_context.policy_deliveries)
if len(delivery['headers']) > 0:
with open(test_headers_filename, "w") as f:
for header in delivery['headers']:
f.write('%s: %s\n' % (header, delivery['headers'][header]))
selection['meta_delivery_method'] = delivery['meta']
# Obey the lint and pretty format.
if len(selection['meta_delivery_method']) > 0:
selection['meta_delivery_method'] = "\n " + \
selection['meta_delivery_method']
# Write out the generated HTML file.
util.write_file(test_filename, test_html_template % selection)
def generate_test_file(spec_directory, test_helper_filenames,
test_html_template_basename, test_filename, scenarios):
'''
Generates a test HTML file (and possibly its associated .headers file)
from `scenarios`.
'''
# Scenarios for the same file should have the same `source_context_list`,
# including the top-level one.
# Note: currently, non-top-level source contexts aren't necessarily required
# to be the same, but we set this requirement as it will be useful e.g. when
# we e.g. reuse a worker among multiple scenarios.
for scenario in scenarios:
assert (scenario['source_context_list'] == scenarios[0]
['source_context_list'])
# We process the top source context below, and do not include it in
# the JSON objects (i.e. `scenarios`) in generated HTML files.
top_source_context = scenarios[0]['source_context_list'].pop(0)
assert (top_source_context.source_context_type == 'top')
for scenario in scenarios[1:]:
assert (scenario['source_context_list'].pop(0) == top_source_context)
parameters = {}
# Sort scenarios, to avoid unnecessary diffs due to different orders in
# `scenarios`.
serialized_scenarios = sorted(
[dump_test_parameters(scenario) for scenario in scenarios])
parameters['scenarios'] = ",\n".join(serialized_scenarios).replace(
"\n", "\n" + " " * 10)
test_directory = os.path.dirname(test_filename)
parameters['helper_js'] = ""
for test_helper_filename in test_helper_filenames:
parameters['helper_js'] += ' <script src="%s"></script>\n' % (
os.path.relpath(test_helper_filename, test_directory))
parameters['sanity_checker_js'] = os.path.relpath(
os.path.join(spec_directory, 'generic', 'sanity-checker.js'),
test_directory)
parameters['spec_json_js'] = os.path.relpath(
os.path.join(spec_directory, 'generic', 'spec_json.js'),
test_directory)
test_headers_filename = test_filename + ".headers"
test_html_template = util.get_template(test_html_template_basename)
disclaimer_template = util.get_template('disclaimer.template')
html_template_filename = os.path.join(util.template_directory,
test_html_template_basename)
generated_disclaimer = disclaimer_template \
% {'generating_script_filename': os.path.relpath(sys.argv[0],
util.test_root_directory),
'spec_directory': os.path.relpath(spec_directory,
util.test_root_directory)}
# Adjust the template for the test invoking JS. Indent it to look nice.
parameters['generated_disclaimer'] = generated_disclaimer.rstrip()
# Directory for the test files.
try:
os.makedirs(test_directory)
except:
pass
delivery = handle_deliveries(top_source_context.policy_deliveries)
if len(delivery['headers']) > 0:
with open(test_headers_filename, "w") as f:
for header in delivery['headers']:
f.write('%s: %s\n' % (header, delivery['headers'][header]))
parameters['meta_delivery_method'] = delivery['meta']
# Obey the lint and pretty format.
if len(parameters['meta_delivery_method']) > 0:
parameters['meta_delivery_method'] = "\n " + \
parameters['meta_delivery_method']
# Write out the generated HTML file.
util.write_file(test_filename, test_html_template % parameters)
def generate_test_source_files(spec_directory, test_helper_filenames,
spec_json, target):
test_expansion_schema = spec_json['test_expansion_schema']
specification = spec_json['specification']
if target == "debug":
spec_json_js_template = util.get_template('spec_json.js.template')
util.write_file(
os.path.join(spec_directory, "generic", "spec_json.js"),
spec_json_js_template % {'spec_json': json.dumps(spec_json)})
util.write_file(
os.path.join(spec_directory, "generic",
"debug-output.spec.src.json"),
json.dumps(spec_json, indent=2, separators=(',', ': ')))
# Choose a debug/release template depending on the target.
html_template = "test.%s.html.template" % target
artifact_order = test_expansion_schema.keys()
artifact_order.remove('expansion')
excluded_selection_pattern = ''
for key in artifact_order:
excluded_selection_pattern += '%(' + key + ')s/'
# Create list of excluded tests.
exclusion_dict = set()
for excluded_pattern in spec_json['excluded_tests']:
excluded_expansion = \
expand_pattern(excluded_pattern, test_expansion_schema)
for excluded_selection in permute_expansion(excluded_expansion,
artifact_order):
excluded_selection['delivery_key'] = spec_json['delivery_key']
exclusion_dict.add(excluded_selection_pattern % excluded_selection)
# `scenarios[filename]` represents the list of scenario objects to be
# generated into `filename`.
scenarios = {}
for spec in specification:
# Used to make entries with expansion="override" override preceding
# entries with the same |selection_path|.
output_dict = {}
for expansion_pattern in spec['test_expansion']:
expansion = expand_pattern(expansion_pattern,
test_expansion_schema)
for selection in permute_expansion(expansion, artifact_order):
selection['delivery_key'] = spec_json['delivery_key']
selection_path = spec_json['selection_pattern'] % selection
if selection_path in output_dict:
if expansion_pattern['expansion'] != 'override':
print("Error: expansion is default in:")
print(dump_test_parameters(selection))
print("but overrides:")
print(dump_test_parameters(
output_dict[selection_path]))
sys.exit(1)
output_dict[selection_path] = copy.deepcopy(selection)
for selection_path in output_dict:
selection = output_dict[selection_path]
if (excluded_selection_pattern % selection) in exclusion_dict:
print('Excluding selection:', selection_path)
continue
try:
test_filename = get_test_filename(spec_directory, spec_json,
selection)
scenario = generate_selection(spec_json, selection)
scenarios[test_filename] = scenarios.get(test_filename,
[]) + [scenario]
except util.ShouldSkip:
continue
for filename in scenarios:
generate_test_file(spec_directory, test_helper_filenames,
html_template, filename, scenarios[filename])
def submit_issue_response(repo_username, repo_id, number, action_type, message_type, custom_message):
data = {
'repo_username': repo_username,
'repo_id': repo_id,
'number': number,
'action_type': action_type,
'message_type': message_type,
'custom_message': custom_message
}
try:
issue = github_api.fetch_issue(repo_username, repo_id, number)
if not issue or issue.get('error'):
data['error'] = 'could not find issue %s' % number
return data
context = {
'issue': issue,
'user': issue.get('user')
}
msg = None
if message_type == 'expire':
msg = util.get_template('EXPIRE_TEMPLATE', context)
elif message_type == 'forum':
msg = util.get_template('FORUM_TEMPLATE', context)
elif message_type == 'inapplicable':
msg = util.get_template('INAPPLICABLE_TEMPLATE', context)
elif message_type == 'more':
msg = util.get_template('MORE_TEMPLATE', context)
elif message_type == 'feature':
github_api.add_issue_labels(repo_username, repo_id, number, [cvar['FEATURE_REQUEST_LABEL']], issue=issue)
msg = util.get_template('FEATURE_REQUEST_TEMPLATE', context)
elif message_type == 'no_reply':
msg = util.get_template('CLOSING_NOREPLY_TEMPLATE', context)
elif message_type == 'pr_close':
msg = util.get_template('CLOSE_PULL_REQUEST_TEMPLATE', context)
elif message_type == 'custom':
msg = custom_message
elif message_type == 'close':
msg = None
else:
data['error'] = 'invalid message_type: %s' % message_type
return data
if msg and len(msg.strip()):
data['created_comment'] = github_api.create_issue_comment(repo_username, repo_id, number, msg)
if action_type == 'close':
data['issue_closed'] = github_api.close_issue(repo_username, repo_id, number, issue)
elif action_type == 'reply':
github_api.add_issue_labels(repo_username, repo_id, number, [cvar['NEEDS_REPLY_LABEL']], issue=issue)
except Exception as ex:
print 'submit_issue_response error, %s: %s' % (number, ex)
data['error'] = '%s' % ex
return data
continue
market = get_correct_market(s, rf.year)
ann_market_return = market.get_annualised_return(rf.year)
req_rate = CAPM(rf.spot_yield, ann_market_return, s.beta)
try:
greeks = get_greeks(s, market, rf)
gg_result = gordon_growth_range(s, req_rate)
fcf_result = fcf_growth_range(s, req_rate)
forward_pe_result = forward_pe_range(s, req_rate)
report_name = f"{util.today_date()}.html"
template = util.get_template("report_template.html")
template_vars = {"spot_yield": util.convert_to_pct(rf.spot_yield), "market_ann_return": util.convert_to_pct( market.get_annualised_return(rf.year)), "benchmark_year": rf.year, "req_rate": util.convert_to_pct(req_rate),
"greeks": greeks, "gordon_growth": gg_result, "fcf_growth": fcf_result, "forward_pe": forward_pe_result,
"timestamp": util.today_date()}
html_out = template.render(template_vars, s = s)
logging.debug("Generated report file")
util.dump_report(html_out, f"/tmp/{report_name}")
logging.debug("Created report file")
util.upload_report_to_s3("/tmp/" + report_name, s3_bucket, s3_report_dir + "/" + s.stock_code + "/" +report_name)
db.update_item(Key={"ticker": s.stock_code}, UpdateExpression = f"set price = :price",
ExpressionAttributeValues = {":price": Decimal(str(s.get_current_price()))})
util.update_valuation(db, s.stock_code, util.Valuation.GORDON_GROWTH.value, util.extract_valuation(gg_result))
def build(
from_,
name,
apt,
pip3,
with_azurecli,
with_kubectl,
with_velero,
force,
dry_run,
output,
):
"""
Build Docker images with custom packages.
\n
Examples:
\n
Build an image and install vim and curl using apt-get.
\n
$ dugaire build --apt=vim,curl
\n
Build an image and install python3 using apt-get and ansible using pip3.
\n
$ dugaire build --apt=python3-pip --pip3=ansible
\n
Build an image and install the latest version of kubectl.
\n
$ dugaire build --with-kubectl=latest
\n
"""
dockerfile = ""
template = util.get_template("base.j2")
dockerfile += template.render(
from_=from_, label=util.get_dugaire_image_label("dockerfile"))
if apt:
packages = apt.replace(",", " ")
template = util.get_template("apt.j2")
dockerfile += template.render(packages=packages)
if pip3:
dependency_list = {}
dependency_list["azure-cli"] = ["gcc", "python3-dev"]
pip3_install = pip3.split(",")
for package in pip3_install:
package_name = package.split("==")[0]
if package_name in dependency_list:
dependency = " ".join(dependency_list[package_name])
apt_template = util.get_template("apt.j2")
dockerfile += apt_template.render(packages=dependency)
packages = " ".join(pip3_install)
template = util.get_template("pip3.j2")
dockerfile += template.render(packages=packages)
if with_kubectl:
current_option_name = "--with-kubectl"
current_option_value = with_kubectl
if not util.string_is_latest_or_version(current_option_value):
usage_msg = f"{current_option_name}=<latest | semantic versioning>"
example_msg = f"{current_option_name}=latest | {current_option_name}=1.17.0"
exc_msg = f"Bad usage {current_option_name}={current_option_value} \n"
exc_msg += f"Valid usage: {usage_msg} \n"
exc_msg += f"Examples: {example_msg}"
raise click.BadOptionUsage(current_option_name, exc_msg)
dependency_list = {}
dependency_list = ["curl", "ca-certificates"]
dependency = " ".join(dependency_list)
apt_template = util.get_template("apt.j2")
dockerfile += apt_template.render(packages=dependency)
url = "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl"
if with_kubectl != "latest":
url = f"https://storage.googleapis.com/kubernetes-release/release/v{with_kubectl}/bin/linux/amd64/kubectl"
template = util.get_template("with_kubectl.j2")
dockerfile += template.render(url=url)
if with_azurecli:
dependency_list = {}
dependency_list = ["curl", "ca-certificates"]
dependency = " ".join(dependency_list)
apt_template = util.get_template("apt.j2")
dockerfile += apt_template.render(packages=dependency)
template = util.get_template("with_azurecli.j2")
dockerfile += template.render()
if with_velero:
current_option_name = "--with-velero"
current_option_value = with_velero
if not util.string_is_latest_or_version(current_option_value):
usage_msg = f"{current_option_name}=<latest | semantic versioning>"
example_msg = f"{current_option_name}=latest | {current_option_name}=1.5.2"
exc_msg = f"Bad usage {current_option_name}={current_option_value} \n"
exc_msg += f"Valid usage: {usage_msg} \n"
exc_msg += f"Examples: {example_msg}"
raise click.BadOptionUsage(current_option_name, exc_msg)
if not with_kubectl:
usage_msg = f"--with-kubectl=<latest | semantic versioning> {current_option_name}=<latest | semantic versioning>"
example_msg = f"--with-kubectl=latest {current_option_name}=latest"
exc_msg = f"Bad usage {current_option_name} requires --with-kubectl \n"
exc_msg += f"Valid usage: {usage_msg} \n"
exc_msg += f"Examples: {example_msg}"
raise click.BadOptionUsage(current_option_name, exc_msg)
dependency_list = {}
dependency_list = ["wget"]
dependency = " ".join(dependency_list)
apt_template = util.get_template("apt.j2")
dockerfile += apt_template.render(packages=dependency)
if with_velero == "latest":
import urllib.request
response = urllib.request.urlopen(
"https://api.github.com/repos/vmware-tanzu/velero/releases/latest"
).read()
response = json.loads(response)
with_velero = response["tag_name"][1:]
template = util.get_template("with_velero.j2")
dockerfile += template.render(version=with_velero)
image_id = None
image_name = None
if not dry_run:
f = BytesIO(dockerfile.encode("utf-8"))
client = docker.from_env()
image_name = name
if image_name == "random":
random_name = str(uuid.uuid4())[:8]
random_tag = str(uuid.uuid4())[:8]
image_name = f"dug-{random_name}:{random_tag}"
image, error = client.images.build(fileobj=f,
tag=image_name,
nocache=force,
rm=True,
forcerm=True)
image_id = image.attrs["Id"]
image_name = image.attrs["RepoTags"][0]
if output == "image.id":
click.echo(image_id)
if output == "image.id.short":
click.echo(image_id.replace("sha256:", "")[:12])
if output == "image.name":
click.echo(image_name)
if output == "dockerfile":
click.echo(dockerfile)