예제 #1
0
def application(environ, start_response):

    # translate ENVIRON variable about query string to dict
    query = parse_qs(environ['QUERY_STRING'])

    # As there can ben more than one value for a variable then
    # a list is provied as a default for parse_qs
    age = query.get('age', [''])[0]
    hobbies = query.get('hobbies', [])

    age = escape(age)
    hobbies = [escape(hobby) for hobby in hobbies]

    html = get_template('template.html')
    body = html % {
        'method': 'GET',
        'checked-software': ('', 'checked')['software' in hobbies],
        'checked-tunning': ('', 'checked')['tunning' in hobbies],
        'age': age or 'Empty',
        'hobbies': ', '.join(hobbies or ['No Hobbies?'])
    }

    status = '200 OK'

    headers = [('Content-Type', 'text/html'),
               ('Content-Lenght', str(len(body)))]

    start_response(status, headers)
    return [bytes(body, 'utf-8')]
예제 #2
0
def main(request, response):
    policyDeliveries = json.loads(request.GET.first('policyDeliveries', '[]'))
    worker_type = request.GET.first('type', 'classic')
    commonjs_url = '%s://%s:%s/common/security-features/resources/common.sub.js' % (
        request.url_parts.scheme, request.url_parts.hostname,
        request.url_parts.port)
    if worker_type == 'classic':
        import_line = 'importScripts("%s");' % commonjs_url
    else:
        import_line = 'import "%s";' % commonjs_url

    maybe_additional_headers = {}
    error = ''
    for delivery in policyDeliveries:
        if delivery['deliveryType'] == 'meta':
            error = '<meta> cannot be used in WorkerGlobalScope'
        elif delivery['deliveryType'] == 'http-rp':
            if delivery['key'] == 'referrerPolicy':
                maybe_additional_headers['Referrer-Policy'] = delivery['value']
            else:
                error = 'invalid delivery key for http-rp: %s' % delivery['key']
        else:
            error = 'invalid deliveryType: %s' % delivery['deliveryType']

    handler = lambda: util.get_template('worker.js.template') % ({
        'import': import_line,
        'error': error
    })
    util.respond(request,
                 response,
                 payload_generator=handler,
                 content_type='text/javascript',
                 maybe_additional_headers=maybe_additional_headers)
예제 #3
0
def main(request, response):
    policyDeliveries = json.loads(request.GET.first(b"policyDeliveries",
                                                    b"[]"))
    maybe_additional_headers = {}
    meta = u''
    error = u''
    for delivery in policyDeliveries:
        if delivery[u'deliveryType'] == u'meta':
            if delivery[u'key'] == u'referrerPolicy':
                meta += u'<meta name="referrer" content="%s">' % delivery[
                    u'value']
            else:
                error = u'invalid delivery key'
        elif delivery[u'deliveryType'] == u'http-rp':
            if delivery[u'key'] == u'referrerPolicy':
                maybe_additional_headers[
                    b'Referrer-Policy'] = isomorphic_encode(delivery[u'value'])
            else:
                error = u'invalid delivery key'
        else:
            error = u'invalid deliveryType'

    handler = lambda: util.get_template(u"document.html.template"
                                        ) % ({
                                            u"meta": meta,
                                            u"error": error
                                        })
    util.respond(request,
                 response,
                 payload_generator=handler,
                 content_type=b"text/html",
                 maybe_additional_headers=maybe_additional_headers)
예제 #4
0
파일: generate.py 프로젝트: ihsavru/servo
def generate_test_source_files(spec_directory, test_helper_filenames,
                               spec_json, target):
    test_expansion_schema = spec_json['test_expansion_schema']
    specification = spec_json['specification']

    spec_json_js_template = util.get_template('spec_json.js.template')
    util.write_file(
        os.path.join(spec_directory, "generic", "spec_json.js"),
        spec_json_js_template % {'spec_json': json.dumps(spec_json)})

    util.write_file(
        os.path.join(spec_directory, "generic", "debug-output.spec.src.json"),
        json.dumps(spec_json, indent=2, separators=(',', ': ')))

    # Choose a debug/release template depending on the target.
    html_template = "test.%s.html.template" % target

    artifact_order = test_expansion_schema.keys() + ['name']
    artifact_order.remove('expansion')

    # Create list of excluded tests.
    exclusion_dict = {}
    for excluded_pattern in spec_json['excluded_tests']:
        excluded_expansion = \
            expand_pattern(excluded_pattern, test_expansion_schema)
        for excluded_selection in permute_expansion(excluded_expansion,
                                                    artifact_order):
            excluded_selection['delivery_key'] = spec_json['delivery_key']
            exclusion_dict[dump_test_parameters(excluded_selection)] = True

    for spec in specification:
        # Used to make entries with expansion="override" override preceding
        # entries with the same |selection_path|.
        output_dict = {}

        for expansion_pattern in spec['test_expansion']:
            expansion = expand_pattern(expansion_pattern,
                                       test_expansion_schema)
            for selection in permute_expansion(expansion, artifact_order):
                selection['delivery_key'] = spec_json['delivery_key']
                selection_path = spec_json['selection_pattern'] % selection
                if selection_path in output_dict:
                    if expansion_pattern['expansion'] != 'override':
                        print(
                            "Error: %s's expansion is default but overrides %s"
                            % (selection['name'],
                               output_dict[selection_path]['name']))
                        sys.exit(1)
                output_dict[selection_path] = copy.deepcopy(selection)

        for selection_path in output_dict:
            selection = output_dict[selection_path]
            if dump_test_parameters(selection) in exclusion_dict:
                print('Excluding selection:', selection_path)
                continue
            try:
                generate_selection(spec_directory, test_helper_filenames,
                                   spec_json, selection, spec, html_template)
            except util.ShouldSkip:
                continue
예제 #5
0
파일: main.py 프로젝트: pheezer/malt.io
def handle_error(request, response, exception, code, msg):
    """
    Render an error template with a code and message.
    """
    logging.exception(exception)

    # Try to get the current user
    session_store = sessions.get_store(request=request)
    session = session_store.get_session()
    auth_id = session.get('auth_id')
    user = None
    if auth_id:
        user = UserPrefs.get(auth_id)

    if user:
        logging.error('Currently logged in user is ' + user.name + ' (' +
                      user.email + ')')

    # Render and error template
    template = get_template('error.html')
    response.status = code
    response.out.write(
        template.render({
            'user': user,
            'code': code,
            'message': msg
        }))
def flag_if_submitted_through_github(repo_username, repo_id, issue):
    """
    Flags any issue that is submitted through github's UI, and not the Ionic site.
    Adds a label, as well as a comment, to force the issue through the custom form.
    @return: whether or not the issue was flagged (bool)
    """

    if not issue:
        return False

    number = issue.get("number")
    if not number:
        return False

    user = issue.get("user")
    if not user:
        return False

    if not issue.get("body"):
        return False

    if is_valid_issue_opened_source(repo_username, repo_id, issue):
        return False

    context = {"issue": issue, "user": user}
    msg = util.get_template("RESUBMIT_TEMPLATE", context)

    github_api.create_issue_comment(repo_username, repo_id, number, msg)

    return True
예제 #7
0
def flag_if_submitted_through_github(repo_username, repo_id, issue):
    return False # temporarily disabling ionitron

    """
    Flags any issue that is submitted through github's UI, and not the Ionic site.
    Adds a label, as well as a comment, to force the issue through the custom form.
    @return: whether or not the issue was flagged (bool)
    """

    if not issue:
        return False

    number = issue.get('number')
    if not number:
        return False

    user = issue.get('user')
    if not user:
        return False

    if not issue.get('body'):
        return False

    if is_valid_issue_opened_source(repo_username, repo_id, issue):
        return False

    context = {
        'issue': issue,
        'user': user
    }
    msg = util.get_template('RESUBMIT_TEMPLATE', context)

    github_api.create_issue_comment(repo_username, repo_id, number, msg)

    return True
def main(request, response):
    policyDeliveries = json.loads(request.GET.first("policyDeliveries", "[]"))
    maybe_additional_headers = {}
    meta = ''
    error = ''
    for delivery in policyDeliveries:
        if delivery['deliveryType'] == 'meta':
            if delivery['key'] == 'referrerPolicy':
                meta += '<meta name="referrer" content="%s">' % delivery[
                    'value']
            else:
                error = 'invalid delivery key'
        elif delivery['deliveryType'] == 'http-rp':
            if delivery['key'] == 'referrerPolicy':
                maybe_additional_headers['Referrer-Policy'] = delivery['value']
            else:
                error = 'invalid delivery key'
        else:
            error = 'invalid deliveryType'

    handler = lambda: util.get_template("document.html.template"
                                        ) % ({
                                            "meta": meta,
                                            "error": error
                                        })
    util.respond(request,
                 response,
                 payload_generator=handler,
                 content_type="text/html",
                 maybe_additional_headers=maybe_additional_headers)
예제 #9
0
def main(request, response):
  policyDeliveries = json.loads(request.GET.first('policyDeliveries', '[]'))
  worker_type = request.GET.first('type', 'classic')
  commonjs_url = '%s://%s:%s/common/security-features/resources/common.js' % (
      request.url_parts.scheme, request.url_parts.hostname,
      request.url_parts.port)
  if worker_type == 'classic':
    import_line = 'importScripts("%s");' % commonjs_url
  else:
    import_line = 'import "%s";' % commonjs_url

  maybe_additional_headers = {}
  error = ''
  for delivery in policyDeliveries:
    if delivery['deliveryType'] == 'meta':
      error = '<meta> cannot be used in WorkerGlobalScope'
    elif delivery['deliveryType'] == 'http-rp':
      if delivery['key'] == 'referrerPolicy':
        maybe_additional_headers['Referrer-Policy'] = delivery['value']
      else:
        error = 'invalid delivery key for http-rp: %s' % delivery['key']
    else:
      error = 'invalid deliveryType: %s' % delivery['deliveryType']

  handler = lambda: util.get_template('worker.js.template') % ({
      'import': import_line,
      'error': error
  })
  util.respond(
      request,
      response,
      payload_generator=handler,
      content_type='text/javascript',
      maybe_additional_headers=maybe_additional_headers)
예제 #10
0
def application(environ, start_response):

    # PEP 3333 says: CONTENT_LENGTH may be empty or missing
    try:
        request_content_length = int(environ.get('CONTENT_LENGTH', 0))
    except ValueError:
        request_content_length = 0

    # Request Body is passed inside wsgi.input environment variable
    request_body = environ['wsgi.input'].read(request_content_length)
    form = parse_qs(request_body.decode('utf-8'))

    age = form.get('age', [''])[0]
    hobbies = form.get('hobbies', [])

    age = escape(age)
    hobbies = [escape(hobby) for hobby in hobbies]

    html = get_template('template.html')
    body = html % {
        'method': 'POST',
        'checked-software': ('', 'checked')['software' in hobbies],
        'checked-tunning': ('', 'checked')['tunning' in hobbies],
        'age': age or 'Empty',
        'hobbies': ', '.join(hobbies or ['No Hobbies?'])
    }

    status = '200 OK'

    headers = [('Content-Type', 'text/html'),
               ('Content-Lenght', str(len(body)))]

    start_response(status, headers)
    return [bytes(body, 'utf-8')]
예제 #11
0
def generate_test_source_files(config, spec_json, target):
    test_expansion_schema = spec_json['test_expansion_schema']
    specification = spec_json['specification']

    spec_json_js_template = util.get_template('spec_json.js.template')
    generated_spec_json_filename = os.path.join(config.spec_directory,
                                                "spec_json.js")
    util.write_file(
        generated_spec_json_filename,
        spec_json_js_template % {'spec_json': json.dumps(spec_json)})

    # Choose a debug/release template depending on the target.
    html_template = "test.%s.html.template" % target

    artifact_order = test_expansion_schema.keys() + ['name']
    artifact_order.remove('expansion')

    # Create list of excluded tests.
    exclusion_dict = {}
    for excluded_pattern in spec_json['excluded_tests']:
        excluded_expansion = \
            expand_pattern(excluded_pattern, test_expansion_schema)
        for excluded_selection in permute_expansion(excluded_expansion,
                                                    artifact_order):
            excluded_selection_path = config.selection_pattern % excluded_selection
            exclusion_dict[excluded_selection_path] = True

    for spec in specification:
        # Used to make entries with expansion="override" override preceding
        # entries with the same |selection_path|.
        output_dict = {}

        for expansion_pattern in spec['test_expansion']:
            expansion = expand_pattern(expansion_pattern,
                                       test_expansion_schema)
            for selection in permute_expansion(expansion, artifact_order):
                selection['delivery_key'] = spec_json['delivery_key']
                selection_path = config.selection_pattern % selection
                if not selection_path in exclusion_dict:
                    if selection_path in output_dict:
                        if expansion_pattern['expansion'] != 'override':
                            print(
                                "Error: %s's expansion is default but overrides %s"
                                % (selection['name'],
                                   output_dict[selection_path]['name']))
                            sys.exit(1)
                    output_dict[selection_path] = copy.deepcopy(selection)
                else:
                    print('Excluding selection:', selection_path)

        for selection_path in output_dict:
            selection = output_dict[selection_path]
            generate_selection(config, selection, spec, html_template)
예제 #12
0
def close_old_issue(repo_username, repo_id, number, issue):
    context = {
        'issue': issue,
        'user': issue.get('user')
    }
    comment = util.get_template('CLOSING_TEMPLATE', context)

    github_api.close_issue(repo_username, repo_id, number, issue)
    github_api.create_issue_comment(repo_username, repo_id, number, comment)

    return {
        'closed_old_issue': True
    }
예제 #13
0
def generate_test_source_files(config, spec_json, target):
    test_expansion_schema = spec_json['test_expansion_schema']
    specification = spec_json['specification']

    spec_json_js_template = util.get_template('spec_json.js.template')
    generated_spec_json_filename = os.path.join(config.spec_directory, "spec_json.js")
    util.write_file(generated_spec_json_filename,
               spec_json_js_template % {'spec_json': json.dumps(spec_json)})

    # Choose a debug/release template depending on the target.
    html_template = "test.%s.html.template" % target

    artifact_order = test_expansion_schema.keys() + ['name']
    artifact_order.remove('expansion')

    # Create list of excluded tests.
    exclusion_dict = {}
    for excluded_pattern in spec_json['excluded_tests']:
        excluded_expansion = \
            expand_pattern(excluded_pattern, test_expansion_schema)
        for excluded_selection in permute_expansion(excluded_expansion,
                                                    artifact_order):
            excluded_selection_path = config.selection_pattern % excluded_selection
            exclusion_dict[excluded_selection_path] = True

    for spec in specification:
        # Used to make entries with expansion="override" override preceding
        # entries with the same |selection_path|.
        output_dict = {}

        for expansion_pattern in spec['test_expansion']:
            expansion = expand_pattern(expansion_pattern, test_expansion_schema)
            for selection in permute_expansion(expansion, artifact_order):
                selection_path = config.selection_pattern % selection
                if not selection_path in exclusion_dict:
                    if selection_path in output_dict:
                        if expansion_pattern['expansion'] != 'override':
                            print("Error: %s's expansion is default but overrides %s" % (selection['name'], output_dict[selection_path]['name']))
                            sys.exit(1)
                    output_dict[selection_path] = copy.deepcopy(selection)
                else:
                    print('Excluding selection:', selection_path)

        for selection_path in output_dict:
            selection = output_dict[selection_path]
            generate_selection(config,
                               selection,
                               spec,
                               html_template)
예제 #14
0
파일: main.py 프로젝트: jensyt/malt.io
def handle_error(request, response, exception, code, msg):
    """
    Render an error template with a code and message.
    """
    logging.exception(exception)

    # Try to get the current user
    session_store = sessions.get_store(request=request)
    session = session_store.get_session()
    auth_id = session.get("auth_id")
    user = None
    if auth_id:
        user = UserPrefs.get(auth_id)

    # Render and error template
    template = get_template("error.html")
    response.out.write(template.render({"user": user, "code": code, "message": msg}))
예제 #15
0
def gen_code(usr):
    """generate code to be emailed to user"""

    email = usr.get("email")
    if not is_user_exist(email):
        raise EXCEPTION_USER_NOT_EXIST

    part1, part2 = rand_code(), rand_code()
    res = dbase.update("users", "where email=?", [email], {
        "code": part1 + "-" + part2,
        "tstamp": int(time.time())
    })

    if not res:
        raise EXCEPTION_DATABASE

    sendmail({
        "to": email,
        "subj": "logging in",
        "text": get_template("email_verification") % (part1 + "-" + part2)
    })
    return part1
예제 #16
0
def main(request, response):
  policyDeliveries = json.loads(request.GET.first(b'policyDeliveries', b'[]'))
  worker_type = request.GET.first(b'type', b'classic')
  commonjs_url = u'%s://%s:%s/common/security-features/resources/common.sub.js' % (
      request.url_parts.scheme, request.url_parts.hostname,
      request.url_parts.port)
  if worker_type == b'classic':
    import_line = u'importScripts("%s");' % commonjs_url
  else:
    import_line = u'import "%s";' % commonjs_url

  maybe_additional_headers = {}
  error = u''
  for delivery in policyDeliveries:
    if delivery[u'deliveryType'] == u'meta':
      error = u'<meta> cannot be used in WorkerGlobalScope'
    elif delivery[u'deliveryType'] == u'http-rp':
      if delivery[u'key'] == u'referrerPolicy':
        maybe_additional_headers[b'Referrer-Policy'] = isomorphic_encode(delivery[u'value'])
      elif delivery[u'key'] == u'mixedContent' and delivery[u'value'] == u'opt-in':
        maybe_additional_headers[b'Content-Security-Policy'] = b'block-all-mixed-content'
      elif delivery[u'key'] == u'upgradeInsecureRequests' and delivery[u'value'] == u'upgrade':
        maybe_additional_headers[b'Content-Security-Policy'] = b'upgrade-insecure-requests'
      else:
        error = u'invalid delivery key for http-rp: %s' % delivery[u'key']
    else:
      error = u'invalid deliveryType: %s' % delivery[u'deliveryType']

  handler = lambda: util.get_template(u'worker.js.template') % ({
      u'import': import_line,
      u'error': error
  })
  util.respond(
      request,
      response,
      payload_generator=handler,
      content_type=b'text/javascript',
      maybe_additional_headers=maybe_additional_headers)
예제 #17
0
파일: generate.py 프로젝트: xoob/servo
def generate_selection(config, selection, spec, test_html_template_basename):
    test_parameters = dump_test_parameters(selection)
    # Adjust the template for the test invoking JS. Indent it to look nice.
    indent = "\n" + " " * 8
    test_parameters = test_parameters.replace("\n", indent)

    selection['test_js'] = '''
      %s(
        %s,
        document.querySelector("meta[name=assert]").content,
        new SanityChecker()
      ).start();
      ''' % (config.test_case_name, test_parameters)

    selection['spec_name'] = spec['name']
    selection[
        'test_page_title'] = config.test_page_title_template % spec['title']
    selection['spec_description'] = spec['description']
    selection['spec_specification_url'] = spec['specification_url']
    selection['helper_js'] = config.helper_js
    selection['sanity_checker_js'] = config.sanity_checker_js
    selection['spec_json_js'] = config.spec_json_js

    test_filename = get_test_filename(config, selection)
    test_headers_filename = test_filename + ".headers"
    test_directory = os.path.dirname(test_filename)

    test_html_template = util.get_template(test_html_template_basename)
    disclaimer_template = util.get_template('disclaimer.template')

    html_template_filename = os.path.join(util.template_directory,
                                          test_html_template_basename)
    generated_disclaimer = disclaimer_template \
        % {'generating_script_filename': os.path.relpath(sys.argv[0],
                                                         util.test_root_directory),
           'html_template_filename': os.path.relpath(html_template_filename,
                                                     util.test_root_directory)}

    # Adjust the template for the test invoking JS. Indent it to look nice.
    selection['generated_disclaimer'] = generated_disclaimer.rstrip()
    selection[
        'test_description'] = config.test_description_template % selection
    selection['test_description'] = \
        selection['test_description'].rstrip().replace("\n", "\n" + " " * 33)

    # Directory for the test files.
    try:
        os.makedirs(test_directory)
    except:
        pass

    delivery = handle_deliveries([
        util.PolicyDelivery(selection['delivery_type'],
                            selection['delivery_key'],
                            selection['delivery_value'])
    ])

    if len(delivery['headers']) > 0:
        with open(test_headers_filename, "w") as f:
            for header in delivery['headers']:
                f.write('%s: %s\n' % (header, delivery['headers'][header]))

    selection['meta_delivery_method'] = delivery['meta']
    # Obey the lint and pretty format.
    if len(selection['meta_delivery_method']) > 0:
        selection['meta_delivery_method'] = "\n    " + \
                                            selection['meta_delivery_method']

    # Write out the generated HTML file.
    util.write_file(test_filename, test_html_template % selection)
예제 #18
0
def generate_selection(config, selection, spec, test_html_template_basename):
    # TODO: Refactor out this referrer-policy-specific part.
    if 'referrer_policy' in spec:
      # Oddball: it can be None, so in JS it's null.
      selection['referrer_policy'] = spec['referrer_policy']

    test_parameters = json.dumps(selection, indent=2, separators=(',', ':'))
    # Adjust the template for the test invoking JS. Indent it to look nice.
    indent = "\n" + " " * 8
    test_parameters = test_parameters.replace("\n", indent)

    selection['test_js'] = '''
      %s(
        %s,
        document.querySelector("meta[name=assert]").content,
        new SanityChecker()
      ).start();
      ''' % (config.test_case_name, test_parameters)

    selection['spec_name'] = spec['name']
    selection['test_page_title'] = config.test_page_title_template % spec['title']
    selection['spec_description'] = spec['description']
    selection['spec_specification_url'] = spec['specification_url']
    selection['helper_js'] = config.helper_js
    selection['sanity_checker_js'] = config.sanity_checker_js
    selection['spec_json_js'] = config.spec_json_js

    test_filename = os.path.join(config.spec_directory, config.test_file_path_pattern % selection)
    test_headers_filename = test_filename + ".headers"
    test_directory = os.path.dirname(test_filename)

    test_html_template = util.get_template(test_html_template_basename)
    disclaimer_template = util.get_template('disclaimer.template')

    html_template_filename = os.path.join(util.template_directory,
                                          test_html_template_basename)
    generated_disclaimer = disclaimer_template \
        % {'generating_script_filename': os.path.relpath(__file__,
                                                         util.test_root_directory),
           'html_template_filename': os.path.relpath(html_template_filename,
                                                     util.test_root_directory)}

    # Adjust the template for the test invoking JS. Indent it to look nice.
    selection['generated_disclaimer'] = generated_disclaimer.rstrip()
    selection['test_description'] = config.test_description_template % selection
    selection['test_description'] = \
        selection['test_description'].rstrip().replace("\n", "\n" + " " * 33)

    # Directory for the test files.
    try:
        os.makedirs(test_directory)
    except:
        pass

    delivery = config.handleDelivery(selection, spec)

    if len(delivery['headers']) > 0:
        with open(test_headers_filename, "w") as f:
            for header in delivery['headers']:
                f.write(header)
                f.write('\n')

    selection['meta_delivery_method'] = delivery['meta']
    # Obey the lint and pretty format.
    if len(selection['meta_delivery_method']) > 0:
        selection['meta_delivery_method'] = "\n    " + \
                                            selection['meta_delivery_method']

    # Write out the generated HTML file.
    util.write_file(test_filename, test_html_template % selection)
예제 #19
0
def generate_selection(spec_json, config, selection, spec,
                       test_html_template_basename):
    test_filename = get_test_filename(config, selection)

    target_policy_delivery = util.PolicyDelivery(selection['delivery_type'],
                                                 selection['delivery_key'],
                                                 selection['delivery_value'])
    del selection['delivery_type']
    del selection['delivery_key']
    del selection['delivery_value']

    # Parse source context list and policy deliveries of source contexts.
    # `util.ShouldSkip()` exceptions are raised if e.g. unsuppported
    # combinations of source contexts and policy deliveries are used.
    source_context_list_scheme = spec_json['source_context_list_schema'][
        selection['source_context_list']]
    selection['source_context_list'] = [
        util.SourceContext.from_json(source_context, target_policy_delivery,
                                     spec_json['source_context_schema'])
        for source_context in source_context_list_scheme['sourceContextList']
    ]

    # Check if the subresource is supported by the innermost source context.
    innermost_source_context = selection['source_context_list'][-1]
    supported_subresource = spec_json['source_context_schema'][
        'supported_subresource'][innermost_source_context.source_context_type]
    if supported_subresource != '*':
        if selection['subresource'] not in supported_subresource:
            raise util.ShouldSkip()

    # Parse subresource policy deliveries.
    selection[
        'subresource_policy_deliveries'] = util.PolicyDelivery.list_from_json(
            source_context_list_scheme['subresourcePolicyDeliveries'],
            target_policy_delivery, spec_json['subresource_schema']
            ['supported_delivery_type'][selection['subresource']])

    # We process the top source context below, and do not include it in
    # `scenario` field in JavaScript.
    top_source_context = selection['source_context_list'].pop(0)
    assert (top_source_context.source_context_type == 'top')

    # Adjust the template for the test invoking JS. Indent it to look nice.
    indent = "\n" + " " * 8
    selection['scenario'] = dump_test_parameters(selection).replace(
        "\n", indent)

    selection['spec_name'] = spec['name']
    selection[
        'test_page_title'] = config.test_page_title_template % spec['title']
    selection['spec_description'] = spec['description']
    selection['spec_specification_url'] = spec['specification_url']
    selection['helper_js'] = config.helper_js
    selection['sanity_checker_js'] = config.sanity_checker_js
    selection['spec_json_js'] = config.spec_json_js

    test_headers_filename = test_filename + ".headers"
    test_directory = os.path.dirname(test_filename)

    test_html_template = util.get_template(test_html_template_basename)
    disclaimer_template = util.get_template('disclaimer.template')

    html_template_filename = os.path.join(util.template_directory,
                                          test_html_template_basename)
    generated_disclaimer = disclaimer_template \
        % {'generating_script_filename': os.path.relpath(sys.argv[0],
                                                         util.test_root_directory),
           'html_template_filename': os.path.relpath(html_template_filename,
                                                     util.test_root_directory)}

    # Adjust the template for the test invoking JS. Indent it to look nice.
    selection['generated_disclaimer'] = generated_disclaimer.rstrip()
    selection[
        'test_description'] = config.test_description_template % selection
    selection['test_description'] = \
        selection['test_description'].rstrip().replace("\n", "\n" + " " * 33)

    # Directory for the test files.
    try:
        os.makedirs(test_directory)
    except:
        pass

    delivery = handle_deliveries(top_source_context.policy_deliveries)

    if len(delivery['headers']) > 0:
        with open(test_headers_filename, "w") as f:
            for header in delivery['headers']:
                f.write('%s: %s\n' % (header, delivery['headers'][header]))

    selection['meta_delivery_method'] = delivery['meta']
    # Obey the lint and pretty format.
    if len(selection['meta_delivery_method']) > 0:
        selection['meta_delivery_method'] = "\n    " + \
                                            selection['meta_delivery_method']

    # Write out the generated HTML file.
    util.write_file(test_filename, test_html_template % selection)
예제 #20
0
def generate_test_file(spec_directory, test_helper_filenames,
                       test_html_template_basename, test_filename, scenarios):
    '''
    Generates a test HTML file (and possibly its associated .headers file)
    from `scenarios`.
    '''

    # Scenarios for the same file should have the same `source_context_list`,
    # including the top-level one.
    # Note: currently, non-top-level source contexts aren't necessarily required
    # to be the same, but we set this requirement as it will be useful e.g. when
    # we e.g. reuse a worker among multiple scenarios.
    for scenario in scenarios:
        assert (scenario['source_context_list'] == scenarios[0]
                ['source_context_list'])

    # We process the top source context below, and do not include it in
    # the JSON objects (i.e. `scenarios`) in generated HTML files.
    top_source_context = scenarios[0]['source_context_list'].pop(0)
    assert (top_source_context.source_context_type == 'top')
    for scenario in scenarios[1:]:
        assert (scenario['source_context_list'].pop(0) == top_source_context)

    parameters = {}

    # Sort scenarios, to avoid unnecessary diffs due to different orders in
    # `scenarios`.
    serialized_scenarios = sorted(
        [dump_test_parameters(scenario) for scenario in scenarios])

    parameters['scenarios'] = ",\n".join(serialized_scenarios).replace(
        "\n", "\n" + " " * 10)

    test_directory = os.path.dirname(test_filename)

    parameters['helper_js'] = ""
    for test_helper_filename in test_helper_filenames:
        parameters['helper_js'] += '    <script src="%s"></script>\n' % (
            os.path.relpath(test_helper_filename, test_directory))
    parameters['sanity_checker_js'] = os.path.relpath(
        os.path.join(spec_directory, 'generic', 'sanity-checker.js'),
        test_directory)
    parameters['spec_json_js'] = os.path.relpath(
        os.path.join(spec_directory, 'generic', 'spec_json.js'),
        test_directory)

    test_headers_filename = test_filename + ".headers"

    test_html_template = util.get_template(test_html_template_basename)
    disclaimer_template = util.get_template('disclaimer.template')

    html_template_filename = os.path.join(util.template_directory,
                                          test_html_template_basename)
    generated_disclaimer = disclaimer_template \
        % {'generating_script_filename': os.path.relpath(sys.argv[0],
                                                         util.test_root_directory),
           'spec_directory': os.path.relpath(spec_directory,
                                             util.test_root_directory)}

    # Adjust the template for the test invoking JS. Indent it to look nice.
    parameters['generated_disclaimer'] = generated_disclaimer.rstrip()

    # Directory for the test files.
    try:
        os.makedirs(test_directory)
    except:
        pass

    delivery = handle_deliveries(top_source_context.policy_deliveries)

    if len(delivery['headers']) > 0:
        with open(test_headers_filename, "w") as f:
            for header in delivery['headers']:
                f.write('%s: %s\n' % (header, delivery['headers'][header]))

    parameters['meta_delivery_method'] = delivery['meta']
    # Obey the lint and pretty format.
    if len(parameters['meta_delivery_method']) > 0:
        parameters['meta_delivery_method'] = "\n    " + \
                                            parameters['meta_delivery_method']

    # Write out the generated HTML file.
    util.write_file(test_filename, test_html_template % parameters)
예제 #21
0
def generate_test_source_files(spec_directory, test_helper_filenames,
                               spec_json, target):
    test_expansion_schema = spec_json['test_expansion_schema']
    specification = spec_json['specification']

    if target == "debug":
        spec_json_js_template = util.get_template('spec_json.js.template')
        util.write_file(
            os.path.join(spec_directory, "generic", "spec_json.js"),
            spec_json_js_template % {'spec_json': json.dumps(spec_json)})
        util.write_file(
            os.path.join(spec_directory, "generic",
                         "debug-output.spec.src.json"),
            json.dumps(spec_json, indent=2, separators=(',', ': ')))

    # Choose a debug/release template depending on the target.
    html_template = "test.%s.html.template" % target

    artifact_order = test_expansion_schema.keys()
    artifact_order.remove('expansion')

    excluded_selection_pattern = ''
    for key in artifact_order:
        excluded_selection_pattern += '%(' + key + ')s/'

    # Create list of excluded tests.
    exclusion_dict = set()
    for excluded_pattern in spec_json['excluded_tests']:
        excluded_expansion = \
            expand_pattern(excluded_pattern, test_expansion_schema)
        for excluded_selection in permute_expansion(excluded_expansion,
                                                    artifact_order):
            excluded_selection['delivery_key'] = spec_json['delivery_key']
            exclusion_dict.add(excluded_selection_pattern % excluded_selection)

    # `scenarios[filename]` represents the list of scenario objects to be
    # generated into `filename`.
    scenarios = {}

    for spec in specification:
        # Used to make entries with expansion="override" override preceding
        # entries with the same |selection_path|.
        output_dict = {}

        for expansion_pattern in spec['test_expansion']:
            expansion = expand_pattern(expansion_pattern,
                                       test_expansion_schema)
            for selection in permute_expansion(expansion, artifact_order):
                selection['delivery_key'] = spec_json['delivery_key']
                selection_path = spec_json['selection_pattern'] % selection
                if selection_path in output_dict:
                    if expansion_pattern['expansion'] != 'override':
                        print("Error: expansion is default in:")
                        print(dump_test_parameters(selection))
                        print("but overrides:")
                        print(dump_test_parameters(
                            output_dict[selection_path]))
                        sys.exit(1)
                output_dict[selection_path] = copy.deepcopy(selection)

        for selection_path in output_dict:
            selection = output_dict[selection_path]
            if (excluded_selection_pattern % selection) in exclusion_dict:
                print('Excluding selection:', selection_path)
                continue
            try:
                test_filename = get_test_filename(spec_directory, spec_json,
                                                  selection)
                scenario = generate_selection(spec_json, selection)
                scenarios[test_filename] = scenarios.get(test_filename,
                                                         []) + [scenario]
            except util.ShouldSkip:
                continue

    for filename in scenarios:
        generate_test_file(spec_directory, test_helper_filenames,
                           html_template, filename, scenarios[filename])
예제 #22
0
def submit_issue_response(repo_username, repo_id, number, action_type, message_type, custom_message):
    data = {
        'repo_username': repo_username,
        'repo_id': repo_id,
        'number': number,
        'action_type': action_type,
        'message_type': message_type,
        'custom_message': custom_message
    }

    try:
        issue = github_api.fetch_issue(repo_username, repo_id, number)
        if not issue or issue.get('error'):
            data['error'] = 'could not find issue %s' % number
            return data

        context = {
            'issue': issue,
            'user': issue.get('user')
        }
        msg = None

        if message_type == 'expire':
            msg = util.get_template('EXPIRE_TEMPLATE', context)

        elif message_type == 'forum':
            msg = util.get_template('FORUM_TEMPLATE', context)

        elif message_type == 'inapplicable':
            msg = util.get_template('INAPPLICABLE_TEMPLATE', context)

        elif message_type == 'more':
            msg = util.get_template('MORE_TEMPLATE', context)

        elif message_type == 'feature':
            github_api.add_issue_labels(repo_username, repo_id, number, [cvar['FEATURE_REQUEST_LABEL']], issue=issue)
            msg = util.get_template('FEATURE_REQUEST_TEMPLATE', context)

        elif message_type == 'no_reply':
            msg = util.get_template('CLOSING_NOREPLY_TEMPLATE', context)

        elif message_type == 'pr_close':
            msg = util.get_template('CLOSE_PULL_REQUEST_TEMPLATE', context)

        elif message_type == 'custom':
            msg = custom_message

        elif message_type == 'close':
            msg = None

        else:
            data['error'] = 'invalid message_type: %s' % message_type
            return data

        if msg and len(msg.strip()):
            data['created_comment'] = github_api.create_issue_comment(repo_username, repo_id, number, msg)

        if action_type == 'close':
            data['issue_closed'] = github_api.close_issue(repo_username, repo_id, number, issue)

        elif action_type == 'reply':
            github_api.add_issue_labels(repo_username, repo_id, number, [cvar['NEEDS_REPLY_LABEL']], issue=issue)

    except Exception as ex:
        print 'submit_issue_response error, %s: %s' % (number, ex)
        data['error'] = '%s' % ex

    return data
예제 #23
0
            continue

        market = get_correct_market(s, rf.year)
        ann_market_return = market.get_annualised_return(rf.year)

        req_rate = CAPM(rf.spot_yield, ann_market_return, s.beta)
        try:
            greeks = get_greeks(s, market, rf)

            gg_result = gordon_growth_range(s, req_rate)
            fcf_result = fcf_growth_range(s, req_rate)
            forward_pe_result = forward_pe_range(s, req_rate)

            report_name = f"{util.today_date()}.html"

            template = util.get_template("report_template.html")
            template_vars = {"spot_yield": util.convert_to_pct(rf.spot_yield), "market_ann_return": util.convert_to_pct( market.get_annualised_return(rf.year)), "benchmark_year": rf.year, "req_rate": util.convert_to_pct(req_rate),
                            "greeks": greeks, "gordon_growth": gg_result, "fcf_growth": fcf_result, "forward_pe": forward_pe_result,
                            "timestamp": util.today_date()}

            html_out = template.render(template_vars, s = s)
            logging.debug("Generated report file")

            util.dump_report(html_out, f"/tmp/{report_name}")
            logging.debug("Created report file")

            util.upload_report_to_s3("/tmp/" + report_name, s3_bucket, s3_report_dir + "/" + s.stock_code + "/" +report_name)

            db.update_item(Key={"ticker": s.stock_code}, UpdateExpression = f"set price = :price", 
                ExpressionAttributeValues = {":price": Decimal(str(s.get_current_price()))})
            util.update_valuation(db, s.stock_code, util.Valuation.GORDON_GROWTH.value, util.extract_valuation(gg_result))
예제 #24
0
def build(
    from_,
    name,
    apt,
    pip3,
    with_azurecli,
    with_kubectl,
    with_velero,
    force,
    dry_run,
    output,
):
    """
    Build Docker images with custom packages.
    \n
    Examples:
    \n
    Build an image and install vim and curl using apt-get.
    \n
    $ dugaire build --apt=vim,curl
    \n
    Build an image and install python3 using apt-get and ansible using pip3.
    \n
    $ dugaire build --apt=python3-pip --pip3=ansible
    \n
    Build an image and install the latest version of kubectl.
    \n
    $ dugaire build --with-kubectl=latest
    \n

    """

    dockerfile = ""

    template = util.get_template("base.j2")
    dockerfile += template.render(
        from_=from_, label=util.get_dugaire_image_label("dockerfile"))

    if apt:
        packages = apt.replace(",", " ")
        template = util.get_template("apt.j2")
        dockerfile += template.render(packages=packages)

    if pip3:
        dependency_list = {}
        dependency_list["azure-cli"] = ["gcc", "python3-dev"]

        pip3_install = pip3.split(",")
        for package in pip3_install:
            package_name = package.split("==")[0]
            if package_name in dependency_list:
                dependency = " ".join(dependency_list[package_name])
                apt_template = util.get_template("apt.j2")
                dockerfile += apt_template.render(packages=dependency)

        packages = " ".join(pip3_install)
        template = util.get_template("pip3.j2")
        dockerfile += template.render(packages=packages)

    if with_kubectl:
        current_option_name = "--with-kubectl"
        current_option_value = with_kubectl

        if not util.string_is_latest_or_version(current_option_value):
            usage_msg = f"{current_option_name}=<latest | semantic versioning>"
            example_msg = f"{current_option_name}=latest | {current_option_name}=1.17.0"

            exc_msg = f"Bad usage {current_option_name}={current_option_value} \n"
            exc_msg += f"Valid usage: {usage_msg} \n"
            exc_msg += f"Examples: {example_msg}"
            raise click.BadOptionUsage(current_option_name, exc_msg)

        dependency_list = {}
        dependency_list = ["curl", "ca-certificates"]
        dependency = " ".join(dependency_list)

        apt_template = util.get_template("apt.j2")
        dockerfile += apt_template.render(packages=dependency)

        url = "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl"
        if with_kubectl != "latest":
            url = f"https://storage.googleapis.com/kubernetes-release/release/v{with_kubectl}/bin/linux/amd64/kubectl"

        template = util.get_template("with_kubectl.j2")
        dockerfile += template.render(url=url)

    if with_azurecli:
        dependency_list = {}
        dependency_list = ["curl", "ca-certificates"]
        dependency = " ".join(dependency_list)

        apt_template = util.get_template("apt.j2")
        dockerfile += apt_template.render(packages=dependency)

        template = util.get_template("with_azurecli.j2")
        dockerfile += template.render()

    if with_velero:

        current_option_name = "--with-velero"
        current_option_value = with_velero

        if not util.string_is_latest_or_version(current_option_value):
            usage_msg = f"{current_option_name}=<latest | semantic versioning>"
            example_msg = f"{current_option_name}=latest | {current_option_name}=1.5.2"

            exc_msg = f"Bad usage {current_option_name}={current_option_value} \n"
            exc_msg += f"Valid usage: {usage_msg} \n"
            exc_msg += f"Examples: {example_msg}"
            raise click.BadOptionUsage(current_option_name, exc_msg)

        if not with_kubectl:
            usage_msg = f"--with-kubectl=<latest | semantic versioning> {current_option_name}=<latest | semantic versioning>"
            example_msg = f"--with-kubectl=latest {current_option_name}=latest"

            exc_msg = f"Bad usage {current_option_name} requires --with-kubectl \n"
            exc_msg += f"Valid usage: {usage_msg} \n"
            exc_msg += f"Examples: {example_msg}"
            raise click.BadOptionUsage(current_option_name, exc_msg)

        dependency_list = {}
        dependency_list = ["wget"]
        dependency = " ".join(dependency_list)

        apt_template = util.get_template("apt.j2")
        dockerfile += apt_template.render(packages=dependency)

        if with_velero == "latest":
            import urllib.request

            response = urllib.request.urlopen(
                "https://api.github.com/repos/vmware-tanzu/velero/releases/latest"
            ).read()
            response = json.loads(response)
            with_velero = response["tag_name"][1:]

        template = util.get_template("with_velero.j2")
        dockerfile += template.render(version=with_velero)

    image_id = None
    image_name = None
    if not dry_run:
        f = BytesIO(dockerfile.encode("utf-8"))
        client = docker.from_env()
        image_name = name
        if image_name == "random":
            random_name = str(uuid.uuid4())[:8]
            random_tag = str(uuid.uuid4())[:8]
            image_name = f"dug-{random_name}:{random_tag}"

        image, error = client.images.build(fileobj=f,
                                           tag=image_name,
                                           nocache=force,
                                           rm=True,
                                           forcerm=True)

        image_id = image.attrs["Id"]
        image_name = image.attrs["RepoTags"][0]

    if output == "image.id":
        click.echo(image_id)
    if output == "image.id.short":
        click.echo(image_id.replace("sha256:", "")[:12])
    if output == "image.name":
        click.echo(image_name)
    if output == "dockerfile":
        click.echo(dockerfile)