def main(): try: start_moment = time.monotonic() try: cli_args = parse_cli_arguments(sys.argv[1:]) except CliError: sys.exit(1) argument_file = cli_args.get('argument_file') if argument_file: argument_file_string = read_lines_from_commented_json_file( argument_file) try: argument_file_dict = load_json_argument_file( argument_file_string) arguments = parse_argument_file_content( argument_file_dict, cli_args) except UserError as e: raise UserError( f"Error parsing argument file '{argument_file}': " f"{type(e).__name__}: {e}") else: page_variables = {"VARIABLES": {"only-at-page-start": True}} if cli_args["legacy_mode"]: page_variables["METADATA"] = {"only-at-page-start": True} argument_file_dict = { "documents": [{}], # When run without argument file, need implicitly added # plugin for page title extraction from the source text. "plugins": { "page-variables": page_variables } } arguments = parse_argument_file_content(argument_file_dict, cli_args) metadata_handlers = register_page_metadata_handlers(arguments.plugins) for document in arguments.documents: try: md2html(document, arguments.plugins, metadata_handlers, arguments.options) except UserError as e: error_input_file = document["input_file"] raise UserError( f"Error processing input file '{error_input_file}': " f"{type(e).__name__}: {e}") if arguments.options["verbose"]: end_moment = time.monotonic() print('Finished in: ' + str(timedelta(seconds=end_moment - start_moment))) except UserError as ue: print(str(ue)) sys.exit(1)
def accept_page_metadata(self, doc: dict, marker: str, metadata_str: str, metadata_section): try: metadata = json.loads(metadata_str) validate(instance=metadata, schema=self.metadata_schema) except JSONDecodeError as e: raise UserError( f"Incorrect JSON in page metadata: {type(e).__name__}: {str(e)}" ) except ValidationError as e: raise UserError( f"Error validating page metadata: {type(e).__name__}: " + reduce_json_validation_error_message(str(e))) self.page_variables.update(metadata) return ''
def wait_spot_requests_fullfilled( env, spot_req_id ): """ Wait until all spot requests are fulfilled. :param env: The AWS environment to work on, i.e., a Service object :rtype: dictionary containing the status of all spot instances requests """ count_tries = 0 log.warning("Waiting for spot request to be fullfilled...") while count_tries < 10: response = env.ec2client.describe_spot_instance_requests( Filters=[{'Name':'spot-instance-request-id','Values':[spot_req_id]}] ) resp = response["SpotInstanceRequests"] for r in resp: if r['State'] == 'failed': msg = "Spot request failed due to %s:\n%s" % (r['Status']['Code'], r['Status']['Message']) raise UserError(message=msg) if r['State'] == 'open': log.warning("...request pending: %s", r['Status']['Code']) time.sleep(5) count_tries += 1 continue if r['State'] == 'active' and r['Status']['Code'] == 'fulfilled': log.info("Request %s!", r['Status']['Code']) log.info("%s!", r['Status']['Message']) return r['InstanceId']
def terminate_instance(env, instance_id=None): ''' Terminates the instance specified with the specified ID. If no ID is given, stops all instances without termination, and prints a warning message. :return: the id of the terminated instance or the ids of the stopped instances ''' if not env: raise UserError("Cannot start any AWS service without an AWS object") if not instance_id: log.info("No instance ID provided: stopping all instances. " "It will be possible to restart them.") instances = env.ec2.instance.all() stopped = [] for inst in instances: resp = inst.stop() log.info("Stopping instance %s" % resp['StoppingInstances'][0]['InstanceId']) stopped.append(resp['StoppingInstances'][0]['InstanceId']) return stopped else: log.info("Terminating instance %s" % instance_id) resp = env.ec2.Instance(instance_id).terminate() return resp['StoppingInstances'][0]['InstanceId']
def load_json_argument_file(argument_file_string) -> dict: try: arguments_item = json.loads(argument_file_string) except JSONDecodeError as e: raise UserError( f"Error loading JSON argument file: {type(e).__name__}: {e}") try: schema = json.loads( read_lines_from_commented_json_file( MODULE_DIR.joinpath('args_file_schema.json'))) validate(instance=arguments_item, schema=schema) except ValidationError as e: raise UserError( f"Error validating argument file content: {type(e).__name__}: " + reduce_json_validation_error_message(str(e))) return arguments_item
def validate_data(data, schema_file): with open(schema_file, 'r') as schema_file: schema = json.load(schema_file) try: validate(instance=data, schema=schema) except ValidationError as e: raise UserError(f"Error validating plugin data: {type(e).__name__}: " + reduce_json_validation_error_message(str(e)))
def __resolveMe(self): try: # create a new session with available credentials self.__session = boto3.session.Session(region_name=self.region) except Exception as _: raise UserError( "Can't determine current IAM user name. Be sure to put valid AWS " "credentials in environment variables or in ~/.aws/credentials. " "For details, refer to %s." % 'http://boto3.readthedocs.io/en/latest/guide/configuration.html' )
def __assert_state( self, expected_state ): """ Raises a UserError if the instance is not in the expected state. :param expected_state: the expected state :return: the instance """ actual_state = self.instance.state if actual_state != expected_state: raise UserError( "Expected instance state '%s' but got '%s'" % (expected_state, actual_state) )
def __init__( self, env=None ): """ :param env: a Service object, needed to access all EC2 resources available to the current user. """ self.env = env # The actual session environment. This is the object that encapsulates # all the settings used by AWS instances. Further, it permits to manage # those resources. self.release_info = None # Linux distro's release to use. This affects the image_id to be used self.image_id = None # The image the instance was or will be booted from self.__instance = None # The instance represented by this engine self.generation = None # The number of previous generations of this engine. When an instance # is booted from a stock AMI, generation is 0. After that instance is # set up and imaged and another instance is booted from the resulting # AMI, generation will be 1. self.cluster_ordinal = None # The ordinal of this engine within a cluster of boxes. For boxes that # don't join a cluster, this will be 0 self.cluster_name = None # The name of the cluster this engine is a node of, or None if this # engine is not in a cluster. self.placement_group = None # The placement group where this engine is placed, if in a cluster self.role_options = { } # Role-specifc options for this engine self.key_in_use = None # path to the SSH used to create and to SSH to the instance if self.env is None: raise UserError( "A Service is required before creating any engine instance. " "In order to create the Service object, be sure to put valid AWS " "credentials in environment variables or in ~/.aws/credentials. " "For details, refer to %s." % 'http://boto3.readthedocs.io/en/latest/guide/configuration.html' ) else: assert isinstance(self.env, Service)
def create_ec2_spot_instances(spot_price, env, imageId=defaultImage, count=1, secGroup=None, instType=defaultType, keyName=None, Placement=None, subnet=None, usr_data=None, **other_opts): """ Requests boto3 API to create EC2 spotinstance(s) :rtype: list(ec2.Instance) """ if env is None: raise UserError("Cannot start any AWS service without an AWS object") if not keyName: keyName = env.get_key_pair() else: keyName = env.get_key_pair(keyName) ebsopt = True if ec2_instance_types[instType].EBS_optimized else False spot_response = env.ec2client.request_spot_instances( SpotPrice = str(spot_price), InstanceCount = count, LaunchSpecification = { 'ImageId': imageId, 'KeyName': keyName[0], 'InstanceType': instType, 'SecurityGroupIds': secGroup, 'Placement': { 'AvailabilityZone': Placement.AvailabilityZone, 'GroupName' : Placement.GroupName }, 'BlockDeviceMappings': [{ 'DeviceName': '/dev/sdg', 'Ebs': { 'VolumeSize': 12, 'DeleteOnTermination': True, 'VolumeType': 'gp2', # 'Iops': 300, 'Encrypted': False } }], 'EbsOptimized': ebsopt, 'Monitoring': { 'Enabled': False }, 'SubnetId' : subnet, 'UserData' : usr_data }) return spot_response
def create_ec2_instances(env, imageId=defaultImage, count=1, instType=defaultType, secGroup=None, keyName=None, Placement=None, subnet=None, usr_data=None, **other_opts): """ Requests boto3 API to create EC2 on_demand instance(s). Default instance type is 'm3.medium', equipped with 1 core, 3.75GB of RAM, 1 SSD ephemeral disk with 4GB. Default image is a Ubuntu16 @ eu-west-1 :param env: a 'Service' object, that encapsulates AWS-specific services and provides access to user settings. :rtype: public ips of created instances """ if env is None: raise UserError("Cannot start any AWS service without an AWS object") log.info("Creating %s instance(s) using the key '%s'", str(count), keyName) instances = env.ec2.create_instances( ImageId=imageId, MinCount=1, MaxCount=count, InstanceType=instType, KeyName=keyName, SecurityGroupIds=secGroup, UserData=usr_data, #open("/shelf/fabio/lilWS/cloud_provision/init_scripts/sample_script.sh").read(), **other_opts ) #thread = threading.Thread(target=wait_running, args=(instances)) #thread.start() log.info("Instance(s) created. " "Waiting for instances to be in running state...") for inst in instances: inst.wait_until_running() inst.load() log.info("[%s] Instance %s is running.", inst.public_ip_address, inst.id) #thread.join() # pub_ips = [] # for inst in instances: # inst.load() # pub_ips.append(inst.public_ip_address) return instances
def parse_argument_file_content(argument_file_dict: dict, cli_args: dict) -> Arguments: documents = [] plugins = [] options = argument_file_dict.setdefault('options', {}) plugins_item = argument_file_dict.setdefault('plugins', {}) page_flows_plugin_item = plugins_item.get("page-flows") documents_page_flows = {} if bool(options.get('verbose')) and bool(cli_args.get("report")): raise UserError( "'verbose' parameter in 'options' section is incompatible " "with '--report' command line argument.") attr = 'verbose' options[attr] = first_not_none(options.get(attr), cli_args.get(attr), False) options['legacy_mode'] = first_not_none(cli_args.get('legacy_mode'), options.get('legacy-mode'), False) # plugins_item = argument_file_dict.setdefault('plugins', {}) if options['legacy_mode']: page_variables = plugins_item.setdefault("page-variables", {}) if "METADATA" not in page_variables: page_variables["METADATA"] = {"only-at-page-start": True} defaults_item = argument_file_dict.get('default') if defaults_item is None: defaults_item = {} documents_item = argument_file_dict['documents'] if 'no-css' in defaults_item and ('link-css' in defaults_item or 'include-css' in defaults_item): raise UserError( f"'no-css' parameter incompatible with one of the ['link-css', " f"'include-css'] in the 'default' section.") for document_item in documents_item: document = {} v = first_not_none(cli_args.get('input_file'), document_item.get("input"), defaults_item.get("input")) if v is not None: document['input_file'] = v else: raise UserError( f"Undefined input file for 'documents' item: {document_item}.") v = first_not_none(cli_args.get('output_file'), document_item.get("output"), defaults_item.get("output")) document['output_file'] = v attr = 'title' document[attr] = first_not_none(cli_args.get(attr), document_item.get(attr), defaults_item.get(attr), '') attr = 'template' v = first_not_none(cli_args.get(attr), document_item.get(attr), defaults_item.get(attr)) document[attr] = Path(v) if v is not None else None link_css = [] include_css = [] no_css = False if cli_args.get('no_css') or cli_args.get('link_css') or cli_args.get( 'include_css'): if cli_args.get('no_css'): no_css = True else: link_css.extend(first_not_none(cli_args.get('link_css'), [])) include_css.extend( first_not_none(cli_args.get('include_css'), [])) else: link_args = [ "link-css", "add-link-css", "include-css", "add-include-css" ] if 'no-css' in document_item and any( document_item.get(k) for k in link_args): q = '\'' raise UserError( f"'no-css' parameter incompatible with one of " f"[{', '.join([q + a + q for a in link_args])}] " f"in `documents` item: {document_item}.") no_css = first_not_none(document_item.get('no-css'), defaults_item.get('no-css'), False) link_css.extend( first_not_none(document_item.get('link-css'), defaults_item.get('link-css'), [])) link_css.extend( first_not_none(document_item.get('add-link-css'), [])) include_css.extend( first_not_none(document_item.get('include-css'), defaults_item.get('include-css'), [])) include_css.extend( first_not_none(document_item.get('add-include-css'), [])) if link_css or include_css: no_css = False document['link_css'] = link_css document['include_css'] = include_css document['no_css'] = no_css attr = 'force' document[attr] = first_not_none(True if cli_args.get(attr) else None, document_item.get(attr), defaults_item.get(attr), False) attr = 'verbose' document[attr] = first_not_none(True if cli_args.get(attr) else None, document_item.get(attr), defaults_item.get(attr), False) attr = 'report' document[attr] = first_not_none(True if cli_args.get(attr) else None, document_item.get(attr), defaults_item.get(attr), False) if document['report'] and document['verbose']: raise UserError( f"Incompatible 'report' and 'verbose' parameters for 'documents' " f"item: {document_item}.") enrich_document(document) # Even if all page flows are defined in the 'documents' section, at least empty # 'page-flows' plugin must be defined in order to activate page flows processing. if page_flows_plugin_item is not None: if ((1 if 'no-page-flows' in document_item else 0) + (1 if 'page-flows' in document_item else 0) + (1 if 'add-page-flows' in document_item else 0) > 1): raise UserError( f"Incompatible 'no-page-flows', 'page-flows' and 'add-page-flows' " f"parameters for 'documents' item: {document_item}.") attr = 'page-flows' page_flows = first_not_none( [] if document_item.get('no-page-flows') else document_item.get(attr), defaults_item.get(attr), []) for page_flow in page_flows: page_flow_list = documents_page_flows.setdefault(page_flow, []) page_flow_list.append({ "link": document["output_file"], "title": document["title"] }) add_page_flows = first_not_none( document_item.get("add-page-flows"), []) for page_flow in add_page_flows: page_flow_list = documents_page_flows.setdefault(page_flow, []) page_flow_list.append({ "link": document["output_file"], "title": document["title"] }) documents.append(document) add_documents_page_flows_data(page_flows_plugin_item, documents_page_flows) for k, v in plugins_item.items(): plugin = PLUGINS.get(k) if plugin: try: if plugin.accept_data(v): plugins.append(plugin) except UserError as e: raise UserError( f"Error initializing plugin '{k}': {type(e).__name__}: {e}" ) return Arguments(options, documents, plugins)
def md2html(document, plugins, metadata_handlers, options): input_location = document['input_file'] output_location = document['output_file'] title = document['title'] template_file = document['template'] link_css = document['link_css'] include_css = document['include_css'] force = document['force'] verbose = document['verbose'] report = document['report'] output_file = Path(output_location) input_file = Path(input_location) if not force and output_file.exists(): output_file_mtime = os.path.getmtime(output_file) input_file_mtime = os.path.getmtime(input_file) if output_file_mtime > input_file_mtime: if verbose: print( f'The output file is up-to-date. Skipping: {output_location}' ) return current_time = datetime.today() substitutions = { 'title': title, 'exec_name': EXEC_NAME, 'exec_version': EXEC_VERSION, 'generation_date': current_time.strftime('%Y-%m-%d'), 'generation_time': current_time.strftime('%H:%M:%S') } styles = [] if link_css: styles.extend([ f'<link rel="stylesheet" type="text/css" href="{item}">' for item in link_css ]) if include_css: styles.extend([ '<style>\n' + read_lines_from_file(item) + '\n</style>' for item in include_css ]) substitutions['styles'] = '\n'.join(styles) if styles else '' md_lines = read_lines_from_file(input_file) for plugin in plugins: plugin.new_page() md_lines = apply_metadata_handlers(md_lines, metadata_handlers, document) substitutions['content'] = MARKDOWN.convert(source=md_lines) for plugin in plugins: substitutions.update(plugin.variables(document)) if options['legacy_mode']: placeholders = substitutions.get('placeholders') if placeholders is not None: del substitutions['placeholders'] substitutions.update(placeholders) template = read_lines_from_cached_file_legacy(template_file) else: template = read_lines_from_cached_file(template_file) if substitutions['title'] is None: substitutions['title'] = '' try: result = chevron.render(template, substitutions) except chevron.ChevronError as e: raise UserError(f"Error processing template: {type(e).__name__}: {e}") with open(output_file, 'w') as result_file: result_file.write(result) if verbose: print(f'Output file generated: {output_location}') if report: print(output_location)