def main():
try:
start_moment = time.monotonic()
try:
cli_args = parse_cli_arguments(sys.argv[1:])
except CliError:
sys.exit(1)
argument_file = cli_args.get('argument_file')
if argument_file:
argument_file_string = read_lines_from_commented_json_file(
argument_file)
try:
argument_file_dict = load_json_argument_file(
argument_file_string)
arguments = parse_argument_file_content(
argument_file_dict, cli_args)
except UserError as e:
raise UserError(
f"Error parsing argument file '{argument_file}': "
f"{type(e).__name__}: {e}")
else:
page_variables = {"VARIABLES": {"only-at-page-start": True}}
if cli_args["legacy_mode"]:
page_variables["METADATA"] = {"only-at-page-start": True}
argument_file_dict = {
"documents": [{}],
# When run without argument file, need implicitly added
# plugin for page title extraction from the source text.
"plugins": {
"page-variables": page_variables
}
}
arguments = parse_argument_file_content(argument_file_dict,
cli_args)
metadata_handlers = register_page_metadata_handlers(arguments.plugins)
for document in arguments.documents:
try:
md2html(document, arguments.plugins, metadata_handlers,
arguments.options)
except UserError as e:
error_input_file = document["input_file"]
raise UserError(
f"Error processing input file '{error_input_file}': "
f"{type(e).__name__}: {e}")
if arguments.options["verbose"]:
end_moment = time.monotonic()
print('Finished in: ' +
str(timedelta(seconds=end_moment - start_moment)))
except UserError as ue:
print(str(ue))
sys.exit(1)
def accept_page_metadata(self, doc: dict, marker: str, metadata_str: str,
metadata_section):
try:
metadata = json.loads(metadata_str)
validate(instance=metadata, schema=self.metadata_schema)
except JSONDecodeError as e:
raise UserError(
f"Incorrect JSON in page metadata: {type(e).__name__}: {str(e)}"
)
except ValidationError as e:
raise UserError(
f"Error validating page metadata: {type(e).__name__}: " +
reduce_json_validation_error_message(str(e)))
self.page_variables.update(metadata)
return ''
def wait_spot_requests_fullfilled( env, spot_req_id ):
"""
Wait until all spot requests are fulfilled.
:param env: The AWS environment to work on, i.e., a Service object
:rtype: dictionary containing the status of all spot instances requests
"""
count_tries = 0
log.warning("Waiting for spot request to be fullfilled...")
while count_tries < 10:
response = env.ec2client.describe_spot_instance_requests(
Filters=[{'Name':'spot-instance-request-id','Values':[spot_req_id]}]
)
resp = response["SpotInstanceRequests"]
for r in resp:
if r['State'] == 'failed':
msg = "Spot request failed due to %s:\n%s" % (r['Status']['Code'], r['Status']['Message'])
raise UserError(message=msg)
if r['State'] == 'open':
log.warning("...request pending: %s", r['Status']['Code'])
time.sleep(5)
count_tries += 1
continue
if r['State'] == 'active' and r['Status']['Code'] == 'fulfilled':
log.info("Request %s!", r['Status']['Code'])
log.info("%s!", r['Status']['Message'])
return r['InstanceId']
def terminate_instance(env, instance_id=None):
'''
Terminates the instance specified with the specified ID.
If no ID is given, stops all instances without termination, and
prints a warning message.
:return: the id of the terminated instance or the ids of the stopped
instances
'''
if not env:
raise UserError("Cannot start any AWS service without an AWS object")
if not instance_id:
log.info("No instance ID provided: stopping all instances. "
"It will be possible to restart them.")
instances = env.ec2.instance.all()
stopped = []
for inst in instances:
resp = inst.stop()
log.info("Stopping instance %s" % resp['StoppingInstances'][0]['InstanceId'])
stopped.append(resp['StoppingInstances'][0]['InstanceId'])
return stopped
else:
log.info("Terminating instance %s" % instance_id)
resp = env.ec2.Instance(instance_id).terminate()
return resp['StoppingInstances'][0]['InstanceId']
def load_json_argument_file(argument_file_string) -> dict:
try:
arguments_item = json.loads(argument_file_string)
except JSONDecodeError as e:
raise UserError(
f"Error loading JSON argument file: {type(e).__name__}: {e}")
try:
schema = json.loads(
read_lines_from_commented_json_file(
MODULE_DIR.joinpath('args_file_schema.json')))
validate(instance=arguments_item, schema=schema)
except ValidationError as e:
raise UserError(
f"Error validating argument file content: {type(e).__name__}: " +
reduce_json_validation_error_message(str(e)))
return arguments_item
def validate_data(data, schema_file):
with open(schema_file, 'r') as schema_file:
schema = json.load(schema_file)
try:
validate(instance=data, schema=schema)
except ValidationError as e:
raise UserError(f"Error validating plugin data: {type(e).__name__}: " +
reduce_json_validation_error_message(str(e)))
def __resolveMe(self):
try:
# create a new session with available credentials
self.__session = boto3.session.Session(region_name=self.region)
except Exception as _:
raise UserError(
"Can't determine current IAM user name. Be sure to put valid AWS "
"credentials in environment variables or in ~/.aws/credentials. "
"For details, refer to %s." %
'http://boto3.readthedocs.io/en/latest/guide/configuration.html'
)
def __assert_state( self, expected_state ):
"""
Raises a UserError if the instance is not in the expected state.
:param expected_state: the expected state
:return: the instance
"""
actual_state = self.instance.state
if actual_state != expected_state:
raise UserError( "Expected instance state '%s' but got '%s'"
% (expected_state, actual_state) )
def __init__( self, env=None ):
"""
:param env: a Service object, needed to access all EC2 resources
available to the current user.
"""
self.env = env
# The actual session environment. This is the object that encapsulates
# all the settings used by AWS instances. Further, it permits to manage
# those resources.
self.release_info = None
# Linux distro's release to use. This affects the image_id to be used
self.image_id = None
# The image the instance was or will be booted from
self.__instance = None
# The instance represented by this engine
self.generation = None
# The number of previous generations of this engine. When an instance
# is booted from a stock AMI, generation is 0. After that instance is
# set up and imaged and another instance is booted from the resulting
# AMI, generation will be 1.
self.cluster_ordinal = None
# The ordinal of this engine within a cluster of boxes. For boxes that
# don't join a cluster, this will be 0
self.cluster_name = None
# The name of the cluster this engine is a node of, or None if this
# engine is not in a cluster.
self.placement_group = None
# The placement group where this engine is placed, if in a cluster
self.role_options = { }
# Role-specifc options for this engine
self.key_in_use = None
# path to the SSH used to create and to SSH to the instance
if self.env is None:
raise UserError( "A Service is required before creating any engine instance. "
"In order to create the Service object, be sure to put valid AWS "
"credentials in environment variables or in ~/.aws/credentials. "
"For details, refer to %s."
% 'http://boto3.readthedocs.io/en/latest/guide/configuration.html' )
else:
assert isinstance(self.env, Service)
def create_ec2_spot_instances(spot_price, env, imageId=defaultImage, count=1, secGroup=None,
instType=defaultType, keyName=None, Placement=None,
subnet=None, usr_data=None, **other_opts):
"""
Requests boto3 API to create EC2 spotinstance(s)
:rtype: list(ec2.Instance)
"""
if env is None:
raise UserError("Cannot start any AWS service without an AWS object")
if not keyName:
keyName = env.get_key_pair()
else:
keyName = env.get_key_pair(keyName)
ebsopt = True if ec2_instance_types[instType].EBS_optimized else False
spot_response = env.ec2client.request_spot_instances(
SpotPrice = str(spot_price),
InstanceCount = count,
LaunchSpecification = {
'ImageId': imageId,
'KeyName': keyName[0],
'InstanceType': instType,
'SecurityGroupIds': secGroup,
'Placement': {
'AvailabilityZone': Placement.AvailabilityZone,
'GroupName' : Placement.GroupName
},
'BlockDeviceMappings': [{
'DeviceName': '/dev/sdg',
'Ebs': {
'VolumeSize': 12,
'DeleteOnTermination': True,
'VolumeType': 'gp2',
# 'Iops': 300,
'Encrypted': False
}
}],
'EbsOptimized': ebsopt,
'Monitoring': {
'Enabled': False
},
'SubnetId' : subnet,
'UserData' : usr_data
})
return spot_response
def create_ec2_instances(env, imageId=defaultImage, count=1, instType=defaultType,
secGroup=None, keyName=None, Placement=None,
subnet=None, usr_data=None, **other_opts):
"""
Requests boto3 API to create EC2 on_demand instance(s).
Default instance type is 'm3.medium', equipped with 1 core, 3.75GB of RAM,
1 SSD ephemeral disk with 4GB.
Default image is a Ubuntu16 @ eu-west-1
:param env: a 'Service' object, that encapsulates AWS-specific services and
provides access to user settings.
:rtype: public ips of created instances
"""
if env is None:
raise UserError("Cannot start any AWS service without an AWS object")
log.info("Creating %s instance(s) using the key '%s'", str(count), keyName)
instances = env.ec2.create_instances(
ImageId=imageId,
MinCount=1,
MaxCount=count,
InstanceType=instType,
KeyName=keyName,
SecurityGroupIds=secGroup,
UserData=usr_data, #open("/shelf/fabio/lilWS/cloud_provision/init_scripts/sample_script.sh").read(),
**other_opts
)
#thread = threading.Thread(target=wait_running, args=(instances))
#thread.start()
log.info("Instance(s) created. "
"Waiting for instances to be in running state...")
for inst in instances:
inst.wait_until_running()
inst.load()
log.info("[%s] Instance %s is running.", inst.public_ip_address, inst.id)
#thread.join()
# pub_ips = []
# for inst in instances:
# inst.load()
# pub_ips.append(inst.public_ip_address)
return instances
def parse_argument_file_content(argument_file_dict: dict,
cli_args: dict) -> Arguments:
documents = []
plugins = []
options = argument_file_dict.setdefault('options', {})
plugins_item = argument_file_dict.setdefault('plugins', {})
page_flows_plugin_item = plugins_item.get("page-flows")
documents_page_flows = {}
if bool(options.get('verbose')) and bool(cli_args.get("report")):
raise UserError(
"'verbose' parameter in 'options' section is incompatible "
"with '--report' command line argument.")
attr = 'verbose'
options[attr] = first_not_none(options.get(attr), cli_args.get(attr),
False)
options['legacy_mode'] = first_not_none(cli_args.get('legacy_mode'),
options.get('legacy-mode'), False)
# plugins_item = argument_file_dict.setdefault('plugins', {})
if options['legacy_mode']:
page_variables = plugins_item.setdefault("page-variables", {})
if "METADATA" not in page_variables:
page_variables["METADATA"] = {"only-at-page-start": True}
defaults_item = argument_file_dict.get('default')
if defaults_item is None:
defaults_item = {}
documents_item = argument_file_dict['documents']
if 'no-css' in defaults_item and ('link-css' in defaults_item
or 'include-css' in defaults_item):
raise UserError(
f"'no-css' parameter incompatible with one of the ['link-css', "
f"'include-css'] in the 'default' section.")
for document_item in documents_item:
document = {}
v = first_not_none(cli_args.get('input_file'),
document_item.get("input"),
defaults_item.get("input"))
if v is not None:
document['input_file'] = v
else:
raise UserError(
f"Undefined input file for 'documents' item: {document_item}.")
v = first_not_none(cli_args.get('output_file'),
document_item.get("output"),
defaults_item.get("output"))
document['output_file'] = v
attr = 'title'
document[attr] = first_not_none(cli_args.get(attr),
document_item.get(attr),
defaults_item.get(attr), '')
attr = 'template'
v = first_not_none(cli_args.get(attr), document_item.get(attr),
defaults_item.get(attr))
document[attr] = Path(v) if v is not None else None
link_css = []
include_css = []
no_css = False
if cli_args.get('no_css') or cli_args.get('link_css') or cli_args.get(
'include_css'):
if cli_args.get('no_css'):
no_css = True
else:
link_css.extend(first_not_none(cli_args.get('link_css'), []))
include_css.extend(
first_not_none(cli_args.get('include_css'), []))
else:
link_args = [
"link-css", "add-link-css", "include-css", "add-include-css"
]
if 'no-css' in document_item and any(
document_item.get(k) for k in link_args):
q = '\''
raise UserError(
f"'no-css' parameter incompatible with one of "
f"[{', '.join([q + a + q for a in link_args])}] "
f"in `documents` item: {document_item}.")
no_css = first_not_none(document_item.get('no-css'),
defaults_item.get('no-css'), False)
link_css.extend(
first_not_none(document_item.get('link-css'),
defaults_item.get('link-css'), []))
link_css.extend(
first_not_none(document_item.get('add-link-css'), []))
include_css.extend(
first_not_none(document_item.get('include-css'),
defaults_item.get('include-css'), []))
include_css.extend(
first_not_none(document_item.get('add-include-css'), []))
if link_css or include_css:
no_css = False
document['link_css'] = link_css
document['include_css'] = include_css
document['no_css'] = no_css
attr = 'force'
document[attr] = first_not_none(True if cli_args.get(attr) else None,
document_item.get(attr),
defaults_item.get(attr), False)
attr = 'verbose'
document[attr] = first_not_none(True if cli_args.get(attr) else None,
document_item.get(attr),
defaults_item.get(attr), False)
attr = 'report'
document[attr] = first_not_none(True if cli_args.get(attr) else None,
document_item.get(attr),
defaults_item.get(attr), False)
if document['report'] and document['verbose']:
raise UserError(
f"Incompatible 'report' and 'verbose' parameters for 'documents' "
f"item: {document_item}.")
enrich_document(document)
# Even if all page flows are defined in the 'documents' section, at least empty
# 'page-flows' plugin must be defined in order to activate page flows processing.
if page_flows_plugin_item is not None:
if ((1 if 'no-page-flows' in document_item else 0) +
(1 if 'page-flows' in document_item else 0) +
(1 if 'add-page-flows' in document_item else 0) > 1):
raise UserError(
f"Incompatible 'no-page-flows', 'page-flows' and 'add-page-flows' "
f"parameters for 'documents' item: {document_item}.")
attr = 'page-flows'
page_flows = first_not_none(
[] if document_item.get('no-page-flows') else
document_item.get(attr), defaults_item.get(attr), [])
for page_flow in page_flows:
page_flow_list = documents_page_flows.setdefault(page_flow, [])
page_flow_list.append({
"link": document["output_file"],
"title": document["title"]
})
add_page_flows = first_not_none(
document_item.get("add-page-flows"), [])
for page_flow in add_page_flows:
page_flow_list = documents_page_flows.setdefault(page_flow, [])
page_flow_list.append({
"link": document["output_file"],
"title": document["title"]
})
documents.append(document)
add_documents_page_flows_data(page_flows_plugin_item, documents_page_flows)
for k, v in plugins_item.items():
plugin = PLUGINS.get(k)
if plugin:
try:
if plugin.accept_data(v):
plugins.append(plugin)
except UserError as e:
raise UserError(
f"Error initializing plugin '{k}': {type(e).__name__}: {e}"
)
return Arguments(options, documents, plugins)
def md2html(document, plugins, metadata_handlers, options):
input_location = document['input_file']
output_location = document['output_file']
title = document['title']
template_file = document['template']
link_css = document['link_css']
include_css = document['include_css']
force = document['force']
verbose = document['verbose']
report = document['report']
output_file = Path(output_location)
input_file = Path(input_location)
if not force and output_file.exists():
output_file_mtime = os.path.getmtime(output_file)
input_file_mtime = os.path.getmtime(input_file)
if output_file_mtime > input_file_mtime:
if verbose:
print(
f'The output file is up-to-date. Skipping: {output_location}'
)
return
current_time = datetime.today()
substitutions = {
'title': title,
'exec_name': EXEC_NAME,
'exec_version': EXEC_VERSION,
'generation_date': current_time.strftime('%Y-%m-%d'),
'generation_time': current_time.strftime('%H:%M:%S')
}
styles = []
if link_css:
styles.extend([
f'<link rel="stylesheet" type="text/css" href="{item}">'
for item in link_css
])
if include_css:
styles.extend([
'<style>\n' + read_lines_from_file(item) + '\n</style>'
for item in include_css
])
substitutions['styles'] = '\n'.join(styles) if styles else ''
md_lines = read_lines_from_file(input_file)
for plugin in plugins:
plugin.new_page()
md_lines = apply_metadata_handlers(md_lines, metadata_handlers, document)
substitutions['content'] = MARKDOWN.convert(source=md_lines)
for plugin in plugins:
substitutions.update(plugin.variables(document))
if options['legacy_mode']:
placeholders = substitutions.get('placeholders')
if placeholders is not None:
del substitutions['placeholders']
substitutions.update(placeholders)
template = read_lines_from_cached_file_legacy(template_file)
else:
template = read_lines_from_cached_file(template_file)
if substitutions['title'] is None:
substitutions['title'] = ''
try:
result = chevron.render(template, substitutions)
except chevron.ChevronError as e:
raise UserError(f"Error processing template: {type(e).__name__}: {e}")
with open(output_file, 'w') as result_file:
result_file.write(result)
if verbose:
print(f'Output file generated: {output_location}')
if report:
print(output_location)