def prepare(self, values):
""" apply templates to all the .sb_template files and build the
fix_perms.sh script from the permissions file"""
values.update(self.cfg['pxe'])
utils.rmtree(self.dst)
utils.copy_tree(self.path, self.dst)
for root, _, files in os.walk(self.dst):
for file_name in files:
if file_name.endswith('.sb_template'):
file_name = os.path.join(root, file_name)
utils.write_template(values, file_name)
utils.file_move(file_name, os.path.splitext(file_name)[0])
def prepare(self, values):
""" apply templates to all the .sb_template files and build the
fix_perms.sh script from the permissions file"""
values.update(self.cfg['pxe'])
utils.rmtree(self.dst)
utils.copy_tree(self.path, self.dst)
for root, _, files in os.walk(self.dst):
for file_name in files:
if file_name.endswith('.sb_template'):
file_name = os.path.join(root, file_name)
utils.write_template(values, file_name)
utils.file_move(file_name, os.path.splitext(file_name)[0])
def add_templates(self, distribution):
"""process and add the rc.local and isolinux templates"""
path = self.cfg['paths']['templates']
dst = os.path.join(self.work_iso, 'isolinux/isolinux.cfg')
if not os.path.isfile(dst):
template = 'template_isolinux_mini'
dst = os.path.join(self.work_iso, 'isolinux.cfg')
else:
template = 'template_isolinux'
src = os.path.join(path, self.cfg[distribution][template])
utils.write_template(self.data, src, dst)
src = os.path.join(path, self.cfg['templates']['rc_local'])
dst = os.path.join(self.work_iso, 'seedbank/etc/rc.local')
utils.write_template(self.data, src, dst)
def add_templates(self, distribution):
"""process and add the rc.local and isolinux templates"""
path = self.cfg['paths']['templates']
dst = os.path.join(self.work_iso, 'isolinux/isolinux.cfg')
if not os.path.isfile(dst):
template = 'template_isolinux_mini'
dst = os.path.join(self.work_iso, 'isolinux.cfg')
else:
template = 'template_isolinux'
src = os.path.join(path, self.cfg[distribution][template])
utils.write_template(self.data, src, dst)
src = os.path.join(path, self.cfg['templates']['rc_local'])
dst = os.path.join(self.work_iso, 'seedbank/etc/rc.local')
utils.write_template(self.data, src, dst)
def pimp(self, seeds, overlay, manifests):
"""pimp the seed file template"""
commands = self.cfg['commands']
values = self.cfg['seed']
if self.target == 'iso':
cmd_overlay = commands['iso_overlay']
cmd_early = commands['iso_early_command']
cmd_late = commands['iso_late_command']
values['late_command'] += commands['iso_mount_command']
elif self.target == 'pxe':
cmd_overlay = commands['pxe_overlay']
cmd_puppet_manifest = commands['pxe_puppet_manifest']
cmd_early = commands['pxe_early_command']
cmd_late = commands['pxe_late_command']
if overlay:
values['late_command'] += cmd_overlay
if self.target == 'pxe' and manifests:
values['late_command'] += commands['pxe_puppet_manifests']
for manifest in manifests:
values['manifest'] = manifest
if self.target == 'pxe':
puppet_command = commands_merge(cmd_puppet_manifest, values)
values['late_command'] += [puppet_command]
elif self.target == 'iso':
src = os.path.join(self.cfg['paths']['templates'],
self.cfg['templates']['puppet_manifest'])
path = os.path.join(self.cfg['paths']['temp'], 'seedbank',
values['fqdn'],
'iso/iso/seedbank/etc/runonce.d')
utils.make_dirs(path)
dst = os.path.join(path,
'puppet_manifest_%s.enabled' % manifest)
utils.write_template(values, src, dst)
values['early_command'] += cmd_early
values['early_command'] = commands_merge(values['early_command'],
values)
values['late_command'] += cmd_late
values['late_command'] = commands_merge(values['late_command'], values)
seed_file = self._merge_seeds(seeds, values)
logging.debug(seed_file)
logging.info('%(fqdn)s - generated preseed file', values)
return seed_file
def pimp(self, seeds, overlay, manifests):
"""pimp the seed file template"""
commands = self.cfg['commands']
values = self.cfg['seed']
if self.target == 'iso':
cmd_overlay = commands['iso_overlay']
cmd_early = commands['iso_early_command']
cmd_late = commands['iso_late_command']
values['late_command'] += commands['iso_mount_command']
elif self.target == 'pxe':
cmd_overlay = commands['pxe_overlay']
cmd_puppet_manifest = commands['pxe_puppet_manifest']
cmd_early = commands['pxe_early_command']
cmd_late = commands['pxe_late_command']
if overlay:
values['late_command'] += cmd_overlay
if self.target == 'pxe' and manifests:
values['late_command'] += commands['pxe_puppet_manifests']
for manifest in manifests:
values['manifest'] = manifest
if self.target == 'pxe':
puppet_command = commands_merge(cmd_puppet_manifest, values)
values['late_command'] += [puppet_command]
elif self.target == 'iso':
src = os.path.join(self.cfg['paths']['templates'],
self.cfg['templates']['puppet_manifest'])
path = os.path.join(self.cfg['paths']['temp'], 'seedbank',
values['fqdn'], 'iso/iso/seedbank/etc/runonce.d')
utils.make_dirs(path)
dst = os.path.join(path, 'puppet_manifest_%s.enabled' %
manifest)
utils.write_template(values, src, dst)
values['early_command'] += cmd_early
values['early_command'] = commands_merge(values['early_command'],
values)
values['late_command'] += cmd_late
values['late_command'] = commands_merge(values['late_command'], values)
seed_file = self._merge_seeds(seeds, values)
logging.debug(seed_file)
logging.info('%(fqdn)s - generated preseed file', values)
return seed_file
def analyze(chat, brain):
analyzer = Analyzer.analyze(chat)
if analyzer.rdf['predicate'] in analyzer.GRAMMAR['verbs']:
analyzer.rdf['predicate']+='s'
if analyzer.utterance_type == UtteranceType.STATEMENT:
if not utils.check_rdf_completeness(analyzer.rdf):
print('incomplete statement RDF')
return
template = utils.write_template(chat.speaker, analyzer.rdf, chat.id, chat.last_utterance.chat_turn,
analyzer.utterance_type)
print(template)
return chat.last_utterance
def main(base_name,
new_name=None,
min_suspects=999999,
aggressiveness=0.5,
guidance_method=None,
timeout=3600,
pass_timeout=4000,
verbose=False):
if not os.path.exists(base_name + ".template"):
raise ValueError("File %s does not exist" % (base_name + ".template"))
dir = os.path.dirname(base_name)
orig_dir = os.getcwd()
os.chdir(dir)
base_name = os.path.basename(base_name)
new_name = os.path.basename(new_name)
with open("args.txt", "w") as f:
f.write("%i\n" % (min_suspects))
f.write("%.3f\n" % (aggressiveness))
f.write("%i\n" % METHODS.index(guidance_method))
if new_name is None:
success = run_debug(base_name, timeout=timeout, verbose=verbose)
os.chdir(orig_dir)
return success
else:
print base_name, new_name
if guidance_method is not None:
assert os.system(
"cp %s_input_embeddings.txt input_embeddings.txt" %
(base_name)) == 0
assert os.system(
"cp %s_output_embeddings.txt output_embeddings.txt" %
(base_name)) == 0
assert os.system("cp %s.template %s.template" %
(base_name, new_name)) == 0
true_suspectz = utils.parse_suspects(base_name)
with open("true_suspects.txt", "w") as f:
f.write("\n".join(true_suspectz))
# Modify template file as needed
utils.write_template(new_name + ".template", "PROJECT=",
"PROJECT=" + new_name)
general_options = VDB_OPTIONS + " --solver-cpu-limit=%i" % (
pass_timeout)
utils.write_template(new_name + ".template", "GENERAL_OPTIONS=",
"GENERAL_OPTIONS=\"%s\"" % (general_options))
pre_runtime = parse_pre_runtime(base_name)
if verbose:
print "Adding %i seconds for pre-vdb operations" % (pre_runtime)
success = run_debug(new_name,
timeout=timeout + pre_runtime,
verbose=verbose)
if not success:
os.chdir(orig_dir)
return False
assert os.path.exists(new_name + ".vennsawork")
num_suspects = utils.parse_suspects(new_name)
if len(num_suspects) == 0:
return False
if guidance_method == "block":
analysis_func = analyze.blocking_analysis
elif guidance_method in [
"assump", "opt_assump", "rev_assump", "assump_block"
]:
analysis_func = analyze.assumption_analysis
else:
os.chdir(orig_dir)
return True
try:
if "_1pass" in new_name:
analysis_func(base_name + "_1pass",
new_name,
verbose=verbose,
min_runtime=0)
else:
analysis_func(base_name,
new_name,
verbose=verbose,
min_runtime=0)
except:
os.chdir(orig_dir)
return False
os.chdir(orig_dir)
return True
Value=Ref(postgres_security_group)),
Output('RedshiftSecurityGroup',
Description=
'Security group to enable connection to Redshift from Everywhere',
Value=Ref(redshift_security_group)),
Output(
'EMRMasterSecurityGroup',
Description='Security group to enable some app ports for Master Node',
Value=Ref(master_security_group)),
])
template_json = template.to_json(indent=4)
print(template_json)
stack_args = {
'StackName': STACK_NAME,
'TemplateBody': template_json,
'Tags': [{
'Key': 'Purpose',
'Value': 'Networking'
}]
}
cfn = boto3.client('cloudformation')
cfn.validate_template(TemplateBody=template_json)
utils.write_template(**stack_args)
# cfn.create_stack(**stack_args)
# cfn.update_stack(**stack_args)
# cfn.delete_stack(StackName=STACK_NAME)
""" Creates a stack template """
import utils
import core_processor
import boto3
cf_client = boto3.client("cloudformation")
complete_list = ["service", "task", "lambda", "worker", "static", "client", "proxy"]
for t in complete_list:
metadata = utils.load_yaml_file(f"{t}-metadata.yaml")
template = core_processor.process_template(metadata)
project_type = metadata['type']
t = utils.write_template(template, project_type)
print("VALIDATED", template["Description"],cf_client.validate_template(TemplateBody=t))