def generate(self):
root_path = find_nearest_root()
if not root_path:
self.app.log.fatal(
"Could not find the root of the project please change directories"
)
return None
fs.ensure_dir_exists("%s/jobs" % root_path)
job_dir = "{root}/jobs/{name}".format(root=root_path,
name=self.app.pargs.name)
if os.path.exists(job_dir):
self.app.log.fatal(
"Sorry there is already a job in this directory can't overwrite"
)
return None
example_path = os.path.join(fs.abspath(__file__), os.pardir, os.pardir,
"templates", "word_count")
data = {"name": self.app.pargs.name}
self.app.template.copy(example_path, job_dir, data)
self.app.log.info("""
A cluster config is not created when you generate a job,
please refer to the job README.md for instructions on how to generate
a cluster config file that can be used to run jobs on EMR
Alternatively you can type if you are familiar with the process:
cluster_funk clusters generate-config -h
""")
def create_ec2_key(self):
name = self.app.pargs.name
directory = self.app.pargs.dir
client = self._ec2_client()
names = [
pair['KeyName']
for pair in client.describe_key_pairs().get('KeyPairs', [])
]
if name in names:
self.app.log.fatal(
"Key already exists please specify a different key name")
self.app.log.info(
"These are the keys associated with your profile")
self.app.log.info(yaml.dump(names, default_flow_style=False))
return
keypair = client.create_key_pair(KeyName=name)
fs.ensure_dir_exists(directory)
key_location = os.path.join(directory,
"{name}".format(name=keypair['KeyName']))
with open(key_location, 'w') as text_file:
text_file.write(keypair['KeyMaterial'])
self.app.log.info("Wrote new key to %s" % (key_location))
os.chmod(key_location, stat.S_IRUSR | stat.S_IWUSR)
self.app.log.info(
"Changed permissions to prevent other users from reading the key")
def init_certs(app):
certs_folder = app.config.get('esper', 'certs_folder')
path = fs.abspath(certs_folder)
# Check if path exists
if not Path(path).exists():
app.log.debug(f"[init_certs] Creating Certs folder!")
fs.ensure_dir_exists(path)
app.extend('certs_path', path)
app.extend('local_key', fs.abspath(app.config.get('esper', 'local_key')))
app.extend('local_cert', fs.abspath(app.config.get('esper', 'local_cert')))
app.extend('device_cert',
fs.abspath(app.config.get('esper', 'device_cert')))
def setup(self):
super(MixbytesTank, self).setup()
fs.ensure_dir_exists(self.user_dir)
additional_config_defaults = {
'tank': {
'terraform_run_command': os.path.join(self.installation_dir, 'terraform'),
'terraform_inventory_run_command': os.path.join(self.installation_dir, 'terraform-inventory'),
},
}
for section, variables_dict in additional_config_defaults.items():
for key, value in variables_dict.items():
if key not in self.config.keys(section):
self.config.set(section, key, value)
def setup(self):
super(MixbytesTank, self).setup()
self.work_dir = fs.abspath('.')
self.state_dir = self.work_dir + '/.tank/state/'
self.log_dir = self.work_dir + '/.tank/log/'
self.root_dir = os.path.realpath(os.path.dirname(__file__)) + "/"
self.provider = self.config.get(self.Meta.label, "provider")
self.terraform_provider_dir = join(self.root_dir, 'providers',
self.provider)
self.terraform_plan_dir = self.terraform_provider_dir
self.terraform_log_path = self.log_dir + 'terraform.log'
self.terraform_run_command = self.config.get(self.Meta.label,
'terraform_run_command')
self.terraform_inventory_run_command = self.config.get(
self.Meta.label, 'terraform_inventory_run_command')
fs.ensure_dir_exists(self.state_dir)
fs.ensure_dir_exists(self.log_dir)
self.terraform_state_file = self.state_dir + "/blockchain.tfstate"
self.app_env = os.environ.copy()
self.app_env["TF_LOG"] = "TRACE"
self.app_env["TF_LOG_PATH"] = self.terraform_log_path
self.app_env["TF_DATA_DIR"] = self.state_dir
self.app_env["TF_IN_AUTOMATION"] = "true"
self.app_env["TF_VAR_state_path"] = self.terraform_state_file
self.app_env["TF_VAR_bc_count_prod_instances"] = str(
self.blockchain_instances - 1)
for common_key in self.config.keys(self.Meta.label):
self.app_env["TF_VAR_" + common_key] = \
self.config.get(self.Meta.label, common_key)
for provider_key in self.config.keys(self.provider):
self.app_env["TF_VAR_" + provider_key] = \
self.config.get(self.provider, provider_key)
self.app_env["ANSIBLE_ROLES_PATH"] = \
self.state_dir + "/roles"
self.app_env["ANSIBLE_CONFIG"] = \
self.root_dir + "/tools/ansible/ansible.cfg"
def test_ensure_dir_exists(tmp, rando):
fs.ensure_dir_exists(fs.join(tmp.dir, rando))
assert os.path.exists(fs.join(tmp.dir, rando))
with raises(AssertionError, match='(.*)exists but is not a directory(.*)'):
fs.ensure_dir_exists(tmp.file)
def setup(self):
super(MixbytesTank, self).setup()
fs.ensure_dir_exists(self.user_dir)
def generate_config(self):
args = self.app.pargs
job = args.job or "unknown job"
env = args.env or "dev"
client = self._cloudformation_client()
stacks = StackCollection(client). \
filter_by(StackCollection.is_cf_stack). \
filter_by(partial(StackCollection.has_env, args.env)). \
filter_by(partial(StackCollection.has_prefix, args.stack_prefix))
output = stacks.output_dict()
cluster_name = "Apache Spark for job {env}/{job}".format(env=env,
job=job)
# acquire latest Base AMI
config = {
'Name':
cluster_name,
'WorkLoadBucket':
args.bucket,
'WorkLoadPrefix':
"apache_spark_for_job_{env}_{job}".format(env=env, job=job),
'Schedule': {
'Period': 'hourly',
'Starting': datetime.now(),
'Ending': datetime.now()
},
'Master': {
'Instance': {
'Type': 'm4.large',
'Storage': 32
}
},
'Worker': {
'Instance': {
'Type': 'm4.large',
'Count': 1,
'Storage': 100,
'SpotCount': 0
}
},
'KeepJobFlowAliveWhenNoSteps':
True,
'PipInstall': ["pandas==0.24.0"],
'JobFlowRole':
output.get('EMREc2RoleName', 'EMR_EC2_DefaultRole'),
'ServiceRole':
output.get('EMRRoleName', 'EMR_DefaultRole'),
'AutoScalingRole':
output.get('EMRAutoScalingRoleName',
'EMR_AutoScaling_DefaultRole'),
'Ec2SubnetId':
output.get('PublicSubnets', 'WARNING_INVALID_VALUE'),
'EmrManagedMasterSecurityGroup':
output.get('EmrManagedMasterSecurityGroup',
'WARNING_INVALID_VALUE'),
'EmrManagedSlaveSecurityGroup':
output.get('EmrManagedSlaveSecurityGroup',
'WARNING_INVALID_VALUE'),
'EbsRootVolumeSize':
40,
'AwsXrayEnabled':
False,
'ShutdownAfterInactivity':
'2 hours',
'Tags': {
'Name': cluster_name,
'Application': job,
'job': job,
'tool': 'cluster_funk',
'Environment': env,
'environment': env,
}
}
# make sure env and job are valid!
if env and job:
root = find_nearest_root()
env_config_path = os.path.join(root, 'jobs', job, 'config',
'clusters', env)
config_path = os.path.join(env_config_path, "config.yml")
fs.ensure_dir_exists(env_config_path)
if not os.path.exists(config_path) or args.force:
with open(config_path, "w") as text_file:
text_file.write(yaml.dump(config,
default_flow_style=False))
self.app.log.info(
"Wrote a new config file too. {config_path}".format(
config_path=config_path))
else:
self.app.log.info(
"config already exists use -f if you really want to overwrite the existing one"
)
self.app.log.info("\n%s" % yaml.dump(config, default_flow_style=False))