def generate(self): root_path = find_nearest_root() if not root_path: self.app.log.fatal( "Could not find the root of the project please change directories" ) return None fs.ensure_dir_exists("%s/jobs" % root_path) job_dir = "{root}/jobs/{name}".format(root=root_path, name=self.app.pargs.name) if os.path.exists(job_dir): self.app.log.fatal( "Sorry there is already a job in this directory can't overwrite" ) return None example_path = os.path.join(fs.abspath(__file__), os.pardir, os.pardir, "templates", "word_count") data = {"name": self.app.pargs.name} self.app.template.copy(example_path, job_dir, data) self.app.log.info(""" A cluster config is not created when you generate a job, please refer to the job README.md for instructions on how to generate a cluster config file that can be used to run jobs on EMR Alternatively you can type if you are familiar with the process: cluster_funk clusters generate-config -h """)
def create_ec2_key(self): name = self.app.pargs.name directory = self.app.pargs.dir client = self._ec2_client() names = [ pair['KeyName'] for pair in client.describe_key_pairs().get('KeyPairs', []) ] if name in names: self.app.log.fatal( "Key already exists please specify a different key name") self.app.log.info( "These are the keys associated with your profile") self.app.log.info(yaml.dump(names, default_flow_style=False)) return keypair = client.create_key_pair(KeyName=name) fs.ensure_dir_exists(directory) key_location = os.path.join(directory, "{name}".format(name=keypair['KeyName'])) with open(key_location, 'w') as text_file: text_file.write(keypair['KeyMaterial']) self.app.log.info("Wrote new key to %s" % (key_location)) os.chmod(key_location, stat.S_IRUSR | stat.S_IWUSR) self.app.log.info( "Changed permissions to prevent other users from reading the key")
def init_certs(app): certs_folder = app.config.get('esper', 'certs_folder') path = fs.abspath(certs_folder) # Check if path exists if not Path(path).exists(): app.log.debug(f"[init_certs] Creating Certs folder!") fs.ensure_dir_exists(path) app.extend('certs_path', path) app.extend('local_key', fs.abspath(app.config.get('esper', 'local_key'))) app.extend('local_cert', fs.abspath(app.config.get('esper', 'local_cert'))) app.extend('device_cert', fs.abspath(app.config.get('esper', 'device_cert')))
def setup(self): super(MixbytesTank, self).setup() fs.ensure_dir_exists(self.user_dir) additional_config_defaults = { 'tank': { 'terraform_run_command': os.path.join(self.installation_dir, 'terraform'), 'terraform_inventory_run_command': os.path.join(self.installation_dir, 'terraform-inventory'), }, } for section, variables_dict in additional_config_defaults.items(): for key, value in variables_dict.items(): if key not in self.config.keys(section): self.config.set(section, key, value)
def setup(self): super(MixbytesTank, self).setup() self.work_dir = fs.abspath('.') self.state_dir = self.work_dir + '/.tank/state/' self.log_dir = self.work_dir + '/.tank/log/' self.root_dir = os.path.realpath(os.path.dirname(__file__)) + "/" self.provider = self.config.get(self.Meta.label, "provider") self.terraform_provider_dir = join(self.root_dir, 'providers', self.provider) self.terraform_plan_dir = self.terraform_provider_dir self.terraform_log_path = self.log_dir + 'terraform.log' self.terraform_run_command = self.config.get(self.Meta.label, 'terraform_run_command') self.terraform_inventory_run_command = self.config.get( self.Meta.label, 'terraform_inventory_run_command') fs.ensure_dir_exists(self.state_dir) fs.ensure_dir_exists(self.log_dir) self.terraform_state_file = self.state_dir + "/blockchain.tfstate" self.app_env = os.environ.copy() self.app_env["TF_LOG"] = "TRACE" self.app_env["TF_LOG_PATH"] = self.terraform_log_path self.app_env["TF_DATA_DIR"] = self.state_dir self.app_env["TF_IN_AUTOMATION"] = "true" self.app_env["TF_VAR_state_path"] = self.terraform_state_file self.app_env["TF_VAR_bc_count_prod_instances"] = str( self.blockchain_instances - 1) for common_key in self.config.keys(self.Meta.label): self.app_env["TF_VAR_" + common_key] = \ self.config.get(self.Meta.label, common_key) for provider_key in self.config.keys(self.provider): self.app_env["TF_VAR_" + provider_key] = \ self.config.get(self.provider, provider_key) self.app_env["ANSIBLE_ROLES_PATH"] = \ self.state_dir + "/roles" self.app_env["ANSIBLE_CONFIG"] = \ self.root_dir + "/tools/ansible/ansible.cfg"
def test_ensure_dir_exists(tmp, rando): fs.ensure_dir_exists(fs.join(tmp.dir, rando)) assert os.path.exists(fs.join(tmp.dir, rando)) with raises(AssertionError, match='(.*)exists but is not a directory(.*)'): fs.ensure_dir_exists(tmp.file)
def setup(self): super(MixbytesTank, self).setup() fs.ensure_dir_exists(self.user_dir)
def generate_config(self): args = self.app.pargs job = args.job or "unknown job" env = args.env or "dev" client = self._cloudformation_client() stacks = StackCollection(client). \ filter_by(StackCollection.is_cf_stack). \ filter_by(partial(StackCollection.has_env, args.env)). \ filter_by(partial(StackCollection.has_prefix, args.stack_prefix)) output = stacks.output_dict() cluster_name = "Apache Spark for job {env}/{job}".format(env=env, job=job) # acquire latest Base AMI config = { 'Name': cluster_name, 'WorkLoadBucket': args.bucket, 'WorkLoadPrefix': "apache_spark_for_job_{env}_{job}".format(env=env, job=job), 'Schedule': { 'Period': 'hourly', 'Starting': datetime.now(), 'Ending': datetime.now() }, 'Master': { 'Instance': { 'Type': 'm4.large', 'Storage': 32 } }, 'Worker': { 'Instance': { 'Type': 'm4.large', 'Count': 1, 'Storage': 100, 'SpotCount': 0 } }, 'KeepJobFlowAliveWhenNoSteps': True, 'PipInstall': ["pandas==0.24.0"], 'JobFlowRole': output.get('EMREc2RoleName', 'EMR_EC2_DefaultRole'), 'ServiceRole': output.get('EMRRoleName', 'EMR_DefaultRole'), 'AutoScalingRole': output.get('EMRAutoScalingRoleName', 'EMR_AutoScaling_DefaultRole'), 'Ec2SubnetId': output.get('PublicSubnets', 'WARNING_INVALID_VALUE'), 'EmrManagedMasterSecurityGroup': output.get('EmrManagedMasterSecurityGroup', 'WARNING_INVALID_VALUE'), 'EmrManagedSlaveSecurityGroup': output.get('EmrManagedSlaveSecurityGroup', 'WARNING_INVALID_VALUE'), 'EbsRootVolumeSize': 40, 'AwsXrayEnabled': False, 'ShutdownAfterInactivity': '2 hours', 'Tags': { 'Name': cluster_name, 'Application': job, 'job': job, 'tool': 'cluster_funk', 'Environment': env, 'environment': env, } } # make sure env and job are valid! if env and job: root = find_nearest_root() env_config_path = os.path.join(root, 'jobs', job, 'config', 'clusters', env) config_path = os.path.join(env_config_path, "config.yml") fs.ensure_dir_exists(env_config_path) if not os.path.exists(config_path) or args.force: with open(config_path, "w") as text_file: text_file.write(yaml.dump(config, default_flow_style=False)) self.app.log.info( "Wrote a new config file too. {config_path}".format( config_path=config_path)) else: self.app.log.info( "config already exists use -f if you really want to overwrite the existing one" ) self.app.log.info("\n%s" % yaml.dump(config, default_flow_style=False))