def test_update_config2(run_task_awsem_event_data2): data = run_task_awsem_event_data2 config = data['config'] update_config(config, data['args']['app_name'], data['args']['input_files'], data['args']['input_parameters']) assert config['instance_type'] == 't2.xlarge' assert config['EBS_optimized'] is False assert config['ebs_size'] >= 10 assert config['copy_to_s3'] is True # check the other fields are preserved in the returned config
def test_update_config5(run_task_awsem_event_omit_fields2): data = run_task_awsem_event_omit_fields2 config = data['config'] update_config(config, data['args'].get('app_name', ''), data['args']['input_files'], {}) assert config['instance_type'] == 't3.micro' assert config['EBS_optimized'] is True assert config['ebs_size'] >= 10 assert config['shutdown_min'] == "now"
def test_update_config3(run_task_awsem_event_data_chipseq): data = run_task_awsem_event_data_chipseq config = data['config'] update_config(config, data['args']['app_name'], data['args']['input_files'], data['args']['input_parameters']) assert config['instance_type'] == 'c5.4xlarge' assert config['EBS_optimized'] is True assert config['ebs_size'] == 87
def test_update_config(run_task_awsem_event_data): data = run_task_awsem_event_data config = data['config'] update_config(config, data['args']['app_name'], data['args']['input_files'], data['args']['input_parameters']) assert config['instance_type'] == 't3.micro' assert config['EBS_optimized'] is True assert config['ebs_size'] >= 10 assert config[ 'shutdown_min'] == 30 # check the other fields are preserved in the returned config
def handler(event, context): ''' config: instance_type: EC2 instance type ebs_size: EBS storage size in GB ebs_type: EBS storage type (available values: gp2, io1, st1, sc1, standard (default: io1) ebs_iops: EBS storage IOPS password: password for ssh connection for user ec2-user EBS_optimized: Use this flag if the instance type is EBS-optimized (default: EBS-optimized) shutdown_min: Number of minutes before shutdown after the jobs are finished. (default now) copy_to_s3: Upload or copy the json file to S3 bucket json_bucket launch_instance: Launch instance based on the json file log_bucket: bucket for collecting logs (started, postrun, success, error, log) args: cwl_main_filename: main cwl file name cwl_child_filenames: names of the other cwl files used by main cwl file, delimiated by comma app_name: name of the app app_version: version of the app cwl_directory_url: the url and subdirectories for the main cwl file cwl_version: the version of cwl (either 'draft3' or 'v1') input_reference_files_directory: bucket name and subdirectory for input reference files output_S3_bucket: bucket name and subdirectory for output files and logs input_files: input files in json format (parametername: {'bucket_name':bucketname, 'object_key':filename}) secondary_files: secondary files in json format (parametername: {'bucket_name':bucketnname, 'object_ke':filename}) input_parameters: input parameters in json format (parametername:value) ''' # read default variables in config CONFIG_FIELD = "config" CONFIG_KEYS = [ "EBS_optimized", "shutdown_min", "copy_to_s3", "instance_type", "ebs_size", "launch_instance", "key_name", "ebs_type", "ebs_iops", "json_bucket", "password", "log_bucket" ] ARGS_FIELD = "args" ARGS_KEYS = [ "cwl_main_filename", "cwl_child_filenames", "app_name", "app_version", "input_files", "output_S3_bucket", "cwl_directory_url", "input_parameters", "secondary_files", "output_target", "secondary_output_target" ] cfg = event.get(CONFIG_FIELD) for k in CONFIG_KEYS: assert k in cfg, "%s not in config_field" % k args = event.get(ARGS_FIELD) for k in ARGS_KEYS: assert k in args, "%s not in args field" % k # args: parameters needed by the instance to run a workflow # cfg: parameters needed to launch an instance cfg['job_tag'] = args.get('app_name') cfg['userdata_dir'] = '/tmp/userdata' # local directory in which the json file will be first created. cfg['json_dir'] = '/tmp/json' # AMI and script directory according to cwl version if args['cwl_version'] == 'v1': cfg['ami_id'] = os.environ.get('AMI_ID_CWL_V1') cfg['script_url'] = 'https://raw.githubusercontent.com/' + \ os.environ.get('TIBANNA_REPO_NAME') + '/' + \ os.environ.get('TIBANNA_REPO_BRANCH') + '/awsf_cwl_v1/' else: cfg['ami_id'] = os.environ.get('AMI_ID_CWL_DRAFT3') cfg['script_url'] = 'https://raw.githubusercontent.com/' + \ os.environ.get('TIBANNA_REPO_NAME') + '/' + \ os.environ.get('TIBANNA_REPO_BRANCH') + '/awsf_cwl_draft3/' utils.update_config(cfg, args['app_name'], args['input_files'], args['input_parameters']) # create json and copy to s3 jobid = utils.create_json(event, '') # launch instance and execute workflow if cfg.get('launch_instance'): launch_instance_log = utils.launch_instance(cfg, jobid) event.update({'jobid': jobid}) event.update(launch_instance_log) return (event)