def create_and_generate(bucket_name, frequency_width, ami_id, spot_price1, spot_price2, volume, days_per_node, add_shutdown): boto_data = get_aws_credentials('aws-chiles02') if boto_data is not None: work_to_do = WorkToDo(frequency_width, bucket_name, get_s3_split_name(frequency_width)) work_to_do.calculate_work_to_do() days = work_to_do.work_to_do.keys() nodes_required, node_count = get_nodes_required(days, days_per_node, spot_price1, spot_price2) if len(nodes_required) > 0: uuid = get_uuid() ec2_data = EC2Controller( ami_id, nodes_required, get_node_manager_user_data(boto_data, uuid), AWS_REGION, tags=[ { 'Key': 'Owner', 'Value': getpass.getuser(), }, { 'Key': 'Name', 'Value': 'Daliuge NM - MsTransform', }, { 'Key': 'uuid', 'Value': uuid, } ] ) ec2_data.start_instances() reported_running = get_reported_running( uuid, node_count, wait=600 ) hosts = build_hosts(reported_running) # Create the Data Island Manager data_island_manager = EC2Controller( ami_id, [ { 'number_instances': 1, 'instance_type': 'm4.large', 'spot_price': spot_price1 } ], get_data_island_manager_user_data(boto_data, hosts, uuid), AWS_REGION, tags=[ { 'Key': 'Owner', 'Value': getpass.getuser(), }, { 'Key': 'Name', 'Value': 'Daliuge DIM - MsTransform', }, { 'Key': 'uuid', 'Value': uuid, } ] ) data_island_manager.start_instances() data_island_manager_running = get_reported_running( uuid, 1, wait=600 ) if len(data_island_manager_running['m4.large']) == 1: # Now build the graph session_id = get_session_id() instance_details = data_island_manager_running['m4.large'][0] host = instance_details['ip_address'] graph = BuildGraphMsTransform( work_to_do.work_to_do, bucket_name, volume, 7, reported_running, add_shutdown, frequency_width, session_id, hosts, ) graph.build_graph() graph.tag_all_app_drops({ "session_id": session_id, }) LOG.info('Connection to {0}:{1}'.format(host, DIM_PORT)) client = DataIslandManagerClient(host, DIM_PORT) client.create_session(session_id) client.append_graph(session_id, graph.drop_list) client.deploy_session(session_id, graph.start_oids) else: LOG.error('Unable to find the AWS credentials')