def generate_json(input_dir, bucket, nodes, volume, shutdown): work_to_do = WorkToDo(width, bucket, get_s3_uvsub_name(width), database_connection) work_to_do.calculate_work_to_do() node_details = { 'i2.2xlarge': [{'ip_address': 'node_i2_{0}'.format(i)} for i in range(0, nodes)], } graph = BuildGraphStats( work_to_do.work_to_do, bucket, volume, PARALLEL_STREAMS, node_details, shutdown, width, 'session_id', map_day_name, password, database_ip, '1.2.3.4' ) graph.build_graph() json_dumps = json.dumps(graph.drop_list, indent=2) LOG.info(json_dumps) with open("/tmp/json_stats.txt", "w") as json_file: json_file.write(json_dumps)
def use_and_generate(host, port, bucket_name, frequency_width, volume, add_shutdown, password): boto_data = get_aws_credentials('aws-chiles02') if boto_data is not None: database_connection, map_day_name, database_ip = setup_database(password, bucket_name) connection = httplib.HTTPConnection(host, port) connection.request('GET', '/api', None, {}) response = connection.getresponse() if response.status != httplib.OK: msg = 'Error while processing GET request for {0}:{1}/api (status {2}): {3}'.format(host, port, response.status, response.read()) raise Exception(msg) json_data = response.read() message_details = json.loads(json_data) host_list = message_details['hosts'] nodes_running = get_nodes_running(host_list) if len(nodes_running) > 0: work_to_do = WorkToDo(frequency_width, bucket_name, get_s3_uvsub_name(frequency_width), database_connection) work_to_do.calculate_work_to_do() # Now build the graph session_id = get_session_id() graph = BuildGraphStats( work_to_do.work_to_do, bucket_name, volume, PARALLEL_STREAMS, nodes_running, add_shutdown, frequency_width, session_id, map_day_name, password, database_ip, host ) graph.build_graph() LOG.info('Connection to {0}:{1}'.format(host, port)) client = DataIslandManagerClient(host, port) client.create_session(session_id) client.append_graph(session_id, graph.drop_list) client.deploy_session(session_id, graph.start_oids) else: LOG.warning('No nodes are running')
def use_and_generate(host, port, bucket_name, frequency_width, volume, add_shutdown, password): boto_data = get_aws_credentials('aws-chiles02') if boto_data is not None: database_connection, map_day_name, database_ip = setup_database( password, bucket_name) connection = httplib.HTTPConnection(host, port) connection.request('GET', '/api', None, {}) response = connection.getresponse() if response.status != httplib.OK: msg = 'Error while processing GET request for {0}:{1}/api (status {2}): {3}'.format( host, port, response.status, response.read()) raise Exception(msg) json_data = response.read() message_details = json.loads(json_data) host_list = message_details['hosts'] nodes_running = get_nodes_running(host_list) if len(nodes_running) > 0: work_to_do = WorkToDo(frequency_width, bucket_name, get_s3_uvsub_name(frequency_width), database_connection) work_to_do.calculate_work_to_do() # Now build the graph session_id = get_session_id() graph = BuildGraphStats(work_to_do.work_to_do, bucket_name, volume, PARALLEL_STREAMS, nodes_running, add_shutdown, frequency_width, session_id, map_day_name, password, database_ip, host) graph.build_graph() LOG.info('Connection to {0}:{1}'.format(host, port)) client = DataIslandManagerClient(host, port) client.create_session(session_id) client.append_graph(session_id, graph.drop_list) client.deploy_session(session_id, graph.start_oids) else: LOG.warning('No nodes are running')
def generate_json(width, bucket, nodes, volume, shutdown, password): database_connection, map_day_name, database_ip = setup_database( password, bucket) work_to_do = WorkToDo(width, bucket, get_s3_uvsub_name(width), database_connection) work_to_do.calculate_work_to_do() node_details = { 'i2.2xlarge': [{ 'ip_address': 'node_i2_{0}'.format(i) } for i in range(0, nodes)], } graph = BuildGraphStats(work_to_do.work_to_do, bucket, volume, PARALLEL_STREAMS, node_details, shutdown, width, 'session_id', map_day_name, password, database_ip, '1.2.3.4') graph.build_graph() json_dumps = json.dumps(graph.drop_list, indent=2) LOG.info(json_dumps) with open("/tmp/json_stats.txt", "w") as json_file: json_file.write(json_dumps)
def create_and_generate(bucket_name, frequency_width, ami_id, spot_price, volume, nodes, add_shutdown, password, log_level): boto_data = get_aws_credentials('aws-chiles02') if boto_data is not None: database_connection, map_day_name, database_ip = setup_database( password, bucket_name) work_to_do = WorkToDo(frequency_width, bucket_name, get_s3_uvsub_name(frequency_width), database_connection) work_to_do.calculate_work_to_do() nodes_required, node_count = get_nodes_required(nodes, spot_price) if len(nodes_required) > 0: uuid = get_uuid() ec2_data = EC2Controller(ami_id, nodes_required, get_node_manager_user_data( boto_data, uuid, max_request_size=50, log_level=log_level), AWS_REGION, tags=[{ 'Key': 'Owner', 'Value': getpass.getuser(), }, { 'Key': 'Name', 'Value': 'Daliuge NM - Stats', }, { 'Key': 'uuid', 'Value': uuid, }]) ec2_data.start_instances() reported_running = get_reported_running(uuid, node_count, wait=600) if len(reported_running) == 0: LOG.error('Nothing has reported ready') else: hosts = build_hosts(reported_running) # Create the Data Island Manager data_island_manager = EC2Controller( ami_id, [{ 'number_instances': 1, 'instance_type': 'm4.large', 'spot_price': spot_price }], get_data_island_manager_user_data(boto_data, hosts, uuid, need_node_manager=True, max_request_size=50), AWS_REGION, tags=[{ 'Key': 'Owner', 'Value': getpass.getuser(), }, { 'Key': 'Name', 'Value': 'Daliuge DIM - Stats', }, { 'Key': 'uuid', 'Value': uuid, }]) data_island_manager.start_instances() data_island_manager_running = get_reported_running( uuid, 1, wait=600, ) if len(data_island_manager_running['m4.large']) == 1: # Now build the graph session_id = get_session_id() instance_details = data_island_manager_running['m4.large'][ 0] host = instance_details['ip_address'] graph = BuildGraphStats(work_to_do.work_to_do, bucket_name, volume, PARALLEL_STREAMS, reported_running, add_shutdown, frequency_width, session_id, map_day_name, password, database_ip, host) graph.build_graph() LOG.info('Connection to {0}:{1}'.format(host, DIM_PORT)) client = DataIslandManagerClient(host, DIM_PORT) client.create_session(session_id) client.append_graph(session_id, graph.drop_list) client.deploy_session(session_id, graph.start_oids) else: LOG.error('Unable to find the AWS credentials')
def create_and_generate(bucket_name, input_dir, ami_id, spot_price, volume, nodes, add_shutdown, log_level): boto_data = get_aws_credentials('aws-chiles02') if boto_data is not None: work_to_do = WorkToDo(frequency_width, bucket_name, get_s3_uvsub_name(frequency_width), database_connection) work_to_do.calculate_work_to_do() nodes_required, node_count = get_nodes_required(nodes, spot_price) if len(nodes_required) > 0: uuid = get_uuid() ec2_data = EC2Controller( ami_id, nodes_required, get_node_manager_user_data(boto_data, uuid, max_request_size=50, log_level=log_level), AWS_REGION, tags=[ { 'Key': 'Owner', 'Value': getpass.getuser(), }, { 'Key': 'Name', 'Value': 'DALiuGE NM - Stats', }, { 'Key': 'uuid', 'Value': uuid, } ] ) ec2_data.start_instances() reported_running = get_reported_running( uuid, node_count, wait=600 ) if len(reported_running) == 0: LOG.error('Nothing has reported ready') else: hosts = build_hosts(reported_running) # Create the Data Island Manager data_island_manager = EC2Controller( ami_id, [ { 'number_instances': 1, 'instance_type': 'm4.large', 'spot_price': spot_price } ], get_data_island_manager_user_data( boto_data, hosts, uuid, need_node_manager=True, max_request_size=50 ), AWS_REGION, tags=[ { 'Key': 'Owner', 'Value': getpass.getuser(), }, { 'Key': 'Name', 'Value': 'DALiuGE DIM - Stats', }, { 'Key': 'uuid', 'Value': uuid, } ] ) data_island_manager.start_instances() data_island_manager_running = get_reported_running( uuid, 1, wait=600, ) if len(data_island_manager_running['m4.large']) == 1: # Now build the graph session_id = get_session_id() instance_details = data_island_manager_running['m4.large'][0] host = instance_details['ip_address'] graph = BuildGraphStats( work_to_do.work_to_do, bucket_name, volume, PARALLEL_STREAMS, reported_running, add_shutdown, frequency_width, session_id, map_day_name, password, database_ip, host ) graph.build_graph() LOG.info('Connection to {0}:{1}'.format(host, DIM_PORT)) client = DataIslandManagerClient(host, DIM_PORT) client.create_session(session_id) client.append_graph(session_id, graph.drop_list) client.deploy_session(session_id, get_roots(graph.drop_list)) else: LOG.error('Unable to find the AWS credentials')