def build_graph(self):
start_drop = dropdict({
"type": 'plain',
"storage": 'memory',
"oid": get_oid('memory_in'),
"uid": get_uuid(),
})
self._start_oids.append(start_drop['uid'])
self.append(start_drop)
shutdown_drop = dropdict({
"type": 'app',
"app": get_module_name(BashShellApp),
"oid": get_oid('app_bash_shell_app'),
"uid": get_uuid(),
"command": 'sudo shutdown -h +5 "DFMS node shutting down" &',
"user": '******',
"input_error_threshold": 100,
})
shutdown_drop.addInput(start_drop)
self.append(shutdown_drop)
def build_graph(self):
start_drop = dropdict({
"type": 'plain',
"storage": 'memory',
"oid": get_oid('memory_in'),
"uid": get_uuid(),
})
self._start_oids.append(start_drop['uid'])
self.append(start_drop)
shutdown_drop = dropdict({
"type": 'app',
"app": get_module_name(BashShellApp),
"oid": get_oid('app_bash_shell_app'),
"uid": get_uuid(),
"command": 'sudo shutdown -h +5 "DFMS node shutting down" &',
"user": '******',
"input_error_threshold": 100,
})
shutdown_drop.addInput(start_drop)
self.append(shutdown_drop)
def test_sql_create(self):
sqlite01 = get_oid('sqlite')
sqlite_drop = FileDROP(
sqlite01,
get_uuid(),
precious=False,
dirname=os.path.join(TestSqlite._temp, sqlite01),
check_exists=False,
)
initialize_sqlite = InitializeSqliteApp(
get_oid('app'),
get_uuid(),
user='******',
)
sqlite_in_memory = InMemoryDROP(
get_oid('memory'),
get_uuid(),
precious=False,
)
initialize_sqlite.addInput(sqlite_drop)
initialize_sqlite.addOutput(sqlite_in_memory)
with droputils.DROPWaiterCtx(self, sqlite_in_memory, 50000):
sqlite_drop.setCompleted()
def create_and_generate(bucket_name, frequency_width, ami_id, spot_price, volume, bottom_frequency, nodes, add_shutdown):
boto_data = get_aws_credentials('aws-chiles02')
if boto_data is not None:
uuid = get_uuid()
ec2_data = EC2Controller(
ami_id,
[
{
'number_instances': nodes,
'instance_type': 'i2.2xlarge',
'spot_price': spot_price
}
],
get_node_manager_user_data(boto_data, uuid),
AWS_REGION,
tags=[
{
'Key': 'Owner',
'Value': getpass.getuser(),
},
{
'Key': 'Name',
'Value': 'Daliuge NM - Find bad MS',
},
{
'Key': 'uuid',
'Value': uuid,
}
]
)
ec2_data.start_instances()
reported_running = get_reported_running(
uuid,
nodes,
wait=600
)
if len(reported_running) == 0:
LOG.error('Nothing has reported ready')
else:
hosts = build_hosts(reported_running)
# Create the Data Island Manager
data_island_manager = EC2Controller(
ami_id,
[
{
'number_instances': 1,
'instance_type': 'm4.large',
'spot_price': spot_price
}
],
get_data_island_manager_user_data(boto_data, hosts, uuid),
AWS_REGION,
tags=[
{
'Key': 'Owner',
'Value': getpass.getuser(),
},
{
'Key': 'Name',
'Value': 'Daliuge DIM - Find bad MS',
},
{
'Key': 'uuid',
'Value': uuid,
}
]
)
data_island_manager.start_instances()
data_island_manager_running = get_reported_running(
uuid,
1,
wait=600
)
if len(data_island_manager_running['m4.large']) == 1:
# Now build the graph
session_id = get_session_id()
instance_details = data_island_manager_running['m4.large'][0]
host = instance_details['ip_address']
graph = BuildGraphFindBadMeasurementSet(
bucket_name,
volume,
PARALLEL_STREAMS,
reported_running,
add_shutdown,
frequency_width,
bottom_frequency,
session_id,
host,
)
graph.build_graph()
LOG.info('Connection to {0}:{1}'.format(host, DIM_PORT))
client = DataIslandManagerClient(host, DIM_PORT)
client.create_session(session_id)
client.append_graph(session_id, graph.drop_list)
client.deploy_session(session_id, graph.start_oids)
else:
LOG.error('Unable to find the AWS credentials')
def create_and_generate(
bucket_name,
frequency_width,
w_projection_planes,
ami_id,
spot_price,
volume,
nodes,
add_shutdown,
min_frequency,
max_frequency,
scan_statistics,
dump_json):
boto_data = get_aws_credentials('aws-chiles02')
if boto_data is not None:
work_to_do = WorkToDo(
width=frequency_width,
bucket_name=bucket_name,
s3_uvsub_name=get_s3_uvsub_name(frequency_width),
s3_split_name=get_s3_split_name(frequency_width),
min_frequency=min_frequency,
max_frequency=max_frequency)
work_to_do.calculate_work_to_do()
nodes_required, node_count = get_nodes_required(nodes, spot_price)
if len(nodes_required) > 0:
uuid = get_uuid()
ec2_data = EC2Controller(
ami_id,
nodes_required,
get_node_manager_user_data(boto_data, uuid, max_request_size=50),
AWS_REGION,
tags=[
{
'Key': 'Owner',
'Value': getpass.getuser(),
},
{
'Key': 'Name',
'Value': 'DALiuGE NM - Uvsub',
},
{
'Key': 'uuid',
'Value': uuid,
}
]
)
ec2_data.start_instances()
reported_running = get_reported_running(
uuid,
node_count,
wait=600
)
if len(reported_running) == 0:
LOG.error('Nothing has reported ready')
else:
hosts = build_hosts(reported_running)
# Create the Data Island Manager
data_island_manager = EC2Controller(
ami_id,
[
{
'number_instances': 1,
'instance_type': 'm4.large',
'spot_price': spot_price
}
],
get_data_island_manager_user_data(boto_data, hosts, uuid, max_request_size=50),
AWS_REGION,
tags=[
{
'Key': 'Owner',
'Value': getpass.getuser(),
},
{
'Key': 'Name',
'Value': 'DALiuGE DIM - Uvsub',
},
{
'Key': 'uuid',
'Value': uuid,
}
]
)
data_island_manager.start_instances()
data_island_manager_running = get_reported_running(
uuid,
1,
wait=600
)
if len(data_island_manager_running['m4.large']) == 1:
# Now build the graph
session_id = get_session_id()
instance_details = data_island_manager_running['m4.large'][0]
host = instance_details['ip_address']
graph = BuildGraphUvsub(
work_to_do=work_to_do.work_to_do,
bucket_name=bucket_name,
volume=volume,
parallel_streams=PARALLEL_STREAMS,
node_details=reported_running,
shutdown=add_shutdown,
scan_statistics=scan_statistics,
width=frequency_width,
w_projection_planes=w_projection_planes,
session_id=session_id,
dim_ip=host)
graph.build_graph()
if dump_json:
json_dumps = json.dumps(graph.drop_list, indent=2)
with open("/tmp/json_uvsub.txt", "w") as json_file:
json_file.write(json_dumps)
LOG.info('Connection to {0}:{1}'.format(host, DIM_PORT))
client = DataIslandManagerClient(host, DIM_PORT)
client.create_session(session_id)
client.append_graph(session_id, graph.drop_list)
client.deploy_session(session_id, get_roots(graph.drop_list))
else:
LOG.error('Unable to find the AWS credentials')
def create_and_generate(bucket_name, frequency_width, ami_id, spot_price,
volume, nodes, add_shutdown, password, log_level):
boto_data = get_aws_credentials('aws-chiles02')
if boto_data is not None:
database_connection, map_day_name, database_ip = setup_database(
password, bucket_name)
work_to_do = WorkToDo(frequency_width, bucket_name,
get_s3_uvsub_name(frequency_width),
database_connection)
work_to_do.calculate_work_to_do()
nodes_required, node_count = get_nodes_required(nodes, spot_price)
if len(nodes_required) > 0:
uuid = get_uuid()
ec2_data = EC2Controller(ami_id,
nodes_required,
get_node_manager_user_data(
boto_data,
uuid,
max_request_size=50,
log_level=log_level),
AWS_REGION,
tags=[{
'Key': 'Owner',
'Value': getpass.getuser(),
}, {
'Key': 'Name',
'Value': 'Daliuge NM - Stats',
}, {
'Key': 'uuid',
'Value': uuid,
}])
ec2_data.start_instances()
reported_running = get_reported_running(uuid, node_count, wait=600)
if len(reported_running) == 0:
LOG.error('Nothing has reported ready')
else:
hosts = build_hosts(reported_running)
# Create the Data Island Manager
data_island_manager = EC2Controller(
ami_id, [{
'number_instances': 1,
'instance_type': 'm4.large',
'spot_price': spot_price
}],
get_data_island_manager_user_data(boto_data,
hosts,
uuid,
need_node_manager=True,
max_request_size=50),
AWS_REGION,
tags=[{
'Key': 'Owner',
'Value': getpass.getuser(),
}, {
'Key': 'Name',
'Value': 'Daliuge DIM - Stats',
}, {
'Key': 'uuid',
'Value': uuid,
}])
data_island_manager.start_instances()
data_island_manager_running = get_reported_running(
uuid,
1,
wait=600,
)
if len(data_island_manager_running['m4.large']) == 1:
# Now build the graph
session_id = get_session_id()
instance_details = data_island_manager_running['m4.large'][
0]
host = instance_details['ip_address']
graph = BuildGraphStats(work_to_do.work_to_do, bucket_name,
volume, PARALLEL_STREAMS,
reported_running, add_shutdown,
frequency_width, session_id,
map_day_name, password,
database_ip, host)
graph.build_graph()
LOG.info('Connection to {0}:{1}'.format(host, DIM_PORT))
client = DataIslandManagerClient(host, DIM_PORT)
client.create_session(session_id)
client.append_graph(session_id, graph.drop_list)
client.deploy_session(session_id, graph.start_oids)
else:
LOG.error('Unable to find the AWS credentials')
def create_and_generate(bucket_name, frequency_width, ami_id, spot_price1, spot_price2, volume, days_per_node, add_shutdown):
boto_data = get_aws_credentials('aws-chiles02')
if boto_data is not None:
work_to_do = WorkToDo(frequency_width, bucket_name, get_s3_split_name(frequency_width))
work_to_do.calculate_work_to_do()
days = work_to_do.work_to_do.keys()
nodes_required, node_count = get_nodes_required(days, days_per_node, spot_price1, spot_price2)
if len(nodes_required) > 0:
uuid = get_uuid()
ec2_data = EC2Controller(
ami_id,
nodes_required,
get_node_manager_user_data(boto_data, uuid),
AWS_REGION,
tags=[
{
'Key': 'Owner',
'Value': getpass.getuser(),
},
{
'Key': 'Name',
'Value': 'Daliuge NM - MsTransform',
},
{
'Key': 'uuid',
'Value': uuid,
}
]
)
ec2_data.start_instances()
reported_running = get_reported_running(
uuid,
node_count,
wait=600
)
hosts = build_hosts(reported_running)
# Create the Data Island Manager
data_island_manager = EC2Controller(
ami_id,
[
{
'number_instances': 1,
'instance_type': 'm4.large',
'spot_price': spot_price1
}
],
get_data_island_manager_user_data(boto_data, hosts, uuid),
AWS_REGION,
tags=[
{
'Key': 'Owner',
'Value': getpass.getuser(),
},
{
'Key': 'Name',
'Value': 'Daliuge DIM - MsTransform',
},
{
'Key': 'uuid',
'Value': uuid,
}
]
)
data_island_manager.start_instances()
data_island_manager_running = get_reported_running(
uuid,
1,
wait=600
)
if len(data_island_manager_running['m4.large']) == 1:
# Now build the graph
session_id = get_session_id()
instance_details = data_island_manager_running['m4.large'][0]
host = instance_details['ip_address']
graph = BuildGraphMsTransform(
work_to_do.work_to_do,
bucket_name,
volume,
7,
reported_running,
add_shutdown,
frequency_width,
session_id,
hosts,
)
graph.build_graph()
graph.tag_all_app_drops({
"session_id": session_id,
})
LOG.info('Connection to {0}:{1}'.format(host, DIM_PORT))
client = DataIslandManagerClient(host, DIM_PORT)
client.create_session(session_id)
client.append_graph(session_id, graph.drop_list)
client.deploy_session(session_id, graph.start_oids)
else:
LOG.error('Unable to find the AWS credentials')
def create_and_generate(bucket_name, input_dir, ami_id, spot_price, volume, nodes, add_shutdown, log_level):
boto_data = get_aws_credentials('aws-chiles02')
if boto_data is not None:
work_to_do = WorkToDo(frequency_width, bucket_name, get_s3_uvsub_name(frequency_width), database_connection)
work_to_do.calculate_work_to_do()
nodes_required, node_count = get_nodes_required(nodes, spot_price)
if len(nodes_required) > 0:
uuid = get_uuid()
ec2_data = EC2Controller(
ami_id,
nodes_required,
get_node_manager_user_data(boto_data, uuid, max_request_size=50, log_level=log_level),
AWS_REGION,
tags=[
{
'Key': 'Owner',
'Value': getpass.getuser(),
},
{
'Key': 'Name',
'Value': 'DALiuGE NM - Stats',
},
{
'Key': 'uuid',
'Value': uuid,
}
]
)
ec2_data.start_instances()
reported_running = get_reported_running(
uuid,
node_count,
wait=600
)
if len(reported_running) == 0:
LOG.error('Nothing has reported ready')
else:
hosts = build_hosts(reported_running)
# Create the Data Island Manager
data_island_manager = EC2Controller(
ami_id,
[
{
'number_instances': 1,
'instance_type': 'm4.large',
'spot_price': spot_price
}
],
get_data_island_manager_user_data(
boto_data,
hosts,
uuid,
need_node_manager=True,
max_request_size=50
),
AWS_REGION,
tags=[
{
'Key': 'Owner',
'Value': getpass.getuser(),
},
{
'Key': 'Name',
'Value': 'DALiuGE DIM - Stats',
},
{
'Key': 'uuid',
'Value': uuid,
}
]
)
data_island_manager.start_instances()
data_island_manager_running = get_reported_running(
uuid,
1,
wait=600,
)
if len(data_island_manager_running['m4.large']) == 1:
# Now build the graph
session_id = get_session_id()
instance_details = data_island_manager_running['m4.large'][0]
host = instance_details['ip_address']
graph = BuildGraphStats(
work_to_do.work_to_do,
bucket_name,
volume,
PARALLEL_STREAMS,
reported_running,
add_shutdown,
frequency_width,
session_id,
map_day_name,
password,
database_ip,
host
)
graph.build_graph()
LOG.info('Connection to {0}:{1}'.format(host, DIM_PORT))
client = DataIslandManagerClient(host, DIM_PORT)
client.create_session(session_id)
client.append_graph(session_id, graph.drop_list)
client.deploy_session(session_id, get_roots(graph.drop_list))
else:
LOG.error('Unable to find the AWS credentials')
def create_and_generate(
bucket_name,
frequency_width,
ami_id,
spot_price,
volume,
frequencies_per_node,
add_shutdown,
iterations,
arcsec,
w_projection_planes,
robust,
image_size,
clean_channel_average,
min_frequency,
max_frequency,
clean_directory_name,
only_image,
log_level):
boto_data = get_aws_credentials('aws-chiles02')
if boto_data is not None:
work_to_do = WorkToDo(
frequency_width,
bucket_name,
get_s3_clean_name(frequency_width, iterations, arcsec) if clean_directory_name is None else clean_directory_name,
min_frequency,
max_frequency
)
work_to_do.calculate_work_to_do()
nodes_required, node_count = get_nodes_required(work_to_do.work_to_do, frequencies_per_node, spot_price)
if len(nodes_required) > 0:
uuid = get_uuid()
ec2_data = EC2Controller(
ami_id,
nodes_required,
get_node_manager_user_data(boto_data, uuid, log_level=log_level),
AWS_REGION,
tags=[
{
'Key': 'Owner',
'Value': getpass.getuser(),
},
{
'Key': 'Name',
'Value': 'DALiuGE NM - Clean',
},
{
'Key': 'uuid',
'Value': uuid,
}
]
)
ec2_data.start_instances()
reported_running = get_reported_running(
uuid,
node_count,
wait=600
)
if len(reported_running) == 0:
LOG.error('Nothing has reported ready')
else:
hosts = build_hosts(reported_running)
# Create the Data Island Manager
data_island_manager = EC2Controller(
ami_id,
[
{
'number_instances': 1,
'instance_type': 'm4.large',
'spot_price': spot_price
}
],
get_data_island_manager_user_data(boto_data, hosts, uuid, need_node_manager=True, log_level=log_level),
AWS_REGION,
tags=[
{
'Key': 'Owner',
'Value': getpass.getuser(),
},
{
'Key': 'Name',
'Value': 'DALiuGE DIM - Clean',
},
{
'Key': 'uuid',
'Value': uuid,
}
]
)
data_island_manager.start_instances()
data_island_manager_running = get_reported_running(
uuid,
1,
wait=600
)
if len(data_island_manager_running['m4.large']) == 1:
# Now build the graph
session_id = get_session_id()
instance_details = data_island_manager_running['m4.large'][0]
host = instance_details['ip_address']
graph = BuildGraphClean(
work_to_do=work_to_do.work_to_do,
bucket_name=bucket_name,
volume=volume,
parallel_streams=PARALLEL_STREAMS,
node_details=reported_running,
shutdown=add_shutdown,
width=frequency_width,
iterations=iterations,
arcsec=arcsec,
w_projection_planes=w_projection_planes,
robust=robust,
image_size=image_size,
clean_channel_average=clean_channel_average,
clean_directory_name=clean_directory_name,
only_image=only_image,
session_id=session_id,
dim_ip=host)
graph.build_graph()
# TODO: Safe the run parameters
LOG.info('Connection to {0}:{1}'.format(host, DIM_PORT))
client = DataIslandManagerClient(host, DIM_PORT)
client.create_session(session_id)
client.append_graph(session_id, graph.drop_list)
client.deploy_session(session_id, get_roots(graph.drop_list))
else:
LOG.error('Unable to find the AWS credentials')
def create_and_generate(bucket_name, frequency_width, ami_id, spot_price, volume, add_shutdown, iterations):
boto_data = get_aws_credentials('aws-chiles02')
if boto_data is not None:
uuid = get_uuid()
ec2_data = EC2Controller(
ami_id,
[
{
'number_instances': 1,
'instance_type': 'i2.4xlarge',
'spot_price': spot_price
}
],
get_node_manager_user_data(boto_data, uuid),
AWS_REGION,
tags=[
{
'Key': 'Owner',
'Value': getpass.getuser(),
},
{
'Key': 'Name',
'Value': 'DALiuGE NM - Concatenate',
},
{
'Key': 'uuid',
'Value': uuid,
}
]
)
ec2_data.start_instances()
reported_running = get_reported_running(
uuid,
1,
wait=600
)
hosts = build_hosts(reported_running)
# Create the Data Island Manager
data_island_manager = EC2Controller(
ami_id,
[
{
'number_instances': 1,
'instance_type': 'm4.large',
'spot_price': spot_price
}
],
get_data_island_manager_user_data(boto_data, hosts, uuid),
AWS_REGION,
tags=[
{
'Key': 'Owner',
'Value': getpass.getuser(),
},
{
'Key': 'Name',
'Value': 'DALiuGE DIM - Concatenate',
},
{
'Key': 'uuid',
'Value': uuid,
},
]
)
data_island_manager.start_instances()
data_island_manager_running = get_reported_running(
uuid,
1,
wait=600
)
if len(data_island_manager_running['m4.large']) == 1:
# Now build the graph
session_id = get_session_id()
instance_details = data_island_manager_running['m4.large'][0]
host = instance_details['ip_address']
graph = BuildGraphConcatenation(
bucket_name,
volume,
PARALLEL_STREAMS,
reported_running,
add_shutdown,
frequency_width,
iterations,
'2arcsec', # TODO: Pass as a parameter
session_id,
host)
graph.build_graph()
LOG.info('Connection to {0}:{1}'.format(host, DIM_PORT))
client = DataIslandManagerClient(host, DIM_PORT)
client.create_session(session_id)
client.append_graph(session_id, graph.drop_list)
client.deploy_session(session_id, get_roots(graph.drop_list))
else:
LOG.error('Unable to find the AWS credentials')
def create_and_generate(bucket_name, frequency_width, ami_id, spot_price, volume, frequencies_per_node, add_shutdown, iterations):
boto_data = get_aws_credentials('aws-chiles02')
if boto_data is not None:
work_to_do = WorkToDo(frequency_width, bucket_name, get_s3_clean_name(frequency_width, iterations))
work_to_do.calculate_work_to_do()
nodes_required, node_count = get_nodes_required(work_to_do.work_to_do, frequencies_per_node, spot_price)
if len(nodes_required) > 0:
uuid = get_uuid()
ec2_data = EC2Controller(
ami_id,
nodes_required,
get_node_manager_user_data(boto_data, uuid),
AWS_REGION,
tags=[
{
'Key': 'Owner',
'Value': getpass.getuser(),
},
{
'Key': 'Name',
'Value': 'DFMS Node - Clean',
},
{
'Key': 'uuid',
'Value': uuid,
}
]
)
ec2_data.start_instances()
reported_running = get_reported_running(
uuid,
node_count,
wait=600
)
if len(reported_running) == 0:
LOG.error('Nothing has reported ready')
else:
hosts = build_hosts(reported_running)
# Create the Data Island Manager
data_island_manager = EC2Controller(
ami_id,
[
{
'number_instances': 1,
'instance_type': 'm4.large',
'spot_price': spot_price
}
],
get_data_island_manager_user_data(boto_data, hosts, uuid),
AWS_REGION,
tags=[
{
'Key': 'Owner',
'Value': getpass.getuser(),
},
{
'Key': 'Name',
'Value': 'Data Island Manager - Clean',
},
{
'Key': 'uuid',
'Value': uuid,
}
]
)
data_island_manager.start_instances()
data_island_manager_running = get_reported_running(
uuid,
1,
wait=600
)
if len(data_island_manager_running['m4.large']) == 1:
# Now build the graph
session_id = get_session_id()
graph = BuildGraphClean(work_to_do.work_to_do, bucket_name, volume, PARALLEL_STREAMS, reported_running, add_shutdown, frequency_width, iterations, session_id)
graph.build_graph()
instance_details = data_island_manager_running['m4.large'][0]
host = instance_details['ip_address']
LOG.info('Connection to {0}:{1}'.format(host, DIM_PORT))
client = DataIslandManagerClient(host, DIM_PORT)
client.create_session(session_id)
client.append_graph(session_id, graph.drop_list)
client.deploy_session(session_id, graph.start_oids)
else:
LOG.error('Unable to find the AWS credentials')