def use_and_generate(host, port, bucket_name, frequency_width, volume,
add_shutdown, iterations):
boto_data = get_aws_credentials('aws-chiles02')
if boto_data is not None:
connection = httplib.HTTPConnection(host, port)
connection.request('GET', '/api', None, {})
response = connection.getresponse()
if response.status != httplib.OK:
msg = 'Error while processing GET request for {0}:{1}/api (status {2}): {3}'.format(
host, port, response.status, response.read())
raise Exception(msg)
json_data = response.read()
message_details = json.loads(json_data)
host_list = message_details['hosts']
nodes_running = get_nodes_running(host_list)
if len(nodes_running) > 0:
# Now build the graph
session_id = get_session_id()
graph = BuildGraphConcatenation(bucket_name, volume,
PARALLEL_STREAMS, nodes_running,
add_shutdown, frequency_width,
iterations, session_id, host)
graph.build_graph()
LOG.info('Connection to {0}:{1}'.format(host, port))
client = DataIslandManagerClient(host, port)
client.create_session(session_id)
client.append_graph(session_id, graph.drop_list)
client.deploy_session(session_id, graph.start_oids)
else:
LOG.warning('No nodes are running')
def monitor_graph(host, port, dump_path):
"""
monitors the execution status of a graph by polling host/port
"""
# use monitorclient to interact with island manager
dc = DataIslandManagerClient(host=host, port=port, timeout=MM_WAIT_TIME)
# We want to monitor the status of the execution
fp = os.path.dirname(dump_path)
if (not os.path.exists(fp)):
return
gfile = "{0}_g.log".format(dump_path)
sfile = "{0}_s.log".format(dump_path)
graph_dict = dict() # k - ssid, v - graph spec json obj
logger.debug("Ready to check sessions")
while True:
for session in dc.sessions(
): #TODO the interval won't work for multiple sessions
stt = time.time()
ssid = session['sessionId']
wgs = {}
wgs['ssid'] = ssid
wgs['gs'] = dc.graph_status(ssid) #TODO check error
time_str = '%.3f' % time.time()
wgs['ts'] = time_str
if (not graph_dict.has_key(ssid)):
graph = dc.graph(ssid)
graph_dict[ssid] = graph
wg = {}
wg['ssid'] = ssid
wg['g'] = graph
# append to file as a line
with open(gfile, 'a') as fg:
json.dump(wg, fg)
fg.write(os.linesep)
# append to file as a line
with open(sfile, 'a') as fs:
json.dump(wgs, fs)
fs.write(os.linesep)
dt = time.time() - stt
if (dt < GRAPH_MONITOR_INTERVAL):
time.sleep(GRAPH_MONITOR_INTERVAL - dt)
def __init__(self, mhost, mport, timeout=10, sch_algo="sarkar", output=None):
self._host = mhost
self._port = mport
self._rc = RestClient(mhost, mport, timeout)
self._dc = DataIslandManagerClient(mhost, mport)
self._sch_algo = sch_algo
self._output = output
def use_and_generate(host, port, bucket_name, frequency_width, volume, add_shutdown, iterations):
boto_data = get_aws_credentials('aws-chiles02')
if boto_data is not None:
connection = httplib.HTTPConnection(host, port)
connection.request('GET', '/api', None, {})
response = connection.getresponse()
if response.status != httplib.OK:
msg = 'Error while processing GET request for {0}:{1}/api (status {2}): {3}'.format(host, port, response.status, response.read())
raise Exception(msg)
json_data = response.read()
message_details = json.loads(json_data)
host_list = message_details['hosts']
nodes_running = get_nodes_running(host_list)
if len(nodes_running) > 0:
# Now build the graph
session_id = get_session_id()
graph = BuildGraphJpeg2000(bucket_name, volume, PARALLEL_STREAMS, nodes_running, add_shutdown, frequency_width, iterations, session_id, host)
graph.build_graph()
LOG.info('Connection to {0}:{1}'.format(host, port))
client = DataIslandManagerClient(host, port)
client.create_session(session_id)
client.append_graph(session_id, graph.drop_list)
client.deploy_session(session_id, graph.start_oids)
else:
LOG.warning('No nodes are running')
class MonitorClient(object):
def __init__(self, mhost, mport, timeout=10, sch_algo="sarkar", output=None):
self._host = mhost
self._port = mport
self._rc = RestClient(mhost, mport, timeout)
self._dc = DataIslandManagerClient(mhost, mport)
self._sch_algo = sch_algo
self._output = output
def get_avail_hosts(self):
return self._dc.nodes()
"""
ret = self._rc._request('/api', 'get')
ret_dict = json.loads(ret)
return ret_dict['hosts']
"""
def submit_single_graph(self, graph_id, algo="sarkar", deploy=False):
lgn = lgnames[graph_id]
fp = pkg_resources.resource_filename("dfms.dropmake", "web/{0}".format(lgn))
lg = LG(fp)
drop_list = lg.unroll_to_tpl()
# node_list = self.get_avail_hosts()
node_list = self._dc.nodes()
pgtp = MySarkarPGTP(drop_list, len(node_list), merge_parts=True)
pgtp.json
pg_spec = pgtp.to_pg_spec(node_list, ret_str=False)
if self._output:
with open(self._output, "w") as f:
json.dump(pg_spec, f, indent=2)
completed_uids = [x["oid"] for x in droputils.get_roots(pg_spec)]
ssid = "{0}-{1}".format(lgn.split(".")[0], lg._session_id)
self._dc.create_session(ssid)
print "session created"
self._dc.append_graph(ssid, pg_spec)
print "graph appended"
if deploy:
ret = self._dc.deploy_session(ssid, completed_uids=completed_uids)
print "session deployed"
return ret
def produce_physical_graphs(self, graph_id, algo="sarkar", tgt="/tmp"):
lgn = lgnames[graph_id]
fp = pkg_resources.resource_filename("dfms.dropmake", "web/{0}".format(lgn))
lg = LG(fp)
drop_list = lg.unroll_to_tpl()
node_list = self._dc.nodes()
# node_list = ['10.128.0.11', '10.128.0.14', '10.128.0.15', '10.128.0.16']
pgtp = MySarkarPGTP(drop_list, len(node_list), merge_parts=True)
pgtp.json
pg_spec = pgtp.to_pg_spec(node_list)
with open("/{1}/sar_{0}_pgspec.json".format(lgn.split(".")[0], tgt), "w") as f:
f.write(pg_spec)
def use_and_generate(host, port, bucket_name, frequency_width, volume,
add_shutdown, password):
boto_data = get_aws_credentials('aws-chiles02')
if boto_data is not None:
database_connection, map_day_name, database_ip = setup_database(
password, bucket_name)
connection = httplib.HTTPConnection(host, port)
connection.request('GET', '/api', None, {})
response = connection.getresponse()
if response.status != httplib.OK:
msg = 'Error while processing GET request for {0}:{1}/api (status {2}): {3}'.format(
host, port, response.status, response.read())
raise Exception(msg)
json_data = response.read()
message_details = json.loads(json_data)
host_list = message_details['hosts']
nodes_running = get_nodes_running(host_list)
if len(nodes_running) > 0:
work_to_do = WorkToDo(frequency_width, bucket_name,
get_s3_uvsub_name(frequency_width),
database_connection)
work_to_do.calculate_work_to_do()
# Now build the graph
session_id = get_session_id()
graph = BuildGraphStats(work_to_do.work_to_do, bucket_name, volume,
PARALLEL_STREAMS, nodes_running,
add_shutdown, frequency_width, session_id,
map_day_name, password, database_ip, host)
graph.build_graph()
LOG.info('Connection to {0}:{1}'.format(host, port))
client = DataIslandManagerClient(host, port)
client.create_session(session_id)
client.append_graph(session_id, graph.drop_list)
client.deploy_session(session_id, graph.start_oids)
else:
LOG.warning('No nodes are running')
def use_and_generate(host, port, bucket_name, frequency_width, volume, add_shutdown, password):
boto_data = get_aws_credentials('aws-chiles02')
if boto_data is not None:
database_connection, map_day_name, database_ip = setup_database(password, bucket_name)
connection = httplib.HTTPConnection(host, port)
connection.request('GET', '/api', None, {})
response = connection.getresponse()
if response.status != httplib.OK:
msg = 'Error while processing GET request for {0}:{1}/api (status {2}): {3}'.format(host, port, response.status, response.read())
raise Exception(msg)
json_data = response.read()
message_details = json.loads(json_data)
host_list = message_details['hosts']
nodes_running = get_nodes_running(host_list)
if len(nodes_running) > 0:
work_to_do = WorkToDo(frequency_width, bucket_name, get_s3_uvsub_name(frequency_width), database_connection)
work_to_do.calculate_work_to_do()
# Now build the graph
session_id = get_session_id()
graph = BuildGraphStats(
work_to_do.work_to_do,
bucket_name,
volume,
PARALLEL_STREAMS,
nodes_running,
add_shutdown,
frequency_width,
session_id,
map_day_name,
password,
database_ip,
host
)
graph.build_graph()
LOG.info('Connection to {0}:{1}'.format(host, port))
client = DataIslandManagerClient(host, port)
client.create_session(session_id)
client.append_graph(session_id, graph.drop_list)
client.deploy_session(session_id, graph.start_oids)
else:
LOG.warning('No nodes are running')
def use_and_generate(host, port, bucket_name, frequency_width, volume, add_shutdown):
boto_data = get_aws_credentials('aws-chiles02')
if boto_data is not None:
connection = httplib.HTTPConnection(host, port)
connection.request('GET', '/api', None, {})
response = connection.getresponse()
if response.status != httplib.OK:
msg = 'Error while processing GET request for {0}:{1}/api (status {2}): {3}'.format(host, port, response.status, response.read())
raise Exception(msg)
json_data = response.read()
message_details = json.loads(json_data)
host_list = message_details['hosts']
nodes_running = get_nodes_running(host_list)
if len(nodes_running) > 0:
work_to_do = WorkToDo(
width=frequency_width,
bucket_name=bucket_name,
s3_split_name=get_s3_split_name(frequency_width))
work_to_do.calculate_work_to_do()
# Now build the graph
session_id = get_session_id()
graph = BuildGraphMsTransform(
work_to_do=work_to_do.work_to_do,
bucket_name=bucket_name,
volume=volume,
parallel_streams=7,
node_details=nodes_running,
shutdown=add_shutdown,
width=frequency_width,
session_id=session_id,
dim_ip=host,
)
graph.build_graph()
LOG.info('Connection to {0}:{1}'.format(host, port))
client = DataIslandManagerClient(host, port)
client.create_session(session_id)
client.append_graph(session_id, graph.drop_list)
client.deploy_session(session_id, get_roots(graph.drop_list))
else:
LOG.warning('No nodes are running')
def test_recursive(self):
sid = 'lala'
c = DataIslandManagerClient(hostname)
c.createSession(sid)
# invalid dropspec, app doesn't exist
# This is not checked at the DIM level but only at the NM level
# The exception should still pass through though
with self.assertRaises(exceptions.SubManagerException) as cm:
c.addGraphSpec(sid, [{
'oid': 'a',
'type': 'app',
'app': 'doesnt.exist',
'node': hostname
}])
ex = cm.exception
self.assertTrue(hostname in ex.args[0])
self.assertTrue(isinstance(ex.args[0][hostname],
InvalidGraphException))
def create_and_generate(bucket_name, input_dir, ami_id, spot_price, volume, nodes, add_shutdown, log_level):
boto_data = get_aws_credentials('aws-chiles02')
if boto_data is not None:
work_to_do = WorkToDo(frequency_width, bucket_name, get_s3_uvsub_name(frequency_width), database_connection)
work_to_do.calculate_work_to_do()
nodes_required, node_count = get_nodes_required(nodes, spot_price)
if len(nodes_required) > 0:
uuid = get_uuid()
ec2_data = EC2Controller(
ami_id,
nodes_required,
get_node_manager_user_data(boto_data, uuid, max_request_size=50, log_level=log_level),
AWS_REGION,
tags=[
{
'Key': 'Owner',
'Value': getpass.getuser(),
},
{
'Key': 'Name',
'Value': 'DALiuGE NM - Stats',
},
{
'Key': 'uuid',
'Value': uuid,
}
]
)
ec2_data.start_instances()
reported_running = get_reported_running(
uuid,
node_count,
wait=600
)
if len(reported_running) == 0:
LOG.error('Nothing has reported ready')
else:
hosts = build_hosts(reported_running)
# Create the Data Island Manager
data_island_manager = EC2Controller(
ami_id,
[
{
'number_instances': 1,
'instance_type': 'm4.large',
'spot_price': spot_price
}
],
get_data_island_manager_user_data(
boto_data,
hosts,
uuid,
need_node_manager=True,
max_request_size=50
),
AWS_REGION,
tags=[
{
'Key': 'Owner',
'Value': getpass.getuser(),
},
{
'Key': 'Name',
'Value': 'DALiuGE DIM - Stats',
},
{
'Key': 'uuid',
'Value': uuid,
}
]
)
data_island_manager.start_instances()
data_island_manager_running = get_reported_running(
uuid,
1,
wait=600,
)
if len(data_island_manager_running['m4.large']) == 1:
# Now build the graph
session_id = get_session_id()
instance_details = data_island_manager_running['m4.large'][0]
host = instance_details['ip_address']
graph = BuildGraphStats(
work_to_do.work_to_do,
bucket_name,
volume,
PARALLEL_STREAMS,
reported_running,
add_shutdown,
frequency_width,
session_id,
map_day_name,
password,
database_ip,
host
)
graph.build_graph()
LOG.info('Connection to {0}:{1}'.format(host, DIM_PORT))
client = DataIslandManagerClient(host, DIM_PORT)
client.create_session(session_id)
client.append_graph(session_id, graph.drop_list)
client.deploy_session(session_id, get_roots(graph.drop_list))
else:
LOG.error('Unable to find the AWS credentials')
def create_and_generate(bucket_name, frequency_width, ami_id, spot_price,
volume, nodes, add_shutdown, password, log_level):
boto_data = get_aws_credentials('aws-chiles02')
if boto_data is not None:
database_connection, map_day_name, database_ip = setup_database(
password, bucket_name)
work_to_do = WorkToDo(frequency_width, bucket_name,
get_s3_uvsub_name(frequency_width),
database_connection)
work_to_do.calculate_work_to_do()
nodes_required, node_count = get_nodes_required(nodes, spot_price)
if len(nodes_required) > 0:
uuid = get_uuid()
ec2_data = EC2Controller(ami_id,
nodes_required,
get_node_manager_user_data(
boto_data,
uuid,
max_request_size=50,
log_level=log_level),
AWS_REGION,
tags=[{
'Key': 'Owner',
'Value': getpass.getuser(),
}, {
'Key': 'Name',
'Value': 'Daliuge NM - Stats',
}, {
'Key': 'uuid',
'Value': uuid,
}])
ec2_data.start_instances()
reported_running = get_reported_running(uuid, node_count, wait=600)
if len(reported_running) == 0:
LOG.error('Nothing has reported ready')
else:
hosts = build_hosts(reported_running)
# Create the Data Island Manager
data_island_manager = EC2Controller(
ami_id, [{
'number_instances': 1,
'instance_type': 'm4.large',
'spot_price': spot_price
}],
get_data_island_manager_user_data(boto_data,
hosts,
uuid,
need_node_manager=True,
max_request_size=50),
AWS_REGION,
tags=[{
'Key': 'Owner',
'Value': getpass.getuser(),
}, {
'Key': 'Name',
'Value': 'Daliuge DIM - Stats',
}, {
'Key': 'uuid',
'Value': uuid,
}])
data_island_manager.start_instances()
data_island_manager_running = get_reported_running(
uuid,
1,
wait=600,
)
if len(data_island_manager_running['m4.large']) == 1:
# Now build the graph
session_id = get_session_id()
instance_details = data_island_manager_running['m4.large'][
0]
host = instance_details['ip_address']
graph = BuildGraphStats(work_to_do.work_to_do, bucket_name,
volume, PARALLEL_STREAMS,
reported_running, add_shutdown,
frequency_width, session_id,
map_day_name, password,
database_ip, host)
graph.build_graph()
LOG.info('Connection to {0}:{1}'.format(host, DIM_PORT))
client = DataIslandManagerClient(host, DIM_PORT)
client.create_session(session_id)
client.append_graph(session_id, graph.drop_list)
client.deploy_session(session_id, graph.start_oids)
else:
LOG.error('Unable to find the AWS credentials')
def submit_monitor_graph(dim_ip,
graph_id,
dump_status,
zerorun,
app,
mc=None,
unrolled=None):
"""
Submits a graph and then monitors the island manager
"""
logger.debug("dump_status = {0}".format(dump_status))
logger.info("Wait for {0} seconds before submitting graph to DIM".format(
GRAPH_SUBMIT_WAIT_TIME))
time.sleep(GRAPH_SUBMIT_WAIT_TIME)
# use monitorclient to interact with island manager
if (graph_id is not None):
if (mc is None):
mc = MonitorClient('localhost',
ISLAND_DEFAULT_REST_PORT,
algo='metis',
zerorun=zerorun,
app=app)
dc = mc._dc
mc._nodes = [dim_ip] + dc.nodes()
logger.info("Submitting graph {0}".format(graph_id))
lgn, lg, pg_spec = mc.get_physical_graph(graph_id, unrolled=unrolled)
if (unrolled is not None):
del unrolled
mc.submit_single_graph(graph_id, deploy=True, pg=(lgn, lg, pg_spec))
logger.info("graph {0} is successfully submitted".format(graph_id))
else:
dc = DataIslandManagerClient('localhost')
if (dump_status is not None):
fp = os.path.dirname(dump_status)
if (not os.path.exists(fp)):
return
gfile = "{0}_g.log".format(dump_status)
sfile = "{0}_s.log".format(dump_status)
graph_dict = dict() # k - ssid, v - graph spec json obj
logger.debug("Ready to check sessions")
while (True):
#logger.debug("checking sessions")
sessions = dc.sessions()
#logger.debug("len(sessions) = {0}".format(len(sessions)))
#logger.debug("session0 = {0}".format(sessions[0]))
for session in sessions: #TODO the interval won't work for multiple sessions
stt = time.time()
ssid = session['sessionId']
#logger.debug("session id = {0}".format(ssid))
wgs = {}
wgs['ssid'] = ssid
wgs['gs'] = dc.graph_status(ssid) #TODO check error
time_str = '%.3f' % time.time()
wgs['ts'] = time_str
if (not graph_dict.has_key(ssid)):
graph = dc.graph(ssid)
graph_dict[ssid] = graph
wg = {}
wg['ssid'] = ssid
wg['g'] = graph
# append to file as a line
with open(gfile, 'a') as fg:
json.dump(wg, fg)
fg.write(os.linesep)
# append to file as a line
with open(sfile, 'a') as fs:
json.dump(wgs, fs)
fs.write(os.linesep)
dt = time.time() - stt
if (dt < GRAPH_MONITOR_INTERVAL):
try:
time.sleep(GRAPH_MONITOR_INTERVAL - dt)
except:
pass
def use_and_generate(
host,
port,
bucket_name,
frequency_width,
w_projection_planes,
volume,
add_shutdown,
min_frequency,
max_frequency,
scan_statistics,
dump_json):
boto_data = get_aws_credentials('aws-chiles02')
if boto_data is not None:
connection = httplib.HTTPConnection(host, port)
connection.request('GET', '/api', None, {})
response = connection.getresponse()
if response.status != httplib.OK:
msg = 'Error while processing GET request for {0}:{1}/api (status {2}): {3}'.format(host, port, response.status, response.read())
raise Exception(msg)
json_data = response.read()
message_details = json.loads(json_data)
host_list = message_details['hosts']
nodes_running = get_nodes_running(host_list)
if len(nodes_running) > 0:
work_to_do = WorkToDo(
width=frequency_width,
bucket_name=bucket_name,
s3_uvsub_name=get_s3_uvsub_name(frequency_width),
s3_split_name=get_s3_split_name(frequency_width),
min_frequency=min_frequency,
max_frequency=max_frequency)
work_to_do.calculate_work_to_do()
# Now build the graph
session_id = get_session_id()
graph = BuildGraphUvsub(
work_to_do=work_to_do.work_to_do,
bucket_name=bucket_name,
volume=volume,
parallel_streams=PARALLEL_STREAMS,
node_details=nodes_running,
shutdown=add_shutdown,
scan_statistics=scan_statistics,
width=frequency_width,
w_projection_planes=w_projection_planes,
session_id=session_id,
dim_ip=host)
graph.build_graph()
if dump_json:
json_dumps = json.dumps(graph.drop_list, indent=2)
with open("/tmp/json_uvsub.txt", "w") as json_file:
json_file.write(json_dumps)
LOG.info('Connection to {0}:{1}'.format(host, port))
client = DataIslandManagerClient(host, port)
client.create_session(session_id)
client.append_graph(session_id, graph.drop_list)
client.deploy_session(session_id, get_roots(graph.drop_list))
else:
LOG.warning('No nodes are running')
interpolation = 'linear',
start = '1408MHz',
width = '1412kHz',
veltype = 'radio',
spw = '',
combinespws = True,
nspw = 1,
createmms = False,
datacolumn = 'data'
)
sp.addInput(vis_in)
sp.addOutput(split_out)
cl.addInput(split_out)
droplist.append(vis_in)
droplist.append(split_out)
droplist.append(sp)
c = DataIslandManagerClient()
c.create_session(sessionId)
c.append_graph(sessionId, droplist)
c.deploy_session(sessionId)
except Exception as e:
import traceback
traceback.print_exc()
print str(e)
sys.exit(-1)
sys.exit(0)
outframe = 'lsrk',
interpolation = 'linear',
start = '1408MHz',
width = '1412kHz',
veltype = 'radio',
spw = '',
combinespws = True,
nspw = 1,
createmms = False,
datacolumn = 'data'
)
sp.addInput(vis_in)
sp.addOutput(split_out)
cl.addInput(split_out)
droplist.append(vis_in)
droplist.append(split_out)
droplist.append(sp)
c = DataIslandManagerClient()
c.create_session(sessionId)
c.append_graph(sessionId, droplist)
c.deploy_session(sessionId)
except Exception as e:
import traceback
traceback.print_exc()
sys.exit(-1)
sys.exit(0)
def use_and_generate(
host,
port,
bucket_name,
frequency_width,
volume,
add_shutdown,
iterations,
arcsec,
w_projection_planes,
robust,
image_size,
clean_channel_average,
min_frequency,
max_frequency,
clean_directory_name,
only_image):
boto_data = get_aws_credentials('aws-chiles02')
if boto_data is not None:
connection = httplib.HTTPConnection(host, port)
connection.request('GET', '/api', None, {})
response = connection.getresponse()
if response.status != httplib.OK:
msg = 'Error while processing GET request for {0}:{1}/api (status {2}): {3}'.format(host, port, response.status, response.read())
raise Exception(msg)
json_data = response.read()
message_details = json.loads(json_data)
host_list = message_details['hosts']
nodes_running = get_nodes_running(host_list)
if len(nodes_running) > 0:
work_to_do = WorkToDo(
frequency_width,
bucket_name,
get_s3_clean_name(frequency_width, iterations, arcsec) if clean_directory_name is None else clean_directory_name,
min_frequency,
max_frequency
)
work_to_do.calculate_work_to_do()
# Now build the graph
session_id = get_session_id()
graph = BuildGraphClean(
work_to_do=work_to_do.work_to_do,
bucket_name=bucket_name,
volume=volume,
parallel_streams=PARALLEL_STREAMS,
node_details=nodes_running,
shutdown=add_shutdown,
width=frequency_width,
iterations=iterations,
arcsec=arcsec,
w_projection_planes=w_projection_planes,
robust=robust,
image_size=image_size,
clean_channel_average=clean_channel_average,
clean_directory_name=clean_directory_name,
only_image=only_image,
session_id=session_id,
dim_ip=host)
graph.build_graph()
# TODO: Save the run parameters
LOG.info('Connection to {0}:{1}'.format(host, port))
client = DataIslandManagerClient(host, port)
client.create_session(session_id)
client.append_graph(session_id, graph.drop_list)
client.deploy_session(session_id, get_roots(graph.drop_list))
else:
LOG.warning('No nodes are running')
def create_and_generate(
bucket_name,
frequency_width,
ami_id,
spot_price,
volume,
frequencies_per_node,
add_shutdown,
iterations,
arcsec,
w_projection_planes,
robust,
image_size,
clean_channel_average,
min_frequency,
max_frequency,
clean_directory_name,
only_image,
log_level):
boto_data = get_aws_credentials('aws-chiles02')
if boto_data is not None:
work_to_do = WorkToDo(
frequency_width,
bucket_name,
get_s3_clean_name(frequency_width, iterations, arcsec) if clean_directory_name is None else clean_directory_name,
min_frequency,
max_frequency
)
work_to_do.calculate_work_to_do()
nodes_required, node_count = get_nodes_required(work_to_do.work_to_do, frequencies_per_node, spot_price)
if len(nodes_required) > 0:
uuid = get_uuid()
ec2_data = EC2Controller(
ami_id,
nodes_required,
get_node_manager_user_data(boto_data, uuid, log_level=log_level),
AWS_REGION,
tags=[
{
'Key': 'Owner',
'Value': getpass.getuser(),
},
{
'Key': 'Name',
'Value': 'DALiuGE NM - Clean',
},
{
'Key': 'uuid',
'Value': uuid,
}
]
)
ec2_data.start_instances()
reported_running = get_reported_running(
uuid,
node_count,
wait=600
)
if len(reported_running) == 0:
LOG.error('Nothing has reported ready')
else:
hosts = build_hosts(reported_running)
# Create the Data Island Manager
data_island_manager = EC2Controller(
ami_id,
[
{
'number_instances': 1,
'instance_type': 'm4.large',
'spot_price': spot_price
}
],
get_data_island_manager_user_data(boto_data, hosts, uuid, need_node_manager=True, log_level=log_level),
AWS_REGION,
tags=[
{
'Key': 'Owner',
'Value': getpass.getuser(),
},
{
'Key': 'Name',
'Value': 'DALiuGE DIM - Clean',
},
{
'Key': 'uuid',
'Value': uuid,
}
]
)
data_island_manager.start_instances()
data_island_manager_running = get_reported_running(
uuid,
1,
wait=600
)
if len(data_island_manager_running['m4.large']) == 1:
# Now build the graph
session_id = get_session_id()
instance_details = data_island_manager_running['m4.large'][0]
host = instance_details['ip_address']
graph = BuildGraphClean(
work_to_do=work_to_do.work_to_do,
bucket_name=bucket_name,
volume=volume,
parallel_streams=PARALLEL_STREAMS,
node_details=reported_running,
shutdown=add_shutdown,
width=frequency_width,
iterations=iterations,
arcsec=arcsec,
w_projection_planes=w_projection_planes,
robust=robust,
image_size=image_size,
clean_channel_average=clean_channel_average,
clean_directory_name=clean_directory_name,
only_image=only_image,
session_id=session_id,
dim_ip=host)
graph.build_graph()
# TODO: Safe the run parameters
LOG.info('Connection to {0}:{1}'.format(host, DIM_PORT))
client = DataIslandManagerClient(host, DIM_PORT)
client.create_session(session_id)
client.append_graph(session_id, graph.drop_list)
client.deploy_session(session_id, get_roots(graph.drop_list))
else:
LOG.error('Unable to find the AWS credentials')
def create_and_generate(bucket_name, frequency_width, ami_id, spot_price, volume, add_shutdown, iterations):
boto_data = get_aws_credentials('aws-chiles02')
if boto_data is not None:
uuid = get_uuid()
ec2_data = EC2Controller(
ami_id,
[
{
'number_instances': 1,
'instance_type': 'i2.4xlarge',
'spot_price': spot_price
}
],
get_node_manager_user_data(boto_data, uuid),
AWS_REGION,
tags=[
{
'Key': 'Owner',
'Value': getpass.getuser(),
},
{
'Key': 'Name',
'Value': 'DALiuGE NM - Concatenate',
},
{
'Key': 'uuid',
'Value': uuid,
}
]
)
ec2_data.start_instances()
reported_running = get_reported_running(
uuid,
1,
wait=600
)
hosts = build_hosts(reported_running)
# Create the Data Island Manager
data_island_manager = EC2Controller(
ami_id,
[
{
'number_instances': 1,
'instance_type': 'm4.large',
'spot_price': spot_price
}
],
get_data_island_manager_user_data(boto_data, hosts, uuid),
AWS_REGION,
tags=[
{
'Key': 'Owner',
'Value': getpass.getuser(),
},
{
'Key': 'Name',
'Value': 'DALiuGE DIM - Concatenate',
},
{
'Key': 'uuid',
'Value': uuid,
},
]
)
data_island_manager.start_instances()
data_island_manager_running = get_reported_running(
uuid,
1,
wait=600
)
if len(data_island_manager_running['m4.large']) == 1:
# Now build the graph
session_id = get_session_id()
instance_details = data_island_manager_running['m4.large'][0]
host = instance_details['ip_address']
graph = BuildGraphConcatenation(
bucket_name,
volume,
PARALLEL_STREAMS,
reported_running,
add_shutdown,
frequency_width,
iterations,
'2arcsec', # TODO: Pass as a parameter
session_id,
host)
graph.build_graph()
LOG.info('Connection to {0}:{1}'.format(host, DIM_PORT))
client = DataIslandManagerClient(host, DIM_PORT)
client.create_session(session_id)
client.append_graph(session_id, graph.drop_list)
client.deploy_session(session_id, get_roots(graph.drop_list))
else:
LOG.error('Unable to find the AWS credentials')
def create_and_generate(
bucket_name,
frequency_width,
w_projection_planes,
ami_id,
spot_price,
volume,
nodes,
add_shutdown,
min_frequency,
max_frequency,
scan_statistics,
dump_json):
boto_data = get_aws_credentials('aws-chiles02')
if boto_data is not None:
work_to_do = WorkToDo(
width=frequency_width,
bucket_name=bucket_name,
s3_uvsub_name=get_s3_uvsub_name(frequency_width),
s3_split_name=get_s3_split_name(frequency_width),
min_frequency=min_frequency,
max_frequency=max_frequency)
work_to_do.calculate_work_to_do()
nodes_required, node_count = get_nodes_required(nodes, spot_price)
if len(nodes_required) > 0:
uuid = get_uuid()
ec2_data = EC2Controller(
ami_id,
nodes_required,
get_node_manager_user_data(boto_data, uuid, max_request_size=50),
AWS_REGION,
tags=[
{
'Key': 'Owner',
'Value': getpass.getuser(),
},
{
'Key': 'Name',
'Value': 'DALiuGE NM - Uvsub',
},
{
'Key': 'uuid',
'Value': uuid,
}
]
)
ec2_data.start_instances()
reported_running = get_reported_running(
uuid,
node_count,
wait=600
)
if len(reported_running) == 0:
LOG.error('Nothing has reported ready')
else:
hosts = build_hosts(reported_running)
# Create the Data Island Manager
data_island_manager = EC2Controller(
ami_id,
[
{
'number_instances': 1,
'instance_type': 'm4.large',
'spot_price': spot_price
}
],
get_data_island_manager_user_data(boto_data, hosts, uuid, max_request_size=50),
AWS_REGION,
tags=[
{
'Key': 'Owner',
'Value': getpass.getuser(),
},
{
'Key': 'Name',
'Value': 'DALiuGE DIM - Uvsub',
},
{
'Key': 'uuid',
'Value': uuid,
}
]
)
data_island_manager.start_instances()
data_island_manager_running = get_reported_running(
uuid,
1,
wait=600
)
if len(data_island_manager_running['m4.large']) == 1:
# Now build the graph
session_id = get_session_id()
instance_details = data_island_manager_running['m4.large'][0]
host = instance_details['ip_address']
graph = BuildGraphUvsub(
work_to_do=work_to_do.work_to_do,
bucket_name=bucket_name,
volume=volume,
parallel_streams=PARALLEL_STREAMS,
node_details=reported_running,
shutdown=add_shutdown,
scan_statistics=scan_statistics,
width=frequency_width,
w_projection_planes=w_projection_planes,
session_id=session_id,
dim_ip=host)
graph.build_graph()
if dump_json:
json_dumps = json.dumps(graph.drop_list, indent=2)
with open("/tmp/json_uvsub.txt", "w") as json_file:
json_file.write(json_dumps)
LOG.info('Connection to {0}:{1}'.format(host, DIM_PORT))
client = DataIslandManagerClient(host, DIM_PORT)
client.create_session(session_id)
client.append_graph(session_id, graph.drop_list)
client.deploy_session(session_id, get_roots(graph.drop_list))
else:
LOG.error('Unable to find the AWS credentials')
def create_and_generate(bucket_name, frequency_width, ami_id, spot_price1, spot_price2, volume, days_per_node, add_shutdown):
boto_data = get_aws_credentials('aws-chiles02')
if boto_data is not None:
work_to_do = WorkToDo(frequency_width, bucket_name, get_s3_split_name(frequency_width))
work_to_do.calculate_work_to_do()
days = work_to_do.work_to_do.keys()
nodes_required, node_count = get_nodes_required(days, days_per_node, spot_price1, spot_price2)
if len(nodes_required) > 0:
uuid = get_uuid()
ec2_data = EC2Controller(
ami_id,
nodes_required,
get_node_manager_user_data(boto_data, uuid),
AWS_REGION,
tags=[
{
'Key': 'Owner',
'Value': getpass.getuser(),
},
{
'Key': 'Name',
'Value': 'Daliuge NM - MsTransform',
},
{
'Key': 'uuid',
'Value': uuid,
}
]
)
ec2_data.start_instances()
reported_running = get_reported_running(
uuid,
node_count,
wait=600
)
hosts = build_hosts(reported_running)
# Create the Data Island Manager
data_island_manager = EC2Controller(
ami_id,
[
{
'number_instances': 1,
'instance_type': 'm4.large',
'spot_price': spot_price1
}
],
get_data_island_manager_user_data(boto_data, hosts, uuid),
AWS_REGION,
tags=[
{
'Key': 'Owner',
'Value': getpass.getuser(),
},
{
'Key': 'Name',
'Value': 'Daliuge DIM - MsTransform',
},
{
'Key': 'uuid',
'Value': uuid,
}
]
)
data_island_manager.start_instances()
data_island_manager_running = get_reported_running(
uuid,
1,
wait=600
)
if len(data_island_manager_running['m4.large']) == 1:
# Now build the graph
session_id = get_session_id()
instance_details = data_island_manager_running['m4.large'][0]
host = instance_details['ip_address']
graph = BuildGraphMsTransform(
work_to_do.work_to_do,
bucket_name,
volume,
7,
reported_running,
add_shutdown,
frequency_width,
session_id,
hosts,
)
graph.build_graph()
graph.tag_all_app_drops({
"session_id": session_id,
})
LOG.info('Connection to {0}:{1}'.format(host, DIM_PORT))
client = DataIslandManagerClient(host, DIM_PORT)
client.create_session(session_id)
client.append_graph(session_id, graph.drop_list)
client.deploy_session(session_id, graph.start_oids)
else:
LOG.error('Unable to find the AWS credentials')
def create_and_generate(bucket_name, frequency_width, ami_id, spot_price, volume, bottom_frequency, nodes, add_shutdown):
boto_data = get_aws_credentials('aws-chiles02')
if boto_data is not None:
uuid = get_uuid()
ec2_data = EC2Controller(
ami_id,
[
{
'number_instances': nodes,
'instance_type': 'i2.2xlarge',
'spot_price': spot_price
}
],
get_node_manager_user_data(boto_data, uuid),
AWS_REGION,
tags=[
{
'Key': 'Owner',
'Value': getpass.getuser(),
},
{
'Key': 'Name',
'Value': 'Daliuge NM - Find bad MS',
},
{
'Key': 'uuid',
'Value': uuid,
}
]
)
ec2_data.start_instances()
reported_running = get_reported_running(
uuid,
nodes,
wait=600
)
if len(reported_running) == 0:
LOG.error('Nothing has reported ready')
else:
hosts = build_hosts(reported_running)
# Create the Data Island Manager
data_island_manager = EC2Controller(
ami_id,
[
{
'number_instances': 1,
'instance_type': 'm4.large',
'spot_price': spot_price
}
],
get_data_island_manager_user_data(boto_data, hosts, uuid),
AWS_REGION,
tags=[
{
'Key': 'Owner',
'Value': getpass.getuser(),
},
{
'Key': 'Name',
'Value': 'Daliuge DIM - Find bad MS',
},
{
'Key': 'uuid',
'Value': uuid,
}
]
)
data_island_manager.start_instances()
data_island_manager_running = get_reported_running(
uuid,
1,
wait=600
)
if len(data_island_manager_running['m4.large']) == 1:
# Now build the graph
session_id = get_session_id()
instance_details = data_island_manager_running['m4.large'][0]
host = instance_details['ip_address']
graph = BuildGraphFindBadMeasurementSet(
bucket_name,
volume,
PARALLEL_STREAMS,
reported_running,
add_shutdown,
frequency_width,
bottom_frequency,
session_id,
host,
)
graph.build_graph()
LOG.info('Connection to {0}:{1}'.format(host, DIM_PORT))
client = DataIslandManagerClient(host, DIM_PORT)
client.create_session(session_id)
client.append_graph(session_id, graph.drop_list)
client.deploy_session(session_id, graph.start_oids)
else:
LOG.error('Unable to find the AWS credentials')
def create_and_generate(bucket_name, frequency_width, ami_id, spot_price, volume, frequencies_per_node, add_shutdown, iterations):
boto_data = get_aws_credentials('aws-chiles02')
if boto_data is not None:
work_to_do = WorkToDo(frequency_width, bucket_name, get_s3_clean_name(frequency_width, iterations))
work_to_do.calculate_work_to_do()
nodes_required, node_count = get_nodes_required(work_to_do.work_to_do, frequencies_per_node, spot_price)
if len(nodes_required) > 0:
uuid = get_uuid()
ec2_data = EC2Controller(
ami_id,
nodes_required,
get_node_manager_user_data(boto_data, uuid),
AWS_REGION,
tags=[
{
'Key': 'Owner',
'Value': getpass.getuser(),
},
{
'Key': 'Name',
'Value': 'DFMS Node - Clean',
},
{
'Key': 'uuid',
'Value': uuid,
}
]
)
ec2_data.start_instances()
reported_running = get_reported_running(
uuid,
node_count,
wait=600
)
if len(reported_running) == 0:
LOG.error('Nothing has reported ready')
else:
hosts = build_hosts(reported_running)
# Create the Data Island Manager
data_island_manager = EC2Controller(
ami_id,
[
{
'number_instances': 1,
'instance_type': 'm4.large',
'spot_price': spot_price
}
],
get_data_island_manager_user_data(boto_data, hosts, uuid),
AWS_REGION,
tags=[
{
'Key': 'Owner',
'Value': getpass.getuser(),
},
{
'Key': 'Name',
'Value': 'Data Island Manager - Clean',
},
{
'Key': 'uuid',
'Value': uuid,
}
]
)
data_island_manager.start_instances()
data_island_manager_running = get_reported_running(
uuid,
1,
wait=600
)
if len(data_island_manager_running['m4.large']) == 1:
# Now build the graph
session_id = get_session_id()
graph = BuildGraphClean(work_to_do.work_to_do, bucket_name, volume, PARALLEL_STREAMS, reported_running, add_shutdown, frequency_width, iterations, session_id)
graph.build_graph()
instance_details = data_island_manager_running['m4.large'][0]
host = instance_details['ip_address']
LOG.info('Connection to {0}:{1}'.format(host, DIM_PORT))
client = DataIslandManagerClient(host, DIM_PORT)
client.create_session(session_id)
client.append_graph(session_id, graph.drop_list)
client.deploy_session(session_id, graph.start_oids)
else:
LOG.error('Unable to find the AWS credentials')
