def test_pipeline(self):
"""A pipeline from an LG all the way to a finished graph execution"""
lg = pkg_resources.resource_filename( # @UndefinedVariable
"test.dropmake", "logical_graphs/cont_img.graph")
fill = tool.start_process("fill", ["-L", lg], stdout=subprocess.PIPE)
unroll = tool.start_process("unroll", ["-z", "--app", "1"],
stdin=fill.stdout,
stdout=subprocess.PIPE)
partition = tool.start_process("partition",
stdin=unroll.stdout,
stdout=subprocess.PIPE)
map_ = tool.start_process(
"map",
["-N", "127.0.0.1,127.0.0.1"],
stdin=partition.stdout,
stdout=subprocess.PIPE,
)
mapped_graph, _ = map_.communicate()
for proc in fill, unroll, partition, map_:
self.assertEqual(proc.wait(), 0)
# It's valid JSON content, and actually a physical graph
mapped_graph = json.loads(common.b2s(mapped_graph))
# TODO: REPRODATA ATTACHED
mapped_graph.pop() # Get rid of reprodata
self.assertTrue(list(common.get_roots(mapped_graph)))
self.assertTrue(list(common.get_leaves(mapped_graph)))
def gen_pg_spec():
"""
RESTful interface to convert a PGT(P) into pg_spec
"""
try:
pgt_id = request.json.get("pgt_id")
node_list = request.json.get("node_list")
manager_host = request.json.get("manager_host")
if manager_host == "localhost":
manager_host = "127.0.0.1"
# try:
# manager_port = int(request.json.get("manager_port"));
# except:
# manager_port = None
# manager_prefix = request.json.get("manager_prefix");
print("pgt_id:" + str(pgt_id))
print("node_list:" + str(node_list))
# print("manager_port:" + str(manager_port))
# print("manager_prefix:" + str(manager_prefix))
except Exception as ex:
response.status = 500
print(traceback.format_exc())
return "Unable to parse json body of request for pg_spec: {0}".format(
ex)
pgtp = pg_mgr.get_pgt(pgt_id)
if pgtp is None:
response.status = 404
return "PGT(P) with id {0} not found in the Physical Graph Manager".format(
pgt_id)
if node_list is None:
response.status = 500
return "Must specify DALiUGE nodes list"
try:
# mgr_client = CompositeManagerClient(host=manager_host, port=manager_port, url_prefix=manager_prefix, timeout=30)
# # 1. get a list of nodes
# node_list = mgr_client.nodes()
# # 2. mapping PGTP to resources (node list)
pg_spec = pgtp.to_pg_spec([manager_host] + node_list, ret_str=False)
# 3. get list of root nodes
root_uids = common.get_roots(pg_spec)
response.content_type = "application/json"
return json.dumps({"pg_spec": pg_spec, "root_uids": list(root_uids)})
except Exception as ex:
response.status = 500
print(traceback.format_exc())
return "Fail to generate pg_spec: {0}".format(ex)
def test_pipeline(self):
"""A pipeline from an LG all the way to a finished graph execution"""
lg = pkg_resources.resource_filename( # @UndefinedVariable
'test.dropmake', 'logical_graphs/cont_img.json')
fill = tool.start_process('fill', ['-L', lg], stdout=subprocess.PIPE)
unroll = tool.start_process('unroll', ['-z', '--app', '1'], stdin=fill.stdout, stdout=subprocess.PIPE)
partition = tool.start_process('partition', stdin=unroll.stdout, stdout=subprocess.PIPE)
map_ = tool.start_process('map', ['-N', '127.0.0.1,127.0.0.1'], stdin=partition.stdout, stdout=subprocess.PIPE)
mapped_graph, _ = map_.communicate()
for proc in fill, unroll, partition, map_:
self.assertEqual(proc.wait(), 0)
# It's valid JSON content, and actually a physical graph
mapped_graph = json.loads(common.b2s(mapped_graph))
self.assertTrue(list(common.get_roots(mapped_graph)))
self.assertTrue(list(common.get_leaves(mapped_graph)))
def gen_pg():
"""
RESTful interface to convert a PGT(P) into PG by mapping
PGT(P) onto a given set of available resources
"""
# if the 'deploy' checkbox is not checked, then the form submission will NOT contain a 'dlg_mgr_deploy' field
deploy = request.query.get("dlg_mgr_deploy") is not None
mprefix = ""
pgt_id = request.query.get("pgt_id")
pgtp = pg_mgr.get_pgt(pgt_id)
if pgtp is None:
response.status = 404
return "PGT(P) with id {0} not found in the Physical Graph Manager".format(
pgt_id)
pgtpj = pgtp._gojs_json_obj
reprodata = pgtp.reprodata
logger.info("PGTP: %s", pgtpj)
num_partitions = 0
num_partitions = len(
list(filter(lambda n: "isGroup" in n, pgtpj["nodeDataArray"])))
surl = urlparse(request.url)
mhost = ""
mport = -1
mprefix = ""
q = parse_qs(surl.query)
if "dlg_mgr_url" in q:
murl = q["dlg_mgr_url"][0]
mparse = urlparse(murl)
try:
(mhost, mport) = mparse.netloc.split(":")
mport = int(mport)
except:
mhost = mparse.netloc
if mparse.scheme == "http":
mport = 80
elif mparse.scheme == "https":
mport = 443
mprefix = mparse.path
if mprefix.endswith("/"):
mprefix = mprefix[:-1]
else:
mhost = request.query.get("dlg_mgr_host")
if request.query.get("dlg_mgr_port"):
mport = int(request.query.get("dlg_mgr_port"))
logger.debug("Manager host: %s", mhost)
logger.debug("Manager port: %s", mport)
logger.debug("Manager prefix: %s", mprefix)
if mhost is None:
if request.query.get("tpl_nodes_len"):
nnodes = int(request.query.get("tpl_nodes_len"))
nnodes = num_partitions
else:
response.status = 500
return "Must specify DALiUGE manager host or tpl_nodes_len"
pg_spec = pgtp.to_pg_spec([], ret_str=False, tpl_nodes_len=nnodes)
pg_spec.append(reprodata)
response.content_type = "application/json"
response.set_header("Content-Disposition",
'attachment; filename="%s"' % (pgt_id))
return json.dumps(pg_spec)
try:
mgr_client = CompositeManagerClient(host=mhost,
port=mport,
url_prefix=mprefix,
timeout=30)
# 1. get a list of nodes
node_list = mgr_client.nodes()
# 2. mapping PGTP to resources (node list)
pg_spec = pgtp.to_pg_spec(node_list, ret_str=False)
if deploy:
dt = datetime.datetime.now().strftime("%Y-%m-%dT%H-%M-%S.%f")
ssid = "{0}_{1}".format(
pgt_id.split(".graph")[0].split("_pgt")[0].split("/")[-1], dt)
mgr_client.create_session(ssid)
# print "session created"
completed_uids = common.get_roots(pg_spec)
pg_spec.append(reprodata)
mgr_client.append_graph(ssid, pg_spec)
# print "graph appended"
mgr_client.deploy_session(ssid, completed_uids=completed_uids)
# mgr_client.deploy_session(ssid, completed_uids=[])
# print "session deployed"
# 3. redirect to the master drop manager
redirect("http://{0}:{1}{2}/session?sessionId={3}".format(
mhost, mport, mprefix, ssid))
else:
response.content_type = "application/json"
return json.dumps(pg_spec)
except restutils.RestClientException as re:
response.status = 500
return "Fail to interact with DALiUGE Drop Manager: {0}".format(re)
except HTTPResponse:
raise
except Exception as ex:
response.status = 500
print(traceback.format_exc())
return "Fail to deploy physical graph: {0}".format(ex)