def test_create_and_check_for_request_rest(self):
""" Request (REST): Test the creation, query, and deletion of a Request """
host = get_rest_host()
properties = get_request_properties()
client = Client(host=host)
request_id = client.add_request(**properties)
requests = client.get_requests(request_id=request_id)
assert_equal(len(requests), 1)
assert_equal(request_id, requests[0]['request_id'])
for key in properties:
if key in ['lifetime']:
continue
assert_equal(requests[0][key], properties[key])
client.update_request(request_id,
parameters={'status': RequestStatus.Failed})
requests = client.get_requests(request_id=request_id)
assert_equal(len(requests), 1)
assert_equal(requests[0]['status'], RequestStatus.Failed)
reqs = client.get_requests(request_id=999999)
assert_equal(len(reqs), 0)
def convert_req2reqv2(req):
# v1: {'created_at': datetime.datetime(2020, 11, 3, 10, 9, 32), 'substatus': None, 'priority': 0, 'transform_tag': '2', 'requester': 'panda', 'request_metadata': {'workload_id': 23083304, 'rule_id': 'bef3da17f17c49ac97863bb9e96af672'}, 'name': 'valid1.361027.Pythia8EvtGen_A14NNPDF23LO_jetjet_JZ7W.simul.HITS.e5112_s3227_tid12560193_00', 'request_id': 3775, 'accessed_at': datetime.datetime(2020, 11, 3, 10, 9, 32), 'updated_at': datetime.datetime(2020, 11, 3, 10, 9, 32), 'locking': <RequestLocking.Idle: 0>, 'status': <RequestStatus.Cancelled: 9>, 'workload_id': 23083304, 'request_type': <RequestType.StageIn: 2>, 'errors': None, 'processing_metadata': None, 'scope': 'valid1', 'expired_at': datetime.datetime(2020, 12, 3, 10, 9, 32), 'next_poll_at': datetime.datetime(2020, 11, 3, 10, 9, 32)} # noqa E501
if req['request_type'] == RequestType.StageIn:
request_metadata = req['request_metadata']
work = ATLASStageinWork(executable=None, arguments=None, parameters=None, setup=None,
exec_type='local', sandbox=None,
primary_input_collection={'scope': req['scope'], 'name': req['name']},
other_input_collections=None,
output_collections={'scope': req['scope'], 'name': req['name'] + '.idds.stagein'},
log_collections=None,
logger=None,
max_waiting_time=request_metadata.get('max_waiting_time', 3600 * 7 * 24),
src_rse=request_metadata.get('src_rse', None),
dest_rse=request_metadata.get('dest_rse', None),
rule_id=request_metadata.get('rule_id', None))
if req['request_type'] == RequestType.Workflow:
pass
ori_workflow = req['request_metadata']['workflow']
ori_work = ori_workflow.works[ori_workflow.primary_initial_work]
input_coll = ori_work.collections[ori_work.primary_input_collection]
work = ATLASStageinWork(executable=None, arguments=None, parameters=None, setup=None,
exec_type='local', sandbox=None,
primary_input_collection={'scope': input_coll['scope'], 'name': input_coll['name']},
other_input_collections=None,
output_collections={'scope': input_coll['scope'], 'name': input_coll['name'] + '.idds.stagein'},
log_collections=None,
logger=None,
max_waiting_time=ori_work.max_waiting_time,
src_rse=ori_work.src_rse,
dest_rse=ori_work.dest_rse,
rule_id=ori_work.rule_id)
workload_id = req['workload_id']
if not workload_id and 'workload_id' in request_metadata:
workload_id = request_metadata['workload_id']
wf = Workflow()
wf.set_workload_id(workload_id)
wf.add_work(work)
host = get_rest_host()
wm = ClientManager(host=host)
request_id = wm.submit(wf)
# print(request_id)
return request_id
def test_workflow_request(self):
workflow = self.init()
props = {
'scope': 'workflow',
'name': workflow.get_name(),
'requester': 'panda',
'request_type': RequestType.Workflow,
'transform_tag': 'workflow',
'status': RequestStatus.New,
'priority': 0,
'lifetime': 30,
'request_metadata': {'workload_id': '20776840', 'workflow': workflow.serialize()}
}
# print(props)
host = get_rest_host()
client = Client(host=host)
request_id = client.add_request(**props)
print(request_id)
def get_req_properties():
req_properties = {
'scope': 'data16_13TeV',
'name': 'data16_13TeV.00298862.physics_Main.daq.RAW',
'requester': 'panda',
'request_type': RequestType.StageIn,
'transform_tag': 'prodsys2',
'status': RequestStatus.New,
'priority': 0,
'lifetime': 30,
'request_metadata': {
'workload_id': '20776840',
'src_rse': 'NDGF-T1_DATATAPE',
'rule_id': '236e4bf87e11490291e3259b14724e30'
}
}
return req_properties
host = get_rest_host()
props = get_req_properties()
# props = get_example_real_tape_stagein_request()
# props = get_example_prodsys2_tape_stagein_request()
# props = get_example_active_learning_request()
client = Client(host=host)
request_id = client.add_request(**props)
print(request_id)
def __init__(self, host=None):
self.host = host
if self.host is None:
self.host = get_rest_host()
self.client = Client(host=self.host)
def core_exec(sandbox_url, log_token, dump_workflow, ops_file, user_name, test_mode):
tmpLog = LogWrapper(_logger, log_token)
is_OK = True
is_fatal = False
request_id = None
if dump_workflow == 'True':
dump_workflow = True
else:
dump_workflow = False
if test_mode == 'True':
test_mode = True
else:
test_mode = False
try:
with open(ops_file) as f:
ops = json.load(f)
try:
os.remove(ops_file)
except Exception:
pass
# go to temp dir
cur_dir = os.getcwd()
with tempfile.TemporaryDirectory() as tmp_dirname:
os.chdir(tmp_dirname)
# download sandbox
tmpLog.info('downloading sandbox from {}'.format(sandbox_url))
with requests.get(sandbox_url, allow_redirects=True, verify=False, stream=True) as r:
if r.status_code == 400:
tmpLog.error("not found")
is_fatal = True
is_OK = False
elif r.status_code != 200:
tmpLog.error("bad HTTP response {}".format(r.status_code))
is_OK = False
# extract sandbox
if is_OK:
with open(ops['data']['sandbox'], 'wb') as fs:
for chunk in r.raw.stream(1024, decode_content=False):
if chunk:
fs.write(chunk)
fs.close()
tmp_stat, tmp_out = commands_get_status_output(
'tar xvfz {}'.format(ops['data']['sandbox']))
if tmp_stat != 0:
tmpLog.error(tmp_out)
dump_str = 'failed to extract {}'.format(ops['data']['sandbox'])
tmpLog.error(dump_str)
is_fatal = True
is_OK = False
# parse workflow files
if is_OK:
tmpLog.info('parse workflow')
if ops['data']['language'] == 'cwl':
nodes, root_in = pcwl_utils.parse_workflow_file(ops['data']['workflowSpecFile'],
tmpLog)
with open(ops['data']['workflowInputFile']) as workflow_input:
data = yaml.safe_load(workflow_input)
s_id, t_nodes, nodes = pcwl_utils.resolve_nodes(nodes, root_in, data, 0, set(),
ops['data']['outDS'], tmpLog)
workflow_utils.set_workflow_outputs(nodes)
id_node_map = workflow_utils.get_node_id_map(nodes)
[node.resolve_params(ops['data']['taskParams'], id_node_map) for node in nodes]
dump_str = "the description was internally converted as follows\n" \
+ workflow_utils.dump_nodes(nodes)
tmpLog.info(dump_str)
for node in nodes:
s_check, o_check = node.verify()
tmp_str = 'Verification failure in ID:{} {}'.format(node.id, o_check)
if not s_check:
tmpLog.error(tmp_str)
dump_str += tmp_str
dump_str += '\n'
is_fatal = True
is_OK = False
else:
dump_str = "{} is not supported to describe the workflow"
tmpLog.error(dump_str)
is_fatal = True
is_OK = False
# convert to workflow
if is_OK:
workflow_to_submit, dump_str_list = workflow_utils.convert_nodes_to_workflow(nodes)
try:
if workflow_to_submit:
if not test_mode:
tmpLog.info('submit workflow')
wm = ClientManager(host=get_rest_host())
request_id = wm.submit(workflow_to_submit, username=user_name)
else:
dump_str = 'workflow is empty'
tmpLog.error(dump_str)
is_fatal = True
is_OK = False
except Exception as e:
dump_str = 'failed to submit the workflow with {}'.format(str(e))
tmpLog.error('{} {}'.format(dump_str, traceback.format_exc()))
if dump_workflow:
tmpLog.debug('\n' + ''.join(dump_str_list))
os.chdir(cur_dir)
except Exception as e:
is_OK = False
is_fatal = True
tmpLog.error("failed to run with {} {}".format(str(e), traceback.format_exc()))
with tempfile.NamedTemporaryFile(delete=False, mode='w') as tmp_json:
json.dump([is_OK, is_fatal, request_id, tmpLog.dumpToString()], tmp_json)
print(tmp_json.name)
sys.exit(0)
def test_catalog_rest(self):
""" Catalog (Rest): Test catalog rest functions """
host = get_rest_host()
client = Client(host=host)
req_properties = get_request_properties()
origin_request_id = add_request(**req_properties)
trans_properties = get_transform_properties()
trans_properties['request_id'] = origin_request_id
origin_trans_id = add_transform(**trans_properties)
coll_properties = get_collection_properties()
coll_properties['transform_id'] = origin_trans_id
coll_properties['request_id'] = origin_request_id
coll_properties['relation_type'] = CollectionRelationType.Output
origin_coll_id = add_collection(**coll_properties)
req_trans_colls = client.get_collections(request_id=origin_request_id, workload_id=None)
assert_equal(len(req_trans_colls.keys()), 1)
req_id = list(req_trans_colls.keys())[0]
assert_equal(origin_request_id, req_id)
assert_equal(len(req_trans_colls[req_id].keys()), 1)
trans_id = list(req_trans_colls[req_id].keys())[0]
assert_equal(trans_id, origin_trans_id)
colls = req_trans_colls[req_id][trans_id]
assert_equal(len(colls), 1)
assert_equal(colls[0]['coll_id'], origin_coll_id)
req_trans_colls1 = client.get_collections(request_id=None, workload_id=req_properties['request_metadata']['workload_id'])
assert_equal(is_same_req_trans_colls(req_trans_colls, req_trans_colls1), True)
req_trans_colls1 = client.get_collections(scope=coll_properties['scope'], name=coll_properties['name'],
request_id=None, workload_id=None)
assert_equal(is_same_req_trans_colls(req_trans_colls, req_trans_colls1), True)
content_output_properties = get_content_properties()
content_output_properties['content_type'] = ContentType.File
content_output_properties['min_id'] = 0
content_output_properties['max_id'] = 1000
content_output_properties['coll_id'] = origin_coll_id
origin_content_output_id_0_1000 = add_content(returning_id=True, **content_output_properties)
content_output_properties_0_100 = copy.deepcopy(content_output_properties)
content_output_properties_0_100['min_id'] = 0
content_output_properties_0_100['max_id'] = 100
content_output_properties['content_type'] = ContentType.Event
origin_content_output_id_0_100 = add_content(returning_id=True, **content_output_properties_0_100)
content_output_properties_100_200 = copy.deepcopy(content_output_properties)
content_output_properties_100_200['min_id'] = 100
content_output_properties_100_200['max_id'] = 200
content_output_properties['content_type'] = ContentType.Event
origin_content_output_id_100_200 = add_content(returning_id=True, **content_output_properties_100_200)
content_output_properties_name1 = copy.deepcopy(content_output_properties)
content_output_properties_name1['name'] = content_output_properties_name1['name'] + '_1'
content_output_properties_name1_id = add_content(returning_id=True, **content_output_properties_name1)
req_trans_coll_contents = client.get_contents(coll_scope=coll_properties['scope'], coll_name=coll_properties['name'],
request_id=origin_request_id, workload_id=req_properties['request_metadata']['workload_id'])
coll_contents = req_trans_coll_contents[origin_request_id][origin_trans_id]
coll_scope_name = '%s:%s' % (coll_properties['scope'], coll_properties['name'])
coll_scope_names = [scope_name for scope_name in coll_contents]
assert_equal(coll_scope_names, [coll_scope_name])
contents = coll_contents[coll_scope_name]['contents']
assert_equal(len(contents), 4)
output_content_ids = [output_content['content_id'] for output_content in contents]
assert_equal(output_content_ids, [origin_content_output_id_0_1000, origin_content_output_id_0_100,
origin_content_output_id_100_200, content_output_properties_name1_id])
req_trans_coll_contents1 = client.get_contents(coll_scope=coll_properties['scope'], coll_name=coll_properties['name'],
request_id=origin_request_id, workload_id=req_properties['request_metadata']['workload_id'],
relation_type=CollectionRelationType.Output)
assert_equal(is_same_req_trans_coll_contents(req_trans_coll_contents, req_trans_coll_contents1), True)
contents = client.get_match_contents(coll_scope=coll_properties['scope'], coll_name=coll_properties['name'],
scope=content_output_properties['scope'], name=content_output_properties['name'],
min_id=None, max_id=None, request_id=origin_request_id,
workload_id=req_properties['request_metadata']['workload_id'], only_return_best_match=False)
assert_equal(len(contents), 3)
content_ids = [content['content_id'] for content in contents]
content_ids.sort()
content_ids1 = [origin_content_output_id_0_1000, origin_content_output_id_0_100, origin_content_output_id_100_200]
content_ids1.sort()
assert_equal(content_ids, content_ids1)
contents = client.get_match_contents(coll_scope=coll_properties['scope'], coll_name=coll_properties['name'],
scope=content_output_properties['scope'], name=content_output_properties['name'],
min_id=0, max_id=50, request_id=origin_request_id,
workload_id=req_properties['request_metadata']['workload_id'], only_return_best_match=False)
assert_equal(len(contents), 2)
content_ids = [content['content_id'] for content in contents]
content_ids.sort()
content_ids1 = [origin_content_output_id_0_1000, origin_content_output_id_0_100]
content_ids1.sort()
assert_equal(content_ids, content_ids1)
contents = client.get_match_contents(coll_scope=coll_properties['scope'], coll_name=coll_properties['name'],
scope=content_output_properties['scope'], name=content_output_properties['name'],
min_id=0, max_id=50, request_id=origin_request_id,
workload_id=req_properties['request_metadata']['workload_id'], only_return_best_match=True)
assert_equal(len(contents), 1)
content_ids = [content['content_id'] for content in contents]
assert_equal(content_ids, [origin_content_output_id_0_100])
contents = [{'scope': content_output_properties['scope'], 'name': content_output_properties['name'],
'min_id': content_output_properties['min_id'], 'max_id': content_output_properties['max_id'],
'status': ContentStatus.Available, 'path': '/abc/test_path'},
{'scope': content_output_properties_name1['scope'], 'name': content_output_properties_name1['name'],
'min_id': content_output_properties_name1['min_id'], 'max_id': content_output_properties_name1['max_id'],
'status': ContentStatus.Failed}]
client.register_contents(coll_scope=coll_properties['scope'], coll_name=coll_properties['name'],
contents=contents, request_id=origin_request_id,
workload_id=req_properties['request_metadata']['workload_id'])
content = get_content(content_id=origin_content_output_id_0_1000)
assert_equal(content['status'], ContentStatus.Available)
assert_equal(content['path'], '/abc/test_path')
content = get_content(content_id=content_output_properties_name1_id)
assert_equal(content['status'], ContentStatus.Failed)
def __init__(self, host=None):
self.host = host
if self.host is None:
self.host = get_rest_host()
