payload_handler.load_payload(options.input)
named_pipe = payload_handler.payload['workflow']['NamedPipe']
stat_client = StatsPipeClient(named_pipe)
initial = payload_handler.payload['workflow']['dataset']
timing = {'stats':{'api':'listFiles', 'query':initial}}
## last step (list all files in DBS3 below the 'initial' root)
with TimingStat(timing, stat_client) as timer:
files = api.listFiles(dataset=initial, detail=True)
timer.update_stats({'server_request_timing' : float(api.request_processing_time)/1000000.0,
'server_request_timestamp' : (api.request_time),
'request_content_length' : api.content_length})
timer.stat_to_server()
print("Found %s files" %(len(files)))
for this_file, interval in zip(files, increase_interval(0.0, 0.1)):
p = payload_handler.clone_payload()
p['workflow']['logical_file_name'] = this_file['logical_file_name']
del p['workflow']['dataset']
#p['workflow']['Intervals']['getFileParents'] += interval
payload_handler.append_payload(p)
payload_handler.save_payload(options.output)
with TimingStat(timing, stat_client) as timer:
if isinstance(initial, str):
datasets = api.listDatasets(dataset=initial)
else:
datasets = api.listDatasets(**initial)
request_processing_time, request_time = api.requestTimingInfo
timer.update_stats({'server_request_timing' : float(request_processing_time)/1000000.0,
'server_request_timestamp' : float(request_time)/1000000.0,
'request_content_length' : api.requestContentLength})
timer.stat_to_server()
#remove T0TEST datasets, since they are not analysed by users
datasets = map(lambda x: x['dataset'], datasets)
datasets = filter(lambda x: x.find('T0TEST')==-1, datasets)
#re-arrange the order of datasets, to have a more realistic chaotic use
shuffle(datasets)
print("Found %s datasets" % (len(datasets)))
for dataset, interval in zip(datasets, increase_interval(start=0.0, step=0.2)):
p = payload_handler.clone_payload()
p['workflow']['dataset'] = dataset
#p['workflow']['Intervals']['getPrimaryDatasetType'] += interval
#p['workflow']['Intervals']['CrabWorkflow'] += interval
payload_handler.append_payload(p)
payload_handler.save_payload(options.output)
with TimingStat(timing, stat_client) as timer:
if isinstance(initial, str):
datasets = api.listDatasets(dataset=initial)
else:
datasets = api.listDatasets(**initial)
request_processing_time, request_time = api.requestTimingInfo
timer.update_stats({'server_request_timing' : float(request_processing_time)/1000000.0,
'server_request_timestamp' : float(request_time)/1000000.0,
'request_content_length' : api.requestContentLength})
timer.stat_to_server()
#remove T0TEST datasets, since they are not analysed by users
datasets = map(lambda x: x['dataset'], datasets)
datasets = filter(lambda x: x.find('T0TEST')==-1, datasets)
#re-arrange the order of datasets, to have a more realistic chaotic use
shuffle(datasets)
print "Found %s datasets" % (len(datasets))
for dataset, interval in zip(datasets, increase_interval(start=0.0, step=0.2)):
p = payload_handler.clone_payload()
p['workflow']['dataset'] = dataset
#p['workflow']['Intervals']['getPrimaryDatasetType'] += interval
#p['workflow']['Intervals']['CrabWorkflow'] += interval
payload_handler.append_payload(p)
payload_handler.save_payload(options.output)
