def scan_experiment_directory(path):
""" Register all xtc file in a directory to RUCIO """
exper_per_scope = True
token = os.path.normpath(path).strip('/').split('/')
instr, exper, dtype = token[3:6]
# get all known dids
known_dids = set()
cl = DIDClient()
scope = exper
ds = "xtc"
for did in cl.list_dids(scope, {'name': "xtc.files.*"}, type='file'):
print(did)
known_dids.add(did)
#print(type(did['name']), did['name'])
print("Know dids for ", scope, ds, known_dids)
to_add = []
for fn in os.listdir(path):
if not op.isfile(op.join(path, fn)):
print("not a file, skip", fn)
continue
try:
_, name, runtok, streamtok = os.path.splitext(fn)[0].split('_')
except ValueError:
print("Wrong file format", fn)
continue
name = "xtc.files.{}".format(fn) # xtc.e_mfxte1234_0002_s0002.xtc
smdname = "xtc.files.{}".format(fn.replace('.xtc', '.smd.xtc'))
if name in known_dids:
print("File alredy in dataset")
continue
run_number = int(runtok[1:])
stream = int(streamtok[1:])
to_add.append({
'pfn': os.path.join(path, fn),
'scope': scope,
'name': name,
'run': run_number,
})
smdpfn = op.join(path, 'smalldata', fn.replace('.xtc', '.smd.xtc'))
to_add.append({
'pfn': smdpfn,
'scope': scope,
'name': smdname,
'run': run_number,
})
return to_add
def LIST_DIDS_WILDCARD(self, scope, wildcard):
jdoe_account = 'jdoe'
client = DIDClient(account=jdoe_account)
print 'run with: ' + str(wildcard)
start = time()
with monitor.record_timer_block('jdoe.list_dids_wildcard'):
dids = [did for did in client.list_dids(scope=scope, filters=wildcard, type='dataset')]
duration = time() - start
cnt = len(dids)
print 'got %d dids' % cnt
monitor.record_counter('jdoe.list_dids_wildcard.num_results', cnt)
if cnt != 0:
monitor.record_counter('jdoe.list_dids_wildcard.time_per_did', duration / cnt)
return {'no_datasets': cnt}
def test_dids_at_different_vos(self):
""" MULTI VO (CLIENT): Test that dids from 2nd vo don't interfere """
scope_uuid = str(generate_uuid()).lower()[:16]
scope = 'shr_%s' % scope_uuid
add_scope(scope, 'root', 'root', **self.vo)
add_scope(scope, 'root', 'root', **self.new_vo)
did_client = DIDClient()
did_uuid = str(generate_uuid()).lower()[:16]
tst = 'tstset_%s' % did_uuid
new = 'newset_%s' % did_uuid
shr = 'shrset_%s' % did_uuid
did_client.add_did(scope, tst, 'DATASET')
did_client.add_did(scope, shr, 'DATASET')
add_did(scope, new, 'DATASET', 'root', **self.new_vo)
add_did(scope, shr, 'DATASET', 'root', **self.new_vo)
did_list_tst = list(did_client.list_dids(scope, {}))
did_list_new = list(list_dids(scope, {}, **self.new_vo))
assert_true(tst in did_list_tst)
assert_false(new in did_list_tst)
assert_true(shr in did_list_tst)
assert_false(tst in did_list_new)
assert_true(new in did_list_new)
assert_true(shr in did_list_new)
class TestDIDClients:
def setup(self):
self.account_client = AccountClient()
self.scope_client = ScopeClient()
self.meta_client = MetaClient()
self.did_client = DIDClient()
self.replica_client = ReplicaClient()
self.rse_client = RSEClient()
def test_list_dids(self):
""" DATA IDENTIFIERS (CLIENT): List dids by pattern."""
tmp_scope = scope_name_generator()
tmp_files = []
tmp_files.append('file_a_1%s' % generate_uuid())
tmp_files.append('file_a_2%s' % generate_uuid())
tmp_files.append('file_b_1%s' % generate_uuid())
tmp_rse = 'MOCK'
self.scope_client.add_scope('jdoe', tmp_scope)
for tmp_file in tmp_files:
self.replica_client.add_replica(tmp_rse, tmp_scope, tmp_file, 1, '0cc737eb')
results = []
for result in self.did_client.list_dids(tmp_scope, {'name': 'file_a_*'}, type='file'):
results.append(result)
assert_equal(len(results), 2)
results = []
for result in self.did_client.list_dids(tmp_scope, {'name': 'file_a_1*'}, type='file'):
results.append(result)
assert_equal(len(results), 1)
results = []
for result in self.did_client.list_dids(tmp_scope, {'name': 'file_*_1*'}, type='file'):
results.append(result)
assert_equal(len(results), 2)
results = []
for result in self.did_client.list_dids(tmp_scope, {'name': 'file*'}, type='file'):
results.append(result)
assert_equal(len(results), 3)
results = []
filters = {'name': 'file*', 'created_after': datetime.utcnow() - timedelta(hours=1)}
for result in self.did_client.list_dids(tmp_scope, filters):
results.append(result)
assert_equal(len(results), 0)
with assert_raises(UnsupportedOperation):
self.did_client.list_dids(tmp_scope, {'name': 'file*'}, type='whateverytype')
def test_list_recursive(self):
""" DATA IDENTIFIERS (CLIENT): List did recursive """
# Create nested containers and datast
tmp_scope_1 = 'list-did-recursive'
tmp_scope_2 = 'list-did-recursive-2'
self.scope_client.add_scope('root', tmp_scope_1)
self.scope_client.add_scope('root', tmp_scope_2)
tmp_container_1 = 'container_%s' % generate_uuid()
self.did_client.add_container(scope=tmp_scope_1, name=tmp_container_1)
tmp_container_2 = 'container_%s' % generate_uuid()
self.did_client.add_container(scope=tmp_scope_1, name=tmp_container_2)
tmp_dataset_1 = 'dataset_%s' % generate_uuid()
self.did_client.add_dataset(scope=tmp_scope_2, name=tmp_dataset_1)
tmp_dataset_2 = 'dataset_%s' % generate_uuid()
self.did_client.add_dataset(scope=tmp_scope_1, name=tmp_dataset_2)
self.did_client.attach_dids(scope=tmp_scope_1, name=tmp_container_1, dids=[{'scope': tmp_scope_2, 'name': tmp_dataset_1}])
self.did_client.attach_dids(scope=tmp_scope_1, name=tmp_container_2, dids=[{'scope': tmp_scope_1, 'name': tmp_dataset_2}])
self.did_client.attach_dids(scope=tmp_scope_1, name=tmp_container_1, dids=[{'scope': tmp_scope_1, 'name': tmp_container_2}])
# List DIDs not recursive - only the first container is expected
dids = [str(did) for did in self.did_client.list_dids(scope=tmp_scope_1, recursive=False, type='all', filters={'name': tmp_container_1})]
assert_equal(dids, [tmp_container_1])
# List DIDs recursive - first container and all attached collections are expected
dids = [str(did) for did in self.did_client.list_dids(scope=tmp_scope_1, recursive=True, type='all', filters={'name': tmp_container_1})]
assert_true(tmp_container_1 in dids)
assert_true(tmp_container_2 in dids)
assert_true(tmp_dataset_1 in dids)
assert_true(tmp_dataset_2 in dids)
assert_equal(len(dids), 4)
# List DIDs recursive - only containers are expected
dids = [str(did) for did in self.did_client.list_dids(scope=tmp_scope_1, recursive=True, type='container', filters={'name': tmp_container_1})]
assert_true(tmp_container_1 in dids)
assert_true(tmp_container_2 in dids)
assert_true(tmp_dataset_1 not in dids)
assert_true(tmp_dataset_2 not in dids)
assert_equal(len(dids), 2)
def test_list_by_length(self):
""" DATA IDENTIFIERS (CLIENT): List did with length """
tmp_scope = 'mock'
tmp_dsn = 'dsn_%s' % generate_uuid()
self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn)
dids = self.did_client.list_dids(tmp_scope, {'length.gt': 0})
results = []
for d in dids:
results.append(d)
assert_not_equal(len(results), 0)
dids = self.did_client.list_dids(tmp_scope, {'length.gt': -1, 'length.lt': 1})
results = []
for d in dids:
results.append(d)
assert_equal(len(results), 0)
dids = self.did_client.list_dids(tmp_scope, {'length': 0})
results = []
for d in dids:
results.append(d)
assert_equal(len(results), 0)
def test_list_by_metadata(self):
""" DATA IDENTIFIERS (CLIENT): List did with metadata"""
dsns = []
tmp_scope = 'mock'
tmp_dsn1 = 'dsn_%s' % generate_uuid()
dsns.append(tmp_dsn1)
dataset_meta = {'project': 'data12_8TeV',
'run_number': 400000,
'stream_name': 'physics_CosmicCalo',
'prod_step': 'merge',
'datatype': 'NTUP_TRIG',
'version': 'f392_m920',
}
self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn1, meta=dataset_meta)
tmp_dsn2 = 'dsn_%s' % generate_uuid()
dsns.append(tmp_dsn2)
dataset_meta['run_number'] = 400001
self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn2, meta=dataset_meta)
tmp_dsn3 = 'dsn_%s' % generate_uuid()
dsns.append(tmp_dsn3)
dataset_meta['stream_name'] = 'physics_Egamma'
dataset_meta['datatype'] = 'NTUP_SMWZ'
self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn3, meta=dataset_meta)
dids = self.did_client.list_dids(tmp_scope, {'project': 'data12_8TeV', 'version': 'f392_m920'})
results = []
for d in dids:
results.append(d)
for dsn in dsns:
assert_in(dsn, results)
dsns.remove(tmp_dsn1)
dids = self.did_client.list_dids(tmp_scope, {'project': 'data12_8TeV', 'run_number': 400001})
results = []
for d in dids:
results.append(d)
for dsn in dsns:
assert_in(dsn, results)
dsns.remove(tmp_dsn2)
dids = self.did_client.list_dids(tmp_scope, {'project': 'data12_8TeV', 'stream_name': 'physics_Egamma', 'datatype': 'NTUP_SMWZ'})
results = []
for d in dids:
results.append(d)
for dsn in dsns:
assert_in(dsn, results)
with assert_raises(KeyNotFound):
self.did_client.list_dids(tmp_scope, {'NotReallyAKey': 'NotReallyAValue'})
def test_add_did(self):
""" DATA IDENTIFIERS (CLIENT): Add, populate, list did content and create a sample"""
tmp_scope = 'mock'
tmp_rse = 'MOCK'
tmp_dsn = 'dsn_%s' % generate_uuid()
root = InternalAccount('root')
set_local_account_limit(root, get_rse_id('MOCK'), -1)
set_local_account_limit(root, get_rse_id('CERN-PROD_TZERO'), -1)
# PFN example: rfio://castoratlas.cern.ch/castor/cern.ch/grid/atlas/tzero/xx/xx/xx/filename
dataset_meta = {'project': 'data13_hip',
'run_number': 300000,
'stream_name': 'physics_CosmicCalo',
'prod_step': 'merge',
'datatype': 'NTUP_TRIG',
'version': 'f392_m927',
}
rules = [{'copies': 1, 'rse_expression': 'MOCK', 'account': 'root'}]
with assert_raises(ScopeNotFound):
self.did_client.add_dataset(scope='Nimportnawak', name=tmp_dsn, statuses={'monotonic': True}, meta=dataset_meta, rules=rules)
files = [{'scope': tmp_scope, 'name': 'lfn.%(tmp_dsn)s.' % locals() + str(generate_uuid()), 'bytes': 724963570, 'adler32': '0cc737eb'}, ]
with assert_raises(DataIdentifierNotFound):
self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn, statuses={'monotonic': True}, meta=dataset_meta, rules=rules, files=files)
with assert_raises(DataIdentifierNotFound):
self.did_client.add_files_to_dataset(scope=tmp_scope, name=tmp_dsn, files=files)
files = []
for i in range(5):
lfn = 'lfn.%(tmp_dsn)s.' % locals() + str(generate_uuid())
pfn = 'mock://localhost/tmp/rucio_rse/%(project)s/%(version)s/%(prod_step)s' % dataset_meta
# it doesn't work with mock: TBF
# pfn = 'srm://mock2.com:2880/pnfs/rucio/disk-only/scratchdisk/rucio_tests/%(project)s/%(version)s/%(prod_step)s' % dataset_meta
pfn += '%(tmp_dsn)s/%(lfn)s' % locals()
file_meta = {'guid': str(generate_uuid()), 'events': 10}
files.append({'scope': tmp_scope, 'name': lfn,
'bytes': 724963570, 'adler32': '0cc737eb',
'pfn': pfn, 'meta': file_meta})
rules = [{'copies': 1, 'rse_expression': 'CERN-PROD_TZERO', 'lifetime': timedelta(days=2), 'account': 'root'}]
with assert_raises(InvalidPath):
self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn, statuses={'monotonic': True}, meta=dataset_meta, rules=rules, files=files, rse=tmp_rse)
files_without_pfn = [{'scope': i['scope'], 'name': i['name'], 'bytes': i['bytes'], 'adler32': i['adler32'], 'meta': i['meta']} for i in files]
self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn, statuses={'monotonic': True}, meta=dataset_meta, rules=rules, files=files_without_pfn, rse=tmp_rse)
with assert_raises(DataIdentifierAlreadyExists):
self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn, files=files, rse=tmp_rse)
files = []
for i in range(5):
lfn = '%(tmp_dsn)s.' % locals() + str(generate_uuid())
pfn = 'mock://localhost/tmp/rucio_rse/%(project)s/%(version)s/%(prod_step)s' % dataset_meta
# it doesn't work with mock: TBF
# pfn = 'srm://mock2.com:2880/pnfs/rucio/disk-only/scratchdisk/rucio_tests/%(project)s/%(version)s/%(prod_step)s' % dataset_meta
pfn += '%(tmp_dsn)s/%(lfn)s' % locals()
file_meta = {'guid': str(generate_uuid()), 'events': 100}
files.append({'scope': tmp_scope, 'name': lfn,
'bytes': 724963570, 'adler32': '0cc737eb',
'pfn': pfn, 'meta': file_meta})
rules = [{'copies': 1, 'rse_expression': 'CERN-PROD_TZERO', 'lifetime': timedelta(days=2)}]
with assert_raises(InvalidPath):
self.did_client.add_files_to_dataset(scope=tmp_scope, name=tmp_dsn, files=files, rse=tmp_rse)
files_without_pfn = [{'scope': i['scope'], 'name': i['name'], 'bytes': i['bytes'], 'adler32': i['adler32'], 'meta': i['meta']} for i in files]
self.did_client.add_files_to_dataset(scope=tmp_scope, name=tmp_dsn, files=files_without_pfn, rse=tmp_rse)
self.did_client.close(scope=tmp_scope, name=tmp_dsn)
tmp_dsn_output = 'dsn_%s' % generate_uuid()
self.did_client.create_did_sample(input_scope=tmp_scope, input_name=tmp_dsn, output_scope=tmp_scope, output_name=tmp_dsn_output, nbfiles=2)
files = [f for f in self.did_client.list_files(scope=tmp_scope, name=tmp_dsn_output)]
assert_equal(len(files), 2)
def test_attach_dids_to_dids(self):
""" DATA IDENTIFIERS (CLIENT): Attach dids to dids"""
tmp_scope = 'mock'
tmp_rse = 'MOCK'
nb_datasets = 5
nb_files = 5
attachments, dsns = list(), list()
guid_to_query = None
dsn = {}
for i in range(nb_datasets):
attachment = {}
attachment['scope'] = tmp_scope
attachment['name'] = 'dsn.%s' % str(generate_uuid())
attachment['rse'] = tmp_rse
files = []
for i in range(nb_files):
files.append({'scope': tmp_scope, 'name': 'lfn.%s' % str(generate_uuid()),
'bytes': 724963570, 'adler32': '0cc737eb',
'meta': {'guid': str(generate_uuid()), 'events': 100}})
attachment['dids'] = files
guid_to_query = files[0]['meta']['guid']
dsn = {'scope': tmp_scope, 'name': attachment['name']}
dsns.append(dsn)
attachments.append(attachment)
self.did_client.add_datasets(dsns=dsns)
self.did_client.attach_dids_to_dids(attachments=attachments)
dsns_l = [i for i in self.did_client.get_dataset_by_guid(guid_to_query)]
assert_equal([dsn], dsns_l)
cnt_name = 'cnt_%s' % generate_uuid()
self.did_client.add_container(scope='mock', name=cnt_name)
with assert_raises(UnsupportedOperation):
self.did_client.attach_dids_to_dids([{'scope': 'mock', 'name': cnt_name, 'rse': tmp_rse, 'dids': attachment['dids']}])
def test_add_files_to_datasets(self):
""" DATA IDENTIFIERS (CLIENT): Add files to Datasets"""
tmp_scope = 'mock'
tmp_rse = 'MOCK'
dsn1 = 'dsn.%s' % str(generate_uuid())
dsn2 = 'dsn.%s' % str(generate_uuid())
meta = {'transient': True}
files1, files2, nb_files = [], [], 5
for i in range(nb_files):
files1.append({'scope': tmp_scope, 'name': 'lfn.%s' % str(generate_uuid()),
'bytes': 724963570, 'adler32': '0cc737eb',
'meta': {'guid': str(generate_uuid()), 'events': 100}})
files2.append({'scope': tmp_scope, 'name': 'lfn.%s' % str(generate_uuid()),
'bytes': 724963570, 'adler32': '0cc737eb',
'meta': {'guid': str(generate_uuid()), 'events': 100}})
self.did_client.add_dataset(scope=tmp_scope, name=dsn1, files=files1,
rse=tmp_rse, meta=meta)
self.did_client.add_dataset(scope=tmp_scope, name=dsn2, files=files2,
rse=tmp_rse, meta=meta)
attachments = [{'scope': tmp_scope, 'name': dsn1, 'dids': files2, 'rse': tmp_rse},
{'scope': tmp_scope, 'name': dsn2, 'dids': files1, 'rse': tmp_rse}]
self.did_client.add_files_to_datasets(attachments)
files = [f for f in self.did_client.list_files(scope=tmp_scope, name=dsn1)]
assert_equal(len(files), 10)
with assert_raises(FileAlreadyExists):
self.did_client.add_files_to_datasets(attachments)
for attachment in attachments:
for i in range(nb_files):
attachment['dids'].append({'scope': tmp_scope,
'name': 'lfn.%s' % str(generate_uuid()),
'bytes': 724963570,
'adler32': '0cc737eb',
'meta': {'guid': str(generate_uuid()),
'events': 100}})
self.did_client.add_files_to_datasets(attachments, ignore_duplicate=True)
files = [f for f in self.did_client.list_files(scope=tmp_scope, name=dsn1)]
assert_equal(len(files), 15)
# Corrupt meta-data
files = []
for attachment in attachments:
for file in attachment['dids']:
file['bytes'] = 1000
break
with assert_raises(FileConsistencyMismatch):
self.did_client.add_files_to_datasets(attachments, ignore_duplicate=True)
def test_add_dataset(self):
""" DATA IDENTIFIERS (CLIENT): Add dataset """
tmp_scope = 'mock'
tmp_dsn = 'dsn_%s' % generate_uuid()
self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn, meta={'project': 'data13_hip'})
did = self.did_client.get_did(tmp_scope, tmp_dsn)
assert_equal(did['scope'], tmp_scope)
assert_equal(did['name'], tmp_dsn)
with assert_raises(DataIdentifierNotFound):
self.did_client.get_did('i_dont_exist', 'neither_do_i')
def test_add_datasets(self):
""" DATA IDENTIFIERS (CLIENT): Bulk add datasets """
tmp_scope = 'mock'
dsns = list()
for i in range(500):
tmp_dsn = {'name': 'dsn_%s' % generate_uuid(), 'scope': tmp_scope, 'meta': {'project': 'data13_hip'}}
dsns.append(tmp_dsn)
self.did_client.add_datasets(dsns)
def test_exists(self):
""" DATA IDENTIFIERS (CLIENT): Check if data identifier exists """
tmp_scope = 'mock'
tmp_file = 'file_%s' % generate_uuid()
tmp_rse = 'MOCK'
self.replica_client.add_replica(tmp_rse, tmp_scope, tmp_file, 1, '0cc737eb')
did = self.did_client.get_did(tmp_scope, tmp_file)
assert_equal(did['scope'], tmp_scope)
assert_equal(did['name'], tmp_file)
with assert_raises(DataIdentifierNotFound):
self.did_client.get_did('i_dont_exist', 'neither_do_i')
def test_did_hierarchy(self):
""" DATA IDENTIFIERS (CLIENT): Check did hierarchy rule """
account = 'jdoe'
rse = 'MOCK'
scope = scope_name_generator()
file = ['file_%s' % generate_uuid() for i in range(10)]
dst = ['dst_%s' % generate_uuid() for i in range(4)]
cnt = ['cnt_%s' % generate_uuid() for i in range(4)]
self.scope_client.add_scope(account, scope)
for i in range(10):
self.replica_client.add_replica(rse, scope, file[i], 1, '0cc737eb')
for i in range(4):
self.did_client.add_did(scope, dst[i], 'DATASET', statuses=None, meta=None, rules=None)
for i in range(4):
self.did_client.add_did(scope, cnt[i], 'CONTAINER', statuses=None, meta=None, rules=None)
for i in range(4):
self.did_client.add_files_to_dataset(scope, dst[i], [{'scope': scope, 'name': file[2 * i], 'bytes': 1, 'adler32': '0cc737eb'},
{'scope': scope, 'name': file[2 * i + 1], 'bytes': 1, 'adler32': '0cc737eb'}])
self.did_client.add_containers_to_container(scope, cnt[1], [{'scope': scope, 'name': cnt[2]}, {'scope': scope, 'name': cnt[3]}])
self.did_client.add_datasets_to_container(scope, cnt[0], [{'scope': scope, 'name': dst[1]}, {'scope': scope, 'name': dst[2]}])
result = self.did_client.scope_list(scope, recursive=True)
for r in result:
pass
# TODO: fix, fix, fix
# if r['name'] == cnt[1]:
# assert_equal(r['type'], 'container')
# assert_equal(r['level'], 0)
# if (r['name'] == cnt[0]) or (r['name'] == dst[0]) or (r['name'] == file[8]) or (r['name'] == file[9]):
# assert_equal(r['level'], 0)
# else:
# assert_equal(r['level'], 1)
def test_detach_did(self):
""" DATA IDENTIFIERS (CLIENT): Detach dids from a did"""
account = 'jdoe'
rse = 'MOCK'
scope = scope_name_generator()
file = ['file_%s' % generate_uuid() for i in range(10)]
dst = ['dst_%s' % generate_uuid() for i in range(5)]
cnt = ['cnt_%s' % generate_uuid() for i in range(2)]
self.scope_client.add_scope(account, scope)
for i in range(10):
self.replica_client.add_replica(rse, scope, file[i], 1, '0cc737eb')
for i in range(5):
self.did_client.add_dataset(scope, dst[i], statuses=None, meta=None, rules=None)
for i in range(2):
self.did_client.add_container(scope, cnt[i], statuses=None, meta=None, rules=None)
for i in range(5):
self.did_client.add_files_to_dataset(scope, dst[i], [{'scope': scope, 'name': file[2 * i], 'bytes': 1, 'adler32': '0cc737eb'},
{'scope': scope, 'name': file[2 * i + 1], 'bytes': 1, 'adler32': '0cc737eb'}])
self.did_client.add_containers_to_container(scope, cnt[1], [{'scope': scope, 'name': dst[2]}, {'scope': scope, 'name': dst[3]}])
with assert_raises(UnsupportedOperation):
self.did_client.add_datasets_to_container(scope, cnt[0], [{'scope': scope, 'name': dst[1]}, {'scope': scope, 'name': cnt[1]}])
self.did_client.add_datasets_to_container(scope, cnt[0], [{'scope': scope, 'name': dst[1]}, {'scope': scope, 'name': dst[2]}])
self.did_client.detach_dids(scope, cnt[0], [{'scope': scope, 'name': dst[1]}])
self.did_client.detach_dids(scope, dst[3], [{'scope': scope, 'name': file[6]}, {'scope': scope, 'name': file[7]}])
result = self.did_client.scope_list(scope, recursive=True)
for r in result:
if r['name'] == dst[1]:
assert_equal(r['level'], 0)
if r['type'] == 'file':
if (r['name'] in file[6:9]):
assert_equal(r['level'], 0)
else:
assert_not_equal(r['level'], 0)
with assert_raises(UnsupportedOperation):
self.did_client.detach_dids(scope=scope, name=cnt[0], dids=[{'scope': scope, 'name': cnt[0]}])
self.did_client.close(scope, dst[4])
metadata = self.did_client.get_metadata(scope, dst[4])
i_bytes, i_length = metadata['bytes'], metadata['length']
metadata = self.did_client.get_metadata(scope, file[8])
file1_bytes = metadata['bytes']
metadata = self.did_client.get_metadata(scope, file[9])
file2_bytes = metadata['bytes']
self.did_client.detach_dids(scope, dst[4], [{'scope': scope, 'name': file[8]}, {'scope': scope, 'name': file[9]}])
metadata = self.did_client.get_metadata(scope, dst[4])
f_bytes, f_length = metadata['bytes'], metadata['length']
assert_equal(i_bytes, f_bytes + file1_bytes + file2_bytes)
assert_equal(i_length, f_length + 1 + 1)
def test_scope_list(self):
""" DATA IDENTIFIERS (CLIENT): Add, aggregate, and list data identifiers in a scope """
# create some dummy data
self.tmp_accounts = ['jdoe' for i in range(3)]
self.tmp_scopes = [scope_name_generator() for i in range(3)]
self.tmp_rses = [rse_name_generator() for i in range(3)]
self.tmp_files = ['file_%s' % generate_uuid() for i in range(3)]
self.tmp_datasets = ['dataset_%s' % generate_uuid() for i in range(3)]
self.tmp_containers = ['container_%s' % generate_uuid() for i in range(3)]
# add dummy data to the catalogue
for i in range(3):
self.scope_client.add_scope(self.tmp_accounts[i], self.tmp_scopes[i])
self.rse_client.add_rse(self.tmp_rses[i])
self.replica_client.add_replica(self.tmp_rses[i], self.tmp_scopes[i], self.tmp_files[i], 1, '0cc737eb')
# put files in datasets
for i in range(3):
for j in range(3):
files = [{'scope': self.tmp_scopes[j], 'name': self.tmp_files[j], 'bytes': 1, 'adler32': '0cc737eb'}]
self.did_client.add_dataset(self.tmp_scopes[i], self.tmp_datasets[j])
self.did_client.add_files_to_dataset(self.tmp_scopes[i], self.tmp_datasets[j], files)
# put datasets in containers
for i in range(3):
for j in range(3):
datasets = [{'scope': self.tmp_scopes[j], 'name': self.tmp_datasets[j]}]
self.did_client.add_container(self.tmp_scopes[i], self.tmp_containers[j])
self.did_client.add_datasets_to_container(self.tmp_scopes[i], self.tmp_containers[j], datasets)
# reverse check if everything is in order
for i in range(3):
result = self.did_client.scope_list(self.tmp_scopes[i], recursive=True)
r_topdids = []
r_otherscopedids = []
r_scope = []
for r in result:
if r['level'] == 0:
r_topdids.append(r['scope'] + ':' + r['name'])
r_scope.append(r['scope'])
if r['scope'] != self.tmp_scopes[i]:
r_otherscopedids.append(r['scope'] + ':' + r['name'])
assert_in(r['level'], [1, 2])
for j in range(3):
assert_equal(self.tmp_scopes[i], r_scope[j])
if j != i:
assert_in(self.tmp_scopes[j] + ':' + self.tmp_files[j], r_otherscopedids)
assert_not_in(self.tmp_scopes[i] + ':' + self.tmp_files[i], r_topdids)
def test_get_did(self):
""" DATA IDENTIFIERS (CLIENT): add a new data identifier and try to retrieve it back"""
rse = 'MOCK'
scope = 'mock'
file = generate_uuid()
dsn = generate_uuid()
self.replica_client.add_replica(rse, scope, file, 1, '0cc737eb')
did = self.did_client.get_did(scope, file)
assert_equal(did['scope'], scope)
assert_equal(did['name'], file)
self.did_client.add_dataset(scope=scope, name=dsn, lifetime=10000000)
did2 = self.did_client.get_did(scope, dsn)
assert_equal(type(did2['expired_at']), datetime)
def test_get_meta(self):
""" DATA IDENTIFIERS (CLIENT): add a new meta data for an identifier and try to retrieve it back"""
rse = 'MOCK'
scope = 'mock'
file = generate_uuid()
keys = ['project', 'run_number']
values = ['data13_hip', 12345678]
self.replica_client.add_replica(rse, scope, file, 1, '0cc737eb')
for i in range(2):
self.did_client.set_metadata(scope, file, keys[i], values[i])
meta = self.did_client.get_metadata(scope, file)
for i in range(2):
assert_equal(meta[keys[i]], values[i])
def test_list_content(self):
""" DATA IDENTIFIERS (CLIENT): test to list contents for an identifier"""
rse = 'MOCK'
scope = 'mock'
nbfiles = 5
dataset1 = generate_uuid()
dataset2 = generate_uuid()
container = generate_uuid()
files1 = [{'scope': scope, 'name': generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb'} for i in range(nbfiles)]
files2 = [{'scope': scope, 'name': generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb'} for i in range(nbfiles)]
self.did_client.add_dataset(scope, dataset1)
with assert_raises(DataIdentifierAlreadyExists):
self.did_client.add_dataset(scope, dataset1)
self.did_client.add_files_to_dataset(scope, dataset1, files1, rse=rse)
self.did_client.add_dataset(scope, dataset2)
self.did_client.add_files_to_dataset(scope, dataset2, files2, rse=rse)
self.did_client.add_container(scope, container)
datasets = [{'scope': scope, 'name': dataset1}, {'scope': scope, 'name': dataset2}]
self.did_client.add_datasets_to_container(scope, container, datasets)
contents = self.did_client.list_content(scope, container)
datasets_s = [d['name'] for d in contents]
assert_in(dataset1, datasets_s)
assert_in(dataset2, datasets_s)
def test_list_files(self):
""" DATA IDENTIFIERS (CLIENT): List files for a container"""
rse = 'MOCK'
scope = 'mock'
dataset1 = generate_uuid()
dataset2 = generate_uuid()
container = generate_uuid()
files1 = []
files2 = []
for i in range(10):
files1.append({'scope': scope, 'name': generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb'})
files2.append({'scope': scope, 'name': generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb'})
for i in range(10):
self.replica_client.add_replica(rse, scope, files1[i]['name'], 1, '0cc737eb')
self.replica_client.add_replica(rse, scope, files2[i]['name'], 1, '0cc737eb')
self.did_client.add_dataset(scope, dataset1)
self.did_client.add_files_to_dataset(scope, dataset1, files1)
self.did_client.add_dataset(scope, dataset2)
self.did_client.add_files_to_dataset(scope, dataset2, files2)
datasets = [{'scope': scope, 'name': dataset1}, {'scope': scope, 'name': dataset2}]
self.did_client.add_container(scope, container)
self.did_client.add_datasets_to_container(scope, container, datasets)
# List file content
content = self.did_client.list_files(scope, files1[i]['name'])
assert_true(content is not None)
for d in content:
assert_true(d['name'] == files1[i]['name'])
# List container content
for d in [{'name': x['name'], 'scope': x['scope'], 'bytes': x['bytes'], 'adler32': x['adler32']} for x in self.did_client.list_files(scope, container)]:
assert_in(d, files1 + files2)
# List non-existing data identifier content
with assert_raises(DataIdentifierNotFound):
self.did_client.list_files(scope, 'Nimportnawak')
def test_list_replicas(self):
""" DATA IDENTIFIERS (CLIENT): List replicas for a container"""
rse = 'MOCK'
scope = 'mock'
dsn1 = generate_uuid()
dsn2 = generate_uuid()
cnt = generate_uuid()
files1 = []
files2 = []
for i in range(10):
files1.append({'scope': scope, 'name': generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb'})
files2.append({'scope': scope, 'name': generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb'})
self.did_client.add_dataset(scope, dsn1)
self.did_client.add_files_to_dataset(scope, dsn1, files1, rse=rse)
self.did_client.add_dataset(scope, dsn2)
self.did_client.add_files_to_dataset(scope, dsn2, files2, rse=rse)
self.did_client.add_container(scope, cnt)
self.did_client.add_datasets_to_container(scope, cnt, [{'scope': scope, 'name': dsn1}, {'scope': scope, 'name': dsn2}])
replicas = self.replica_client.list_replicas(dids=[{'scope': scope, 'name': dsn1}])
assert_true(replicas is not None)
replicas = self.replica_client.list_replicas(dids=[{'scope': scope, 'name': cnt}])
assert_true(replicas is not None)
@raises(UnsupportedOperation)
def test_close(self):
""" DATA IDENTIFIERS (CLIENT): test to close data identifiers"""
tmp_rse = 'MOCK'
tmp_scope = 'mock'
# Add dataset
tmp_dataset = 'dsn_%s' % generate_uuid()
# Add file replica
tmp_file = 'file_%s' % generate_uuid()
self.replica_client.add_replica(rse=tmp_rse, scope=tmp_scope, name=tmp_file, bytes=1, adler32='0cc737eb')
# Add dataset
self.did_client.add_dataset(scope=tmp_scope, name=tmp_dataset)
# Add files to dataset
files = [{'scope': tmp_scope, 'name': tmp_file, 'bytes': 1, 'adler32': '0cc737eb'}, ]
self.did_client.add_files_to_dataset(scope=tmp_scope, name=tmp_dataset, files=files)
# Add a second file replica
tmp_file = 'file_%s' % generate_uuid()
self.replica_client.add_replica(tmp_rse, tmp_scope, tmp_file, 1, '0cc737eb')
# Add files to dataset
files = [{'scope': tmp_scope, 'name': tmp_file, 'bytes': 1, 'adler32': '0cc737eb'}, ]
self.did_client.add_files_to_dataset(scope=tmp_scope, name=tmp_dataset, files=files)
# Close dataset
with assert_raises(UnsupportedStatus):
self.did_client.set_status(scope=tmp_scope, name=tmp_dataset, close=False)
self.did_client.set_status(scope=tmp_scope, name=tmp_dataset, open=False)
# Add a third file replica
tmp_file = 'file_%s' % generate_uuid()
self.replica_client.add_replica(tmp_rse, tmp_scope, tmp_file, 1, '0cc737eb')
# Add files to dataset
files = [{'scope': tmp_scope, 'name': tmp_file, 'bytes': 1, 'adler32': '0cc737eb'}, ]
self.did_client.attach_dids(scope=tmp_scope, name=tmp_dataset, dids=files)
@raises
def test_open(self):
""" DATA IDENTIFIERS (CLIENT): test to re-open data identifiers for priv account"""
tmp_rse = 'MOCK'
tmp_scope = 'mock'
# Add dataset
tmp_dataset = 'dsn_%s' % generate_uuid()
# Add file replica
tmp_file = 'file_%s' % generate_uuid()
self.replica_client.add_replica(rse=tmp_rse, scope=tmp_scope, name=tmp_file, bytes=1, adler32='0cc737eb')
# Add dataset
self.did_client.add_dataset(scope=tmp_scope, name=tmp_dataset)
# Add files to dataset
files = [{'scope': tmp_scope, 'name': tmp_file, 'bytes': 1, 'adler32': '0cc737eb'}, ]
self.did_client.add_files_to_dataset(scope=tmp_scope, name=tmp_dataset, files=files)
# Add a second file replica
tmp_file = 'file_%s' % generate_uuid()
self.replica_client.add_replica(tmp_rse, tmp_scope, tmp_file, 1, '0cc737eb')
# Add files to dataset
files = [{'scope': tmp_scope, 'name': tmp_file, 'bytes': 1, 'adler32': '0cc737eb'}, ]
self.did_client.add_files_to_dataset(scope=tmp_scope, name=tmp_dataset, files=files)
# Close dataset
with assert_raises(UnsupportedStatus):
self.did_client.set_status(scope=tmp_scope, name=tmp_dataset, close=False)
self.did_client.set_status(scope=tmp_scope, name=tmp_dataset, open=False)
# Add a third file replica
self.did_client.set_status(scope=tmp_scope, name=tmp_dataset, open=True)
def test_bulk_get_meta(self):
""" DATA IDENTIFIERS (CLIENT): Add a new meta data for a list of DIDs and try to retrieve them back"""
key = 'project'
rse = 'MOCK'
scope = 'mock'
files = ['file_%s' % generate_uuid() for _ in range(4)]
dst = ['dst_%s' % generate_uuid() for _ in range(4)]
cnt = ['cnt_%s' % generate_uuid() for _ in range(4)]
meta_mapping = {}
list_dids = []
for idx in range(4):
self.replica_client.add_replica(rse, scope, files[idx], 1, '0cc737eb')
self.did_client.set_metadata(scope, files[idx], key, 'file_%s' % idx)
list_dids.append({'scope': scope, 'name': files[idx]})
meta_mapping['%s:%s' % (scope, files[idx])] = (key, 'file_%s' % idx)
for idx in range(4):
self.did_client.add_did(scope, dst[idx], 'DATASET', statuses=None, meta={key: 'dsn_%s' % idx}, rules=None)
list_dids.append({'scope': scope, 'name': dst[idx]})
meta_mapping['%s:%s' % (scope, dst[idx])] = (key, 'dsn_%s' % idx)
for idx in range(4):
self.did_client.add_did(scope, cnt[idx], 'CONTAINER', statuses=None, meta={key: 'cnt_%s' % idx}, rules=None)
list_dids.append({'scope': scope, 'name': cnt[idx]})
meta_mapping['%s:%s' % (scope, cnt[idx])] = (key, 'cnt_%s' % idx)
list_meta = [_ for _ in self.did_client.get_metadata_bulk(list_dids)]
res_list_dids = [{'scope': entry['scope'], 'name': entry['name']} for entry in list_meta]
res_list_dids.sort()
list_dids.sort()
assert_equal(list_dids, res_list_dids)
for meta in list_meta:
did = '%s:%s' % (meta['scope'], meta['name'])
met = meta_mapping[did]
assert_equal((key, meta[key]), met)
cnt = ['cnt_%s' % generate_uuid() for _ in range(4)]
for idx in range(4):
list_dids.append({'scope': scope, 'name': cnt[idx]})
list_meta = [_ for _ in self.did_client.get_metadata_bulk(list_dids)]
assert_equal(len(list_meta), 12)
list_dids = []
for idx in range(4):
list_dids.append({'scope': scope, 'name': cnt[idx]})
list_meta = [_ for _ in self.did_client.get_metadata_bulk(list_dids)]
assert_equal(len(list_meta), 0)
def main(argv=None):
import re
options = get_args()
scope = options.scope.split(',')
campaigns = options.campaigns.split(',')
daod = options.derivation #options.derivation.split(',')
version = options.version
debug = options.debug
save = options.save
# Output director
if version is not "vXX" and save:
print(os.environ['TestArea'])
path = os.environ[
'TestArea'] + "/../STAnalysisCode/VBFInvAnalysis/data/samples/"
if not os.path.exists(path):
os.makedirs(path)
else:
path = "."
outFile = {}
if '_' in daod: daod_name = daod.split('_')[1]
else: daod_name = daod
for proj in scope:
if save:
if not 'data' in proj:
#for campaign in campaigns: outFile.append(campaign:open(campaign+"_"+daod+".txt", "w"))
for campaign in campaigns:
outFile[campaign] = open(
path + "/" + version + "_" + campaign + "_" +
daod_name + ".txt", "w")
else:
if '_' in proj: proj_name = proj.split('_')[0]
outFile[proj] = open(
path + "/" + version + "_" + proj_name + "_" + daod_name +
".txt", "w")
outFile["pilot"] = open(
path + "/" + version + "_" + "pilot.txt", "w")
print '---------------------------------------'
print '-- Script Configuration --'
print '---------------------------------------'
print 'Project tags: ', scope
print 'MC Campaigns ', campaigns
print 'Derivation ', daod
print 'p-tags ', pTagDict
print 'Version ', version
print '---------------------------------------'
try:
from rucio.client.didclient import DIDClient
try:
client = DIDClient()
except:
print "Failed to get DIDClient"
return
except:
print "Please first setup rucio"
return
for proj in scope:
if not 'data' in proj: isData = False
else: isData = True
if not isData:
for campaign in campaigns:
print "\nGetting %s list..." % campaign
for groupname, runs in myMC.items():
if save:
outFile[campaign].write("### %s ######\n" % groupname)
print '#searching ', groupname, ' datasets'
for dsid in runs:
name = proj + "." + str(
dsid) + ".*." + daod + ".*" + mcCampaign[
campaign] + "*" + pTagDict[proj]
dids = {'name': name}
List = list(
DIDClient.list_dids(client, proj, dids,
'container'))
if not List:
print '# %s not found' % (name)
if save:
outFile[campaign].write("# %s not found\n" %
(name))
for dsn in List:
pattern = re.compile(
r'e\d\d\d\d_s\d\d\d\d_r\d*_p\d*')
if not pattern.search(dsn):
pattern_AFII = re.compile(
r'e\d\d\d\d_a\d*_r\d*_p\d*')
if not pattern_AFII.search(dsn):
continue
if debug or not save:
print dsn
if save: outFile[campaign].write("%s\n" % dsn)
if save: outFile["pilot"].write("%s\n" % dsn)
else:
print "Getting %s list..." % (proj)
name = proj + ".*.physics_Main.*" + daod + "*" + pTagDict[proj]
dids = {'name': name}
List = list(DIDClient.list_dids(client, proj, dids, 'container'))
if not List:
print '# %s not found' % (name)
outFile[campaign].write("# %s not found\n" % (name))
for dsn in List:
if debug or not save:
print dsn
if save: outFile[proj].write("%s\n" % dsn)
if save: outFile["pilot"].write("%s\n" % dsn)
class TestDIDClients:
def setup(self):
self.account_client = AccountClient()
self.scope_client = ScopeClient()
self.meta_client = MetaClient()
self.did_client = DIDClient()
self.replica_client = ReplicaClient()
self.rse_client = RSEClient()
def test_list_dids(self):
""" DATA IDENTIFIERS (CLIENT): List dids by pattern."""
tmp_scope = scope_name_generator()
tmp_files = []
tmp_files.append('file_a_1%s' % generate_uuid())
tmp_files.append('file_a_2%s' % generate_uuid())
tmp_files.append('file_b_1%s' % generate_uuid())
tmp_rse = 'MOCK'
self.scope_client.add_scope('jdoe', tmp_scope)
for tmp_file in tmp_files:
self.replica_client.add_replica(tmp_rse, tmp_scope, tmp_file, 1L, '0cc737eb')
results = []
for result in self.did_client.list_dids(tmp_scope, {'name': 'file_a_*'}, type='file'):
results.append(result)
assert_equal(len(results), 2)
results = []
for result in self.did_client.list_dids(tmp_scope, {'name': 'file_a_1*'}, type='file'):
results.append(result)
assert_equal(len(results), 1)
results = []
for result in self.did_client.list_dids(tmp_scope, {'name': 'file_*_1*'}, type='file'):
results.append(result)
assert_equal(len(results), 2)
results = []
for result in self.did_client.list_dids(tmp_scope, {'name': 'file*'}, type='file'):
results.append(result)
assert_equal(len(results), 3)
results = []
filters = {'name': 'file*', 'created_after': datetime.utcnow() - timedelta(hours=1)}
for result in self.did_client.list_dids(tmp_scope, filters):
results.append(result)
assert_equal(len(results), 0)
with assert_raises(UnsupportedOperation):
self.did_client.list_dids(tmp_scope, {'name': 'file*'}, type='whateverytype')
def test_list_by_metadata(self):
""" DATA IDENTIFIERS (CLIENT): List did with metadata"""
dsns = []
tmp_scope = 'mock'
tmp_dsn1 = 'dsn_%s' % generate_uuid()
dsns.append(tmp_dsn1)
dataset_meta = {'project': 'data12_8TeV',
'run_number': 400000,
'stream_name': 'physics_CosmicCalo',
'prod_step': 'merge',
'datatype': 'NTUP_TRIG',
'version': 'f392_m920',
}
self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn1, meta=dataset_meta)
tmp_dsn2 = 'dsn_%s' % generate_uuid()
dsns.append(tmp_dsn2)
dataset_meta['run_number'] = 400001
self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn2, meta=dataset_meta)
tmp_dsn3 = 'dsn_%s' % generate_uuid()
dsns.append(tmp_dsn3)
dataset_meta['stream_name'] = 'physics_Egamma'
dataset_meta['datatype'] = 'NTUP_SMWZ'
self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn3, meta=dataset_meta)
dids = self.did_client.list_dids(tmp_scope, {'project': 'data12_8TeV', 'version': 'f392_m920'})
results = []
for d in dids:
results.append(d)
for dsn in dsns:
assert_in(dsn, results)
dsns.remove(tmp_dsn1)
dids = self.did_client.list_dids(tmp_scope, {'project': 'data12_8TeV', 'run_number': 400001})
results = []
for d in dids:
results.append(d)
for dsn in dsns:
assert_in(dsn, results)
dsns.remove(tmp_dsn2)
dids = self.did_client.list_dids(tmp_scope, {'project': 'data12_8TeV', 'stream_name': 'physics_Egamma', 'datatype': 'NTUP_SMWZ'})
results = []
for d in dids:
results.append(d)
for dsn in dsns:
assert_in(dsn, results)
with assert_raises(KeyNotFound):
self.did_client.list_dids(tmp_scope, {'NotReallyAKey': 'NotReallyAValue'})
def test_add_did(self):
""" DATA IDENTIFIERS (CLIENT): Add, populate, list did content and create a sample"""
tmp_scope = 'mock'
tmp_rse = 'MOCK'
tmp_dsn = 'dsn_%s' % generate_uuid()
set_account_limit('root', get_rse_id('MOCK'), -1)
set_account_limit('root', get_rse_id('CERN-PROD_TZERO'), -1)
# PFN example: rfio://castoratlas.cern.ch/castor/cern.ch/grid/atlas/tzero/xx/xx/xx/filename
dataset_meta = {'project': 'data13_hip',
'run_number': 300000,
'stream_name': 'physics_CosmicCalo',
'prod_step': 'merge',
'datatype': 'NTUP_TRIG',
'version': 'f392_m927',
}
rules = [{'copies': 1, 'rse_expression': 'MOCK', 'account': 'root'}]
with assert_raises(ScopeNotFound):
self.did_client.add_dataset(scope='Nimportnawak', name=tmp_dsn, statuses={'monotonic': True}, meta=dataset_meta, rules=rules)
files = [{'scope': tmp_scope, 'name': 'lfn.%(tmp_dsn)s.' % locals() + str(generate_uuid()), 'bytes': 724963570L, 'adler32': '0cc737eb'}, ]
def main(argv=None):
import pprint
pp = pprint.PrettyPrinter(indent=1)
try:
from rucio.client.didclient import DIDClient
try:
client = DIDClient()
except:
print "Failed to get DIDClient"
return
except:
print "Please first setup rucio"
return
if argv is None: argv = sys.argv
dsid, campaigns, debug, verbose = commandline()
if verbose:
print(__doc__)
scope = "mc16_13TeV"
name = scope + "." + dsid + ".*.merge.AOD.*"
#name = scope + "." + dsid + ".*.recon.AOD.*"
if verbose:
print "Inputs:", scope, dsid, name
print
dids = {'name': name}
count = 0
datasetList = DIDClient.list_dids(client, scope, dids, 'container')
for dsn in datasetList:
count += 1
container = dsn
contevents = 0
contcampaigns = []
matchcampaign = False
if ':' in container:
scope = container.split(':')[0]
container = container.split(':')[1]
else:
scope = container.split('.')[0]
contents = DIDClient.list_content(client, scope, container)
for task in contents:
name = task['name']
scope = name.split('.')[0]
if debug:
print name
meta = DIDClient.get_metadata(client, scope, name)
if debug:
pp.pprint(meta)
campaign = meta['campaign'].split(':')[1]
if debug:
print campaign
if campaign in campaigns:
matchcampaign = True
dsnInfo = DIDClient.list_files(client, scope, name)
events = 0
for data in dsnInfo:
events += int(data['events'])
if debug:
print events
contevents += events
if campaign not in contcampaigns: contcampaigns.append(campaign)
if matchcampaign:
print '%10s %10s %s' % (",".join(contcampaigns), str(contevents),
container)
if count == 0:
print "No merge.AOD containers for", dsid, "found [", name, "]"
class TestDIDClients:
def setup(self):
self.account_client = AccountClient()
self.scope_client = ScopeClient()
self.meta_client = MetaClient()
self.did_client = DIDClient()
self.replica_client = ReplicaClient()
self.rse_client = RSEClient()
def test_list_dids(self):
""" DATA IDENTIFIERS (CLIENT): List dids by pattern."""
tmp_scope = scope_name_generator()
tmp_files = []
tmp_files.append('file_a_1%s' % generate_uuid())
tmp_files.append('file_a_2%s' % generate_uuid())
tmp_files.append('file_b_1%s' % generate_uuid())
tmp_rse = 'MOCK'
self.scope_client.add_scope('jdoe', tmp_scope)
for tmp_file in tmp_files:
self.replica_client.add_replica(tmp_rse, tmp_scope, tmp_file, 1L, '0cc737eb')
results = []
for result in self.did_client.list_dids(tmp_scope, {'name': 'file\_a\_*'}, type='file'):
results.append(result)
assert_equal(len(results), 2)
results = []
for result in self.did_client.list_dids(tmp_scope, {'name': 'file\_a\_1*'}, type='file'):
results.append(result)
assert_equal(len(results), 1)
results = []
for result in self.did_client.list_dids(tmp_scope, {'name': 'file\__\_1*'}, type='file'):
results.append(result)
assert_equal(len(results), 2)
results = []
for result in self.did_client.list_dids(tmp_scope, {'name': 'file*'}, type='file'):
results.append(result)
assert_equal(len(results), 3)
results = []
for result in self.did_client.list_dids(tmp_scope, {'name': 'file*'}):
results.append(result)
assert_equal(len(results), 0)
with assert_raises(UnsupportedOperation):
self.did_client.list_dids(tmp_scope, {'name': 'file*'}, type='whateverytype')
def test_list_by_metadata(self):
""" DATA IDENTIFIERS (CLIENT): List did with metadata"""
dsns = []
tmp_scope = 'mock'
tmp_dsn1 = 'dsn_%s' % generate_uuid()
dsns.append(tmp_dsn1)
dataset_meta = {'project': 'data12_8TeV',
'run_number': 400000,
'stream_name': 'physics_CosmicCalo',
'prod_step': 'merge',
'datatype': 'NTUP_TRIG',
'version': 'f392_m920',
}
self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn1, meta=dataset_meta)
tmp_dsn2 = 'dsn_%s' % generate_uuid()
dsns.append(tmp_dsn2)
dataset_meta['run_number'] = 400001
self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn2, meta=dataset_meta)
tmp_dsn3 = 'dsn_%s' % generate_uuid()
dsns.append(tmp_dsn3)
dataset_meta['stream_name'] = 'physics_Egamma'
dataset_meta['datatype'] = 'NTUP_SMWZ'
self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn3, meta=dataset_meta)
dids = self.did_client.list_dids(tmp_scope, {'project': 'data12_8TeV', 'version': 'f392_m920'})
results = []
for d in dids:
results.append(d)
for dsn in dsns:
assert_in(dsn, results)
dsns.remove(tmp_dsn1)
dids = self.did_client.list_dids(tmp_scope, {'project': 'data12_8TeV', 'run_number': 400001})
results = []
for d in dids:
results.append(d)
for dsn in dsns:
assert_in(dsn, results)
dsns.remove(tmp_dsn2)
dids = self.did_client.list_dids(tmp_scope, {'project': 'data12_8TeV', 'stream_name': 'physics_Egamma', 'datatype': 'NTUP_SMWZ'})
results = []
for d in dids:
results.append(d)
for dsn in dsns:
assert_in(dsn, results)
with assert_raises(KeyNotFound):
self.did_client.list_dids(tmp_scope, {'NotReallyAKey': 'NotReallyAValue'})
def test_add_did(self):
""" DATA IDENTIFIERS (CLIENT): Add, populate and list did content"""
tmp_scope = 'mock'
tmp_rse = 'MOCK'
tmp_dsn = 'dsn_%s' % generate_uuid()
# PFN example: rfio://castoratlas.cern.ch/castor/cern.ch/grid/atlas/tzero/xx/xx/xx/filename
dataset_meta = {'project': 'data13_hip',
'run_number': 300000,
'stream_name': 'physics_CosmicCalo',
'prod_step': 'merge',
'datatype': 'NTUP_TRIG',
'version': 'f392_m927',
}
rules = [{'copies': 1, 'rse_expression': 'MOCK', 'account': 'root'}]
with assert_raises(ScopeNotFound):
self.did_client.add_dataset(scope='Nimportnawak', name=tmp_dsn, statuses={'monotonic': True}, meta=dataset_meta, rules=rules)
self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn, statuses={'monotonic': True}, meta=dataset_meta, rules=rules)
with assert_raises(DataIdentifierNotFound):
self.did_client.add_files_to_dataset(scope=tmp_scope, name=tmp_dsn, files=[{'scope': tmp_scope, 'name': 'lfn.%(tmp_dsn)s.' % locals() + str(generate_uuid()),
'bytes': 724963570L, 'adler32': '0cc737eb'}, ])
files = []
for i in xrange(5):
lfn = 'lfn.%(tmp_dsn)s.' % locals() + str(generate_uuid())
pfn = 'mock://localhost/tmp/rucio_rse/%(project)s/%(version)s/%(prod_step)s' % dataset_meta
# it doesn't work with mock: TBF
# pfn = 'srm://mock2.com:2880/pnfs/rucio/disk-only/scratchdisk/rucio_tests/%(project)s/%(version)s/%(prod_step)s' % dataset_meta
pfn += '%(tmp_dsn)s/%(lfn)s' % locals()
file_meta = {'guid': str(generate_uuid()), 'events': 10}
files.append({'scope': tmp_scope, 'name': lfn,
'bytes': 724963570L, 'adler32': '0cc737eb',
'pfn': pfn, 'meta': file_meta})
rules = [{'copies': 1, 'rse_expression': 'CERN-PROD_TZERO', 'lifetime': timedelta(days=2)}]
self.did_client.add_files_to_dataset(scope=tmp_scope, name=tmp_dsn, files=files, rse=tmp_rse)
files = []
for i in xrange(5):
lfn = '%(tmp_dsn)s.' % locals() + str(generate_uuid())
pfn = 'mock://localhost/tmp/rucio_rse/%(project)s/%(version)s/%(prod_step)s' % dataset_meta
# it doesn't work with mock: TBF
# pfn = 'srm://mock2.com:2880/pnfs/rucio/disk-only/scratchdisk/rucio_tests/%(project)s/%(version)s/%(prod_step)s' % dataset_meta
pfn += '%(tmp_dsn)s/%(lfn)s' % locals()
file_meta = {'guid': str(generate_uuid()), 'events': 100}
files.append({'scope': tmp_scope, 'name': lfn,
'bytes': 724963570L, 'adler32': '0cc737eb',
'pfn': pfn, 'meta': file_meta})
rules = [{'copies': 1, 'rse_expression': 'CERN-PROD_TZERO', 'lifetime': timedelta(days=2)}]
self.did_client.add_files_to_dataset(scope=tmp_scope, name=tmp_dsn, files=files, rse=tmp_rse)
self.did_client.close(scope=tmp_scope, name=tmp_dsn)
def test_attach_dids_to_dids(self):
""" DATA IDENTIFIERS (CLIENT): Attach dids to dids"""
tmp_scope = 'mock'
tmp_rse = 'MOCK'
nb_datasets = 5
nb_files = 5
attachments, dsns = list(), list()
guid_to_query = None
dsn = {}
for i in xrange(nb_datasets):
attachment = {}
attachment['scope'] = tmp_scope
attachment['name'] = 'dsn.%s' % str(generate_uuid())
attachment['rse'] = tmp_rse
files = []
for i in xrange(nb_files):
files.append({'scope': tmp_scope, 'name': 'lfn.%s' % str(generate_uuid()),
'bytes': 724963570L, 'adler32': '0cc737eb',
'meta': {'guid': str(generate_uuid()), 'events': 100}})
attachment['dids'] = files
guid_to_query = files[0]['meta']['guid']
dsn = {'scope': tmp_scope, 'name': attachment['name']}
dsns.append(dsn)
attachments.append(attachment)
self.did_client.add_datasets(dsns=dsns)
self.did_client.attach_dids_to_dids(attachments=attachments)
l = [i for i in self.did_client.get_dataset_by_guid(guid_to_query)]
assert_equal([dsn], l)
cnt_name = 'cnt_%s' % generate_uuid()
self.did_client.add_container(scope='mock', name=cnt_name)
with assert_raises(UnsupportedOperation):
self.did_client.attach_dids_to_dids([{'scope': 'mock', 'name': cnt_name, 'rse': tmp_rse, 'dids': attachment['dids']}])
def test_add_dataset(self):
""" DATA IDENTIFIERS (CLIENT): Add dataset """
tmp_scope = 'mock'
tmp_dsn = 'dsn_%s' % generate_uuid()
self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn, meta={'project': 'data13_hip'})
did = self.did_client.get_did(tmp_scope, tmp_dsn)
assert_equal(did['scope'], tmp_scope)
assert_equal(did['name'], tmp_dsn)
with assert_raises(DataIdentifierNotFound):
self.did_client.get_did('i_dont_exist', 'neither_do_i')
def test_add_datasets(self):
""" DATA IDENTIFIERS (CLIENT): Bulk add datasets """
tmp_scope = 'mock'
dsns = list()
for i in xrange(500):
tmp_dsn = {'name': 'dsn_%s' % generate_uuid(), 'scope': tmp_scope, 'meta': {'project': 'data13_hip'}}
dsns.append(tmp_dsn)
self.did_client.add_datasets(dsns)
def test_exists(self):
""" DATA IDENTIFIERS (CLIENT): Check if data identifier exists """
tmp_scope = 'mock'
tmp_file = 'file_%s' % generate_uuid()
tmp_rse = 'MOCK'
self.replica_client.add_replica(tmp_rse, tmp_scope, tmp_file, 1L, '0cc737eb')
did = self.did_client.get_did(tmp_scope, tmp_file)
assert_equal(did['scope'], tmp_scope)
assert_equal(did['name'], tmp_file)
with assert_raises(DataIdentifierNotFound):
self.did_client.get_did('i_dont_exist', 'neither_do_i')
def test_did_hierarchy(self):
""" DATA IDENTIFIERS (CLIENT): Check did hierarchy rule """
account = 'jdoe'
rse = 'MOCK'
scope = scope_name_generator()
file = ['file_%s' % generate_uuid() for i in range(10)]
dst = ['dst_%s' % generate_uuid() for i in range(4)]
cnt = ['cnt_%s' % generate_uuid() for i in range(4)]
self.scope_client.add_scope(account, scope)
for i in range(10):
self.replica_client.add_replica(rse, scope, file[i], 1, '0cc737eb')
for i in range(4):
self.did_client.add_did(scope, dst[i], 'DATASET', statuses=None, meta=None, rules=None)
for i in range(4):
self.did_client.add_did(scope, cnt[i], 'CONTAINER', statuses=None, meta=None, rules=None)
for i in range(4):
self.did_client.add_files_to_dataset(scope, dst[i], [{'scope': scope, 'name': file[2 * i], 'bytes': 1L, 'adler32': '0cc737eb'},
{'scope': scope, 'name': file[2 * i + 1], 'bytes': 1L, 'adler32': '0cc737eb'}])
self.did_client.add_containers_to_container(scope, cnt[1], [{'scope': scope, 'name': cnt[2]}, {'scope': scope, 'name': cnt[3]}])
self.did_client.add_datasets_to_container(scope, cnt[0], [{'scope': scope, 'name': dst[1]}, {'scope': scope, 'name': dst[2]}])
result = self.did_client.scope_list(scope, recursive=True)
for r in result:
pass
# TODO: fix, fix, fix
# if r['name'] == cnt[1]:
# assert_equal(r['type'], 'container')
# assert_equal(r['level'], 0)
# if (r['name'] == cnt[0]) or (r['name'] == dst[0]) or (r['name'] == file[8]) or (r['name'] == file[9]):
# assert_equal(r['level'], 0)
# else:
# assert_equal(r['level'], 1)
def test_detach_did(self):
""" DATA IDENTIFIERS (CLIENT): Detach dids from a did"""
account = 'jdoe'
rse = 'MOCK'
scope = scope_name_generator()
file = ['file_%s' % generate_uuid() for i in range(10)]
dst = ['dst_%s' % generate_uuid() for i in range(4)]
cnt = ['cnt_%s' % generate_uuid() for i in range(2)]
self.scope_client.add_scope(account, scope)
for i in range(10):
self.replica_client.add_replica(rse, scope, file[i], 1L, '0cc737eb')
for i in range(4):
self.did_client.add_dataset(scope, dst[i], statuses=None, meta=None, rules=None)
for i in range(2):
self.did_client.add_container(scope, cnt[i], statuses=None, meta=None, rules=None)
for i in range(4):
self.did_client.add_files_to_dataset(scope, dst[i], [{'scope': scope, 'name': file[2 * i], 'bytes': 1L, 'adler32': '0cc737eb'},
{'scope': scope, 'name': file[2 * i + 1], 'bytes': 1L, 'adler32': '0cc737eb'}])
self.did_client.add_containers_to_container(scope, cnt[1], [{'scope': scope, 'name': dst[2]}, {'scope': scope, 'name': dst[3]}])
with assert_raises(UnsupportedOperation):
self.did_client.add_datasets_to_container(scope, cnt[0], [{'scope': scope, 'name': dst[1]}, {'scope': scope, 'name': cnt[1]}])
self.did_client.add_datasets_to_container(scope, cnt[0], [{'scope': scope, 'name': dst[1]}, {'scope': scope, 'name': dst[2]}])
self.did_client.detach_dids(scope, cnt[0], [{'scope': scope, 'name': dst[1]}])
self.did_client.detach_dids(scope, dst[3], [{'scope': scope, 'name': file[6]}, {'scope': scope, 'name': file[7]}])
result = self.did_client.scope_list(scope, recursive=True)
for r in result:
if r['name'] == dst[1]:
assert_equal(r['level'], 0)
if r['type'] is 'file':
if (r['name'] in file[6:9]):
assert_equal(r['level'], 0)
else:
assert_not_equal(r['level'], 0)
with assert_raises(UnsupportedOperation):
self.did_client.detach_dids(scope=scope, name=cnt[0], dids=[{'scope': scope, 'name': cnt[0]}])
def test_scope_list(self):
""" DATA IDENTIFIERS (CLIENT): Add, aggregate, and list data identifiers in a scope """
# create some dummy data
self.tmp_accounts = ['jdoe' for i in xrange(3)]
self.tmp_scopes = [scope_name_generator() for i in xrange(3)]
self.tmp_rses = [rse_name_generator() for i in xrange(3)]
self.tmp_files = ['file_%s' % generate_uuid() for i in xrange(3)]
self.tmp_datasets = ['dataset_%s' % generate_uuid() for i in xrange(3)]
self.tmp_containers = ['container_%s' % generate_uuid() for i in xrange(3)]
# add dummy data to the catalogue
for i in xrange(3):
self.scope_client.add_scope(self.tmp_accounts[i], self.tmp_scopes[i])
self.rse_client.add_rse(self.tmp_rses[i])
self.replica_client.add_replica(self.tmp_rses[i], self.tmp_scopes[i], self.tmp_files[i], 1L, '0cc737eb')
# put files in datasets
for i in xrange(3):
for j in xrange(3):
files = [{'scope': self.tmp_scopes[j], 'name': self.tmp_files[j], 'bytes': 1L, 'adler32': '0cc737eb'}]
self.did_client.add_dataset(self.tmp_scopes[i], self.tmp_datasets[j])
self.did_client.add_files_to_dataset(self.tmp_scopes[i], self.tmp_datasets[j], files)
def run(dsid,
campaigns,
summary,
output,
ratio,
aod,
evnt,
verbose,
argv=None,
run_deriv=False):
import pprint
import re
pp = pprint.PrettyPrinter(indent=1)
try:
from rucio.client.didclient import DIDClient
try:
client = DIDClient()
except:
print "Failed to get DIDClient"
return
except:
print "Please first setup rucio"
return
if argv is None: argv = sys.argv
allflags = not (output or ratio or aod or evnt or verbose)
if verbose:
print(__doc__)
if not (output or summary):
print "=================================== TIDs ==================================="
############################################################
# EVNT
############################################################
if not (output or summary):
print "---- EVNT ----"
scope = "mc15_13TeV"
count = 0
evnt_mc_map = {
"Name": "mySAMPLE",
"shortName": "mySAMPLE",
"TOT": 0,
"MC15c": 0,
"MC16a": 0,
"MC16c": 0,
"MC16d": 0,
"MC16e": 0
}
name = scope + "." + dsid + ".*.EVNT.*"
dids = {'name': name}
List = DIDClient.list_dids(client, scope, dids, 'container')
for dsn in List:
short_name = dsn.split(".")[1] + "." + dsn.split(".")[2]
count += 1
container = dsn
contevents = 0
contcampaigns = []
matchcampaign = False
if ':' in container:
scope = container.split(':')[0]
container = container.split(':')[1]
else:
scope = container.split('.')[0]
contents = DIDClient.list_content(client, scope, container)
for task in contents:
name = task['name']
if verbose: print name
scope = name.split('.')[0]
meta = DIDClient.get_metadata(client, scope, name)
if verbose: pp.pprint(meta)
if ':' in meta['campaign']:
campaign = meta['campaign'].split(':')[1]
else:
campaign = meta['campaign']
if verbose: print campaign
if campaign in campaigns:
matchcampaign = True
#campaign = meta['campaign'].split(':')[1]
dsnInfo = DIDClient.list_files(client, scope, name)
events = 0
for data in dsnInfo:
if data['events'] is not None:
events += int(data['events'])
contevents += events
if verbose: print events
# print campaign, events, tid
if not (output or summary):
print '*EVNT %10s %10s %s' % (campaign, str(events), name)
# Store events in each campaign
evnt_mc_map["Name"] = dsn
evnt_mc_map["shortName"] = short_name
if campaign not in contcampaigns: contcampaigns.append(campaign)
if ",".join(contcampaigns) not in evnt_mc_map:
evnt_mc_map[",".join(contcampaigns)] = 0
evnt_mc_map[campaign] += events
if matchcampaign:
evnt_mc_map["TOT"] += contevents
if count == 0:
print "No evgen.EVNT containers for", dsid, "found [", name, "]"
############################################################
# AOD
############################################################
if not (output or summary):
print "---- AOD ----"
count = 0
aod_mc_map = {
"Name": "mySAMPLE",
"shortName": "mySAMPLE",
"TOT": 0,
"MC16a": 0,
"MC16c": 0,
"MC16d": 0,
"MC16e": 0
}
aod_name_map = {"MC16a": "", "MC16c": "", "MC16d": "", "MC16e": ""}
scope = "mc16_13TeV"
name = scope + "." + dsid + ".*.recon.AOD.*"
dids = {'name': name}
List = DIDClient.list_dids(client, scope, dids, 'container')
for dsn in List:
pattern = re.compile(r'AOD.e\d\d\d\d_[s|a]\d{3,4}_r*\d')
if verbose:
print dsn, "\n > pattern AOD.eXXXX_a|sYYYY_rZZZZ is found: ", (
pattern.search(dsn) is not None)
if not pattern.search(dsn):
continue
short_name = dsn.split(".")[1] + "." + dsn.split(".")[2]
count += 1
container = dsn
contevents = 0
contcampaigns = []
matchcampaign = False
if ':' in container:
scope = container.split(':')[0]
container = container.split(':')[1]
else:
scope = container.split('.')[0]
contents = DIDClient.list_content(client, scope, container)
for task in contents:
name = task['name']
if verbose: print name
scope = name.split('.')[0]
meta = DIDClient.get_metadata(client, scope, name)
if verbose: pp.pprint(meta)
if ':' in meta['campaign']:
campaign = meta['campaign'].split(':')[1]
else:
campaign = meta['campaign']
if verbose: print campaign
if campaign in campaigns:
matchcampaign = True
dsnInfo = DIDClient.list_files(client, scope, name)
events = 0
if campaign not in contcampaigns: contcampaigns.append(campaign)
if (campaign == "MC16a" and "r9364" not in name):
if verbose: print "Found MC16a Sample but not r-tag!"
continue
if (campaign == "MC16c" and "r9781" not in name):
if verbose: print "Found MC16c Sample but not r-tag!"
continue
if (campaign == "MC16d" and "r10201" not in name):
if verbose: print "Found MC16d Sample but not r-tag!"
continue
if (campaign == "MC16e" and "r10724" not in name): #10724
if verbose: print "Found MC16e Sample but not r-tag!"
continue
for data in dsnInfo:
events += int(data['events'])
contevents += events
if verbose: print events
# print campaign, events, tid
if not (output or summary):
print '*AOD %10s %10s %s' % (campaign, str(events), name)
# Store events in each campaign
aod_mc_map["Name"] = dsn
aod_mc_map["shortName"] = short_name
if matchcampaign:
aod_mc_map[",".join(contcampaigns)] += contevents
aod_name_map[",".join(contcampaigns)] = container
aod_mc_map["TOT"] += contevents
if count == 0:
print "No recon.AOD containers for", dsid, "found [", name, "]"
# print EXOT5
if not (output or summary) and run_deriv:
print '---Running derivation of EXOT5----'
deriv_mc_map = {
"Name": "mySAMPLE",
"shortName": "mySAMPLE",
"TOT": 0,
"MC16a": 0,
"MC16c": 0,
"MC16d": 0,
"MC16e": 0,
"mc16_13TeV": 0
}
name = scope + "." + dsid + ".*.deriv.DAOD_EXOT5.*"
dids = {'name': name}
List = []
if run_deriv:
List = DIDClient.list_dids(client, scope, dids, 'container')
campaign = ""
for dsn in List:
container = dsn
if ':' in container:
scope = container.split(':')[0]
container = container.split(':')[1]
else:
scope = container.split('.')[0]
contents = DIDClient.list_content(client, scope, container)
if "_r9364" in container:
campaign = 'MC16a'
elif "_r9781" in container:
campaign = 'MC16c'
elif "_r10201" in container:
campaign = 'MC16d'
elif "_r10724" in container:
campaign = 'MC16e'
else:
campaign = 'mc16_13TeV'
if ('p3575' not in container) and ('p3627' not in container):
continue
if campaign == 'MC16a' and 'r9364_p3575' not in container:
continue
for task in contents:
name = task['name']
if verbose: print name
scope = name.split('.')[0]
meta = DIDClient.get_metadata(client, scope, name)
if verbose: pp.pprint(meta)
#if ':' in meta['campaign']:
# campaign = meta['campaign'].split(':')[1]
#else:
# campaign = meta['campaign']
if verbose: print campaign
if campaign in campaigns:
matchcampaign = True
dsnInfo = DIDClient.list_files(client, scope, name)
events = 0
if campaign not in contcampaigns: contcampaigns.append(campaign)
if (campaign == "MC16a" and ("r9364" not in name)):
if verbose: print "Found MC16a Sample but not r-tag!"
continue
if (campaign == "MC16c" and "r9781" not in name):
if verbose: print "Found MC16c Sample but not r-tag!"
continue
if (campaign == "MC16d" and "r10201" not in name):
if verbose: print "Found MC16d Sample but not r-tag!"
continue
if (campaign == "MC16e" and "r10724" not in name): #10724
if verbose: print "Found MC16e Sample but not r-tag!"
continue
for data in dsnInfo:
events += int(data['events'])
contevents += events
if verbose: print events
# print campaign, events, tid
if not (output or summary):
print '*EXOT5 %10s %10s %s' % (campaign, str(events), name)
# Store events in each campaign
deriv_mc_map["Name"] = dsn
deriv_mc_map["shortName"] = short_name
if matchcampaign and campaigns != '':
#deriv_mc_map[",".join(contcampaigns)] += contevenst
deriv_mc_map[campaign] += contevents
#aod_name_map[",".join(contcampaigns)] = container
deriv_mc_map["TOT"] += contevents
############################################################
# Print Summary
############################################################
if not (output):
print "\n=================================== Summary ==================================="
# EVNT
if not (output):
print "---- EVNT ----"
print '#EVNT: %10s' % (str(evnt_mc_map["Name"]))
if (output and evnt and evnt_mc_map["Name"]):
print '%10s' % (str(evnt_mc_map["Name"]))
for key, value in evnt_mc_map.iteritems():
if key in campaigns:
if not (output):
print '+EVNT %10s %10s ' % (key, str(value))
if not (output):
print '=EVNT %10s %10s ' % ("TOT", str(evnt_mc_map["TOT"]))
# AOD
if not (output):
print "---- AOD ----"
print '#AOD: %10s' % (str(aod_mc_map["shortName"]))
for key, value in aod_mc_map.iteritems():
if key in campaigns:
if not (output):
print '+AOD %10s %10s %10s' % (key, str(value),
str(aod_name_map[key]))
if (output and aod_name_map[key] and aod):
print '%s %s' % (key, str(aod_name_map[key]))
if not (output):
print '=AOD %10s %10s' % ("TOT", str(aod_mc_map["TOT"]))
############################################################
# Print Output
############################################################
if not (output):
print "\n=================================== Final ==================================="
if not (output):
print "Short Name - Total EVNT - AOD/EVNT(MC16a) - AOD/EVNT(MC16c) - AOD/EVNT(MC16d) - AOD/EVNT(MC16e)"
'''
print '%10s %10s %10s/%s=%.2f %10s/%s=%.2f %10s/%s=%.2f %10s/%s=%.2f' % (evnt_mc_map["shortName"],
'''
if (ratio or allflags):
print '%s %s %s %s %.2f %s %s %.2f %s %s %.2f %s %s %.2f' % (
evnt_mc_map["shortName"], str(
evnt_mc_map["TOT"] / 1.e6), str(aod_mc_map["MC16a"] / 1e6),
str((evnt_mc_map["MC16a"] + evnt_mc_map["MC15c"]) / 1e6),
safe_div(float(aod_mc_map["MC16a"]),
(evnt_mc_map["MC16a"] + evnt_mc_map["MC15c"])),
str(aod_mc_map["MC16c"] / 1e6), str(evnt_mc_map["MC16c"] / 1e6),
safe_div(float(aod_mc_map["MC16c"]),
evnt_mc_map["MC16c"]), str(aod_mc_map["MC16d"] / 1e6),
str((evnt_mc_map["MC16d"] + evnt_mc_map["MC16c"]) / 1e6),
safe_div(float(aod_mc_map["MC16d"]),
(evnt_mc_map["MC16d"] + evnt_mc_map["MC16c"])),
str(aod_mc_map["MC16e"] / 1e6), str(evnt_mc_map["MC16e"] / 1e6),
safe_div(float(aod_mc_map["MC16e"]), evnt_mc_map["MC16e"]))
# return the event count information
return aod_mc_map, evnt_mc_map, deriv_mc_map