-
Notifications
You must be signed in to change notification settings - Fork 0
/
rucio_psdm.py
77 lines (65 loc) · 2.21 KB
/
rucio_psdm.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
#!/usr/bin/env python
from __future__ import print_function
import os
import sys
import zlib
from rucio.client.replicaclient import ReplicaClient
from rucio.client.scopeclient import ScopeClient
from rucio.client.didclient import DIDClient
from rucio.common.exception import DataIdentifierNotFound
from rucio.common.exception import FileAlreadyExists
def adler32(fn):
data = open(fn).read()
return "{:>08s}".format(hex(zlib.adler32(data) & 0xffffffff)[2:])
def reg_file_list(dids):
""" add files for a run to rucio.
attach the files to a run data set:
filenames: <scope>:xtc.file.<fn>
dataset: <scope>:xtc.runNNNNN
"""
print("Add files to RUCIO")
if len(dids) == 0:
print("No files to add")
return 0
dids_to_register = []
scopes = set()
for did in dids:
nd = {
'pfn': "file:{}".format(did['pfn']),
'bytes': os.path.getsize(did['pfn']),
'adler32': adler32(did['pfn']),
'name': did['name'],
'scope': did['scope']
}
dids_to_register.append(nd)
scopes.add(did['scope'])
print(nd)
print(scopes)
if len(scopes) != 1:
print("Wrong number of scopes", len(scopes))
return 2
# register files to xtc.files
client = ReplicaClient()
client.add_replicas('LCLS_REGD', dids_to_register)
# add files to run dataset
known_run_ds = {}
client = DIDClient()
for did in dids:
run = did['run']
scope = did['scope']
if run not in known_run_ds:
run_ds = "xtc.run%05d" % run
try:
client.get_did(scope, run_ds)
except DataIdentifierNotFound:
print("Create new dataset", run_ds)
client.add_dataset(scope, run_ds)
client.add_datasets_to_container(scope, 'xtc', [{'scope': scope, 'name': run_ds},])
known_run_ds[run] = run_ds
ds = known_run_ds[run]
try:
client.attach_dids(scope, run_ds, [{'scope': scope, 'name': did['name']}])
except FileAlreadyExists:
print("File already exists", did['name'])
else:
print("attached", ds, did['name'])