def setUpClass(cls):
cls.sub_client = SubscriptionClient()
cls.did_client = DIDClient()
cls.projects = [
'data12_900GeV', 'data12_8TeV', 'data13_900GeV', 'data13_8TeV'
]
cls.pattern1 = r'(_tid|physics_(Muons|JetTauEtmiss|Egamma)\..*\.ESD|express_express(?!.*NTUP|.*\.ESD|.*RAW)|(physics|express)(?!.*NTUP).* \
def __init__(self,
dataset,
site,
rse=None,
scope=DEFAULT_SCOPE,
uuid=None,
check=True,
lifetime=None,
dry_run=False):
self.dataset = dataset
self.site = site
if rse is None:
rse = site
self.rse = rse
self.scope = scope
self.uuid = uuid
self.check = check
self.lifetime = lifetime
self.dry_run = dry_run
self.blocks = []
self.url = ''
self.getmetadata()
self.get_global_url()
self.didc = DIDClient()
self.repc = ReplicaClient()
self.gfal = Gfal2Context()
def setup(self):
self.did_client = DIDClient()
self.replica_client = ReplicaClient()
self.base_client = BaseClient(account='root',
ca_cert=config_get('client', 'ca_cert'),
auth_type='x509')
self.token = self.base_client.headers['X-Rucio-Auth-Token']
self.fname = generate_uuid()
rses = ['MOCK', 'MOCK3', 'MOCK4']
dsn = generate_uuid()
self.files = [{
'scope': 'mock',
'name': self.fname,
'bytes': 1,
'adler32': '0cc737eb'
}]
self.did_client.add_dataset(scope='mock', name=dsn)
self.did_client.add_files_to_dataset('mock',
name=dsn,
files=self.files,
rse='MOCK')
for r in rses:
self.replica_client.add_replicas(r, self.files)
def setup(self):
self.did_client = DIDClient()
self.tmp_scope = 'mock'
self.tmp_name = 'name_%s' % uuid()
self.did_client.add_did(scope=self.tmp_scope,
name=self.tmp_name,
type="DATASET")
def __init__(self):
#Take all data types categories
self.RAW_RECORDS_TPC_TYPES = helper.get_hostconfig(
)['raw_records_tpc_types']
self.RAW_RECORDS_MV_TYPES = helper.get_hostconfig(
)['raw_records_mv_types']
self.RAW_RECORDS_NV_TYPES = helper.get_hostconfig(
)['raw_records_nv_types']
self.LIGHT_RAW_RECORDS_TPC_TYPES = helper.get_hostconfig(
)['light_raw_records_tpc_types']
self.LIGHT_RAW_RECORDS_MV_TYPES = helper.get_hostconfig(
)['light_raw_records_mv_types']
self.LIGHT_RAW_RECORDS_NV_TYPES = helper.get_hostconfig(
)['light_raw_records_nv_types']
self.HIGH_LEVEL_TYPES = helper.get_hostconfig()['high_level_types']
self.RECORDS_TYPES = helper.get_hostconfig()['records_types']
#Choose which data type you want to treat
self.DTYPES = self.RAW_RECORDS_TPC_TYPES + self.RAW_RECORDS_MV_TYPES + self.RAW_RECORDS_NV_TYPES + self.LIGHT_RAW_RECORDS_TPC_TYPES + self.LIGHT_RAW_RECORDS_MV_TYPES + self.LIGHT_RAW_RECORDS_NV_TYPES + self.HIGH_LEVEL_TYPES + self.RECORDS_TYPES
#Take the list of all XENON RSEs
self.RSES = helper.get_hostconfig()['rses']
#Take the RSE that is used to perform the upload
self.UPLOAD_TO = helper.get_hostconfig()['upload_to']
#Take the directory where datamanager has to upload data
self.DATADIR = helper.get_hostconfig()['path_data_to_upload']
# Get the sequence of rules to be created according to the data type
self.RAW_RECORDS_TPC_RSES = helper.get_hostconfig(
)["raw_records_tpc_rses"]
self.RAW_RECORDS_MV_RSES = helper.get_hostconfig(
)["raw_records_mv_rses"]
self.RAW_RECORDS_NV_RSES = helper.get_hostconfig(
)["raw_records_nv_rses"]
self.LIGHT_RAW_RECORDS_TPC_RSES = helper.get_hostconfig(
)["light_raw_records_tpc_rses"]
self.LIGHT_RAW_RECORDS_MV_RSES = helper.get_hostconfig(
)["light_raw_records_mv_rses"]
self.LIGHT_RAW_RECORDS_NV_RSES = helper.get_hostconfig(
)["light_raw_records_nv_rses"]
self.HIGH_LEVEL_RSES = helper.get_hostconfig()["high_level_rses"]
self.RECORDS_RSES = helper.get_hostconfig()["records_rses"]
#Init the runDB
self.db = ConnectMongoDB()
#Init Rucio for later uploads and handling:
self.rc = RucioSummoner()
self.didclient = DIDClient()
self.replicaclient = ReplicaClient()
#Rucio Rule assignment priority
self.priority = 3
#Parameters to write warnings
self.minimum_number_acceptable_rses = 2
self.minimum_deltadays_allowed = 3
def setUp(self):
self.account = 'root'
self.scope = 'mock'
self.rse = 'MOCK5'
self.file_sizes = 2
self.dataset = 'dataset_%s' % generate_uuid()
self.rule_client = RuleClient()
self.did_client = DIDClient()
self.replica_client = ReplicaClient()
self.upload_client = UploadClient()
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
self.vo = {
'vo':
config_get('client',
'vo',
raise_exception=False,
default='tst')
}
else:
self.vo = {}
self.rse_id = get_rse_id(rse=self.rse, **self.vo)
def test_list_datasets_per_rse(self):
""" REPLICA (CLIENT): List datasets in RSE."""
rule_client = RuleClient()
did_client = DIDClient()
scope = 'mock'
dataset = 'dataset_' + str(generate_uuid())
did_client.add_dataset(scope=scope, name=dataset)
rule_client.add_replication_rule(dids=[{
'scope': scope,
'name': dataset
}],
account='root',
copies=1,
rse_expression='MOCK',
grouping='DATASET')
replicas = [
r for r in list_datasets_per_rse(
rse_id=get_rse_id(rse='MOCK', **self.vo),
filters={
'scope': InternalScope(scope, **self.vo),
'name': 'data*'
})
]
assert replicas != []
def test_list_dataset_replicas_bulk(self):
""" REPLICA (CLIENT): List dataset replicas bulk."""
replica_client = ReplicaClient()
rule_client = RuleClient()
did_client = DIDClient()
scope = 'mock'
did1 = {'scope': scope, 'name': 'dataset_' + str(generate_uuid())}
did_client.add_dataset(**did1)
did2 = {'scope': scope, 'name': 'dataset_' + str(generate_uuid())}
did_client.add_dataset(**did2)
dids = [did1, did2]
rule_client.add_replication_rule(dids=dids,
account='root',
copies=1,
rse_expression='MOCK',
grouping='DATASET')
with pytest.raises(InvalidObject):
replica_client.list_dataset_replicas_bulk(
dids=[{
'type': "I'm Different"
}])
replicas = list(replica_client.list_dataset_replicas_bulk(dids=dids))
assert len(replicas) == 2
for did in dids:
def replica_contains_did(rep):
return all(map(lambda k: k in rep and did[k] == rep[k], did))
assert any(
map(replica_contains_did,
replicas)), "%s must be in returned replicas" % (did, )
def setup(self):
self.account_client = AccountClient()
self.scope_client = ScopeClient()
self.meta_client = MetaClient()
self.did_client = DIDClient()
self.replica_client = ReplicaClient()
self.rse_client = RSEClient()
def __init__(self):
self.dc = DIDClient()
self.rc = ReplicaClient()
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
self.vo = {'vo': config_get('client', 'vo', raise_exception=False, default='tst')}
else:
self.vo = {}
def reg_file_list(dids):
""" add files for a run to rucio.
attach the files to a run data set:
filenames: <scope>:xtc.file.<fn>
dataset: <scope>:xtc.runNNNNN
"""
print("Add files to RUCIO")
if len(dids) == 0:
print("No files to add")
return 0
dids_to_register = []
scopes = set()
for did in dids:
nd = {
'pfn': "file:{}".format(did['pfn']),
'bytes': os.path.getsize(did['pfn']),
'adler32': adler32(did['pfn']),
'name': did['name'],
'scope': did['scope']
}
dids_to_register.append(nd)
scopes.add(did['scope'])
print(nd)
print(scopes)
if len(scopes) != 1:
print("Wrong number of scopes", len(scopes))
return 2
# register files to xtc.files
client = ReplicaClient()
client.add_replicas('LCLS_REGD', dids_to_register)
# add files to run dataset
known_run_ds = {}
client = DIDClient()
for did in dids:
run = did['run']
scope = did['scope']
if run not in known_run_ds:
run_ds = "xtc.run%05d" % run
try:
client.get_did(scope, run_ds)
except DataIdentifierNotFound:
print("Create new dataset", run_ds)
client.add_dataset(scope, run_ds)
client.add_datasets_to_container(scope, 'xtc', [{'scope': scope, 'name': run_ds},])
known_run_ds[run] = run_ds
ds = known_run_ds[run]
try:
client.attach_dids(scope, run_ds, [{'scope': scope, 'name': did['name']}])
except FileAlreadyExists:
print("File already exists", did['name'])
else:
print("attached", ds, did['name'])
def setUpClass(cls):
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
cls.vo = {'vo': get_vo()}
else:
cls.vo = {}
cls.sub_client = SubscriptionClient()
cls.did_client = DIDClient()
cls.projects = ['data12_900GeV', 'data12_8TeV', 'data13_900GeV', 'data13_8TeV']
cls.pattern1 = r'(_tid|physics_(Muons|JetTauEtmiss|Egamma)\..*\.ESD|express_express(?!.*NTUP|.*\.ESD|.*RAW)|(physics|express)(?!.*NTUP).* \
def setUp(self):
self.account = 'root'
self.scope = 'mock'
self.rule_client = RuleClient()
self.did_client = DIDClient()
self.replica_client = ReplicaClient()
self.upload_client = UploadClient()
self.file_sizes = 2
self.dataset = 'dataset_%s' % generate_uuid()
self.rse = 'MOCK5'
self.rse_id = get_rse_id(rse=self.rse)
def scan_experiment_directory(path):
""" Register all xtc file in a directory to RUCIO """
exper_per_scope = True
token = os.path.normpath(path).strip('/').split('/')
instr, exper, dtype = token[3:6]
# get all known dids
known_dids = set()
cl = DIDClient()
scope = exper
ds = "xtc"
for did in cl.list_dids(scope, {'name': "xtc.files.*"}, type='file'):
print(did)
known_dids.add(did)
#print(type(did['name']), did['name'])
print("Know dids for ", scope, ds, known_dids)
to_add = []
for fn in os.listdir(path):
if not op.isfile(op.join(path, fn)):
print("not a file, skip", fn)
continue
try:
_, name, runtok, streamtok = os.path.splitext(fn)[0].split('_')
except ValueError:
print("Wrong file format", fn)
continue
name = "xtc.files.{}".format(fn) # xtc.e_mfxte1234_0002_s0002.xtc
smdname = "xtc.files.{}".format(fn.replace('.xtc', '.smd.xtc'))
if name in known_dids:
print("File alredy in dataset")
continue
run_number = int(runtok[1:])
stream = int(streamtok[1:])
to_add.append({
'pfn': os.path.join(path, fn),
'scope': scope,
'name': name,
'run': run_number,
})
smdpfn = op.join(path, 'smalldata', fn.replace('.xtc', '.smd.xtc'))
to_add.append({
'pfn': smdpfn,
'scope': scope,
'name': smdname,
'run': run_number,
})
return to_add
def setUp(self):
""" Constructor."""
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
self.vo = {'vo': get_vo()}
else:
self.vo = {}
self.did_client = DIDClient()
def test_client_temporary_dids():
""" TMP DATA IDENTIFIERS (CLIENT): """
client = DIDClient()
temporary_dids = []
for _ in range(10):
temporary_dids.append({'scope': 'mock',
'name': 'object_%s' % generate_uuid(),
'rse': 'MOCK',
'bytes': 1,
'path': None})
client.add_temporary_dids(dids=temporary_dids)
def setUp(self):
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
self.vo = {'vo': config_get('client', 'vo', raise_exception=False, default='tst')}
else:
self.vo = {}
self.account_client = AccountClient()
self.scope_client = ScopeClient()
self.meta_client = MetaClient()
self.did_client = DIDClient()
self.replica_client = ReplicaClient()
self.rse_client = RSEClient()
def setUp(self):
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
self.vo = {'vo': config_get('client', 'vo', raise_exception=False, default='tst')}
else:
self.vo = {}
logger = logging.getLogger('dlul_client')
logger.addHandler(logging.StreamHandler())
logger.setLevel(logging.DEBUG)
self.client = Client()
self.did_client = DIDClient()
self.upload_client = UploadClient(_client=self.client, logger=logger)
self.download_client = DownloadClient(client=self.client, logger=logger)
def test_list_dataset_replicas(self):
""" REPLICA (CLIENT): List dataset replicas."""
replica_client = ReplicaClient()
rule_client = RuleClient()
did_client = DIDClient()
scope = 'mock'
dataset = 'dataset_' + str(generate_uuid())
did_client.add_dataset(scope=scope, name=dataset)
rule_client.add_replication_rule(dids=[{'scope': scope, 'name': dataset}],
account='root', copies=1, rse_expression='MOCK',
grouping='DATASET')
replicas = [r for r in replica_client.list_dataset_replicas(scope=scope, name=dataset)]
assert len(replicas) == 1
def setup(self):
try:
remove('/tmp/.rucio_root/auth_token_root')
except OSError as e:
if e.args[0] != 2:
raise e
self.marker = '$> '
self.host = config_get('client', 'rucio_host')
self.auth_host = config_get('client', 'auth_host')
self.user = '******'
self.def_rse = 'MOCK4'
self.did_client = DIDClient()
self.replica_client = ReplicaClient()
self.account_client = AccountLimitClient()
self.account_client.set_account_limit('root', self.def_rse, -1)
def cmsBlocksInContainer(self, container, scope='cms'):
didClient = DIDClient(account=self.account, auth_type=self.auth_type)
block_names = []
response = didClient.get_did(scope=scope, name=container)
if response['type'].upper() != 'CONTAINER':
return block_names
response = didClient.list_content(scope=scope, name=container)
for item in response:
if item['type'].upper() == 'DATASET':
block_names.append(item['name'])
return block_names
def setUpClass(cls):
cls.dataset = 'dataset_%s' % generate_uuid()
cls.rule_client = RuleClient()
cls.did_client = DIDClient()
cls.replica_client = ReplicaClient()
cls.upload_client = UploadClient()
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
cls.vo = {'vo': get_vo()}
cls.rse_id = get_rse_id(rse=cls.rse, **cls.vo)
def setup(self):
self.did_client = DIDClient()
self.tmp_scope = 'mock'
self.tmp_name = 'name_%s' % uuid()
self.did_client.add_did(scope=self.tmp_scope,
name=self.tmp_name,
type="DATASET")
self.implemented = True
session = get_session()
if session.bind.dialect.name == 'oracle':
oracle_version = int(
session.connection().connection.version.split('.')[0])
if oracle_version < 12:
self.implemented = False
elif session.bind.dialect.name == 'sqlite':
self.implemented = False
def __init__(self, dataset_name, start_time, end_time, frtype,
datafind_server=None, scope=None, site=None, rse=None, check=True,
lifetime=None, dry_run=False, no_multiprocs=False,
nprocs=MAXTHREADS):
if datafind_server is None:
# If undefined, use default from environment
try:
self.LIGO_DATAFIND_SERVER=os.environ['LIGO_DATAFIND_SERVER']
# If no datafind server in env, use cvmfs server
except:
self.LIGO_DATAFIND_SERVER="datafind.ligo.org:443"
else:
self.LIGO_DATAFIND_SERVER=datafind_server
self.dataset_name = dataset_name
self.start_time = start_time
self.end_time = end_time
self.frtype = frtype
self.no_multiprocs = no_multiprocs
logging.info("Attempting to determine scope from GPS time")
if scope is None:
self.scope = get_scope(start_time, end_time)
else:
self.scope=scope
logging.info("Scope: {}".format(self.scope))
self.site = site
if rse is None:
rse = site
self.rse = rse
self.check = check
self.lifetime = lifetime
self.dry_run = dry_run
# Initialization for dataset
self.get_global_url()
self.did_client = DIDClient()
self.rep_client = ReplicaClient()
# Locate frames
frames = self.find_frames()
# Create list of rucio replicas
self.list_files(frames, nprocs=nprocs)
def __init__(self, myscope, orgRse, destRse, account='bruzzese', working_folder=None):
self.myscope = myscope
self.orgRse = orgRse
self.destRse = destRse
self.working_folder = working_folder
self.gfal = Gfal2Context()
self.didc = DIDClient()
self.repc = ReplicaClient()
self.rulesClient = RuleClient()
# Configuration
self.account = account
# account=account
self.client = Client(account=self.account)
def __init__(self,
run,
originrse=DEFAULT_ORIGIN_RSE,
destrse=None,
scope=DEFAULT_SCOPE,
check=True,
lifetime=None,
dry_run=False,
container=None):
"""
:param dataset: Name of the PhEDEx dataset to synchronize with Rucio.
:param pnn: PhEDEx node name to filter on for replica information.
"""
self.run = run
self.originrse = originrse
self.destrse = destrse
self.scope = scope
self.check = check
self.lifetime = lifetime
self.dry_run = dry_run
self.container = container
self.rucio_datasets = {}
self.run_files = {}
self.existent_replica_files = {}
self.url = ''
self.gfal = Gfal2Context()
self.run_Number = None
self.get_run_Number()
self.files_storage = {}
self.get_global_url()
self.didc = DIDClient()
self.repc = ReplicaClient()
self.rulesClient = RuleClient()
# Right now obtaining the Metadata from the storage at WIPAC
# Hopefully in the future from JADE # TODO
self.get_run_Files()
self.get_rucio_metadata()
self.update_run_Files()
self.get_files_metadata()
def rucio_register(self, filenames):
files = []
dids = []
for filename in filenames:
size = os.stat(str(filename)).st_size
adler = adler32(str(filename))
files.append({
'scope': self.scope,
'name': str(filename.parts[-1]),
'bytes': size,
'adler32': adler,
'pfn': self.pfn + str(filename.parts[-1])
})
replica_client = ReplicaClient()
replica_client.add_replicas(rse=self.rse, files=files)
didclient = DIDClient()
didclient.add_files_to_dataset(self.scope, self.dataset, files)
def setup(self):
try:
remove('/tmp/.rucio_root/auth_token_root')
except OSError as e:
if e.args[0] != 2:
raise e
self.marker = '$> '
self.host = config_get('client', 'rucio_host')
self.auth_host = config_get('client', 'auth_host')
self.user = '******'
self.def_rse = 'MOCK4'
self.did_client = DIDClient()
self.replica_client = ReplicaClient()
self.account_client = AccountLimitClient()
self.account_client.set_account_limit('root', self.def_rse, -1)
add_rse_attribute(self.def_rse, 'istape', 'False')
self.upload_success_str = 'Successfully uploaded file %s'
def test_automatix(self):
""" MULTI VO (DAEMON): Test that automatix runs on a single VO """
scope_client = ScopeClient()
scope_uuid = str(generate_uuid()).lower()[:16]
shr_scope = 'shr_%s' % scope_uuid
scope_client.add_scope('root', shr_scope)
add_scope(shr_scope, 'root', 'root', **self.new_vo)
rse_client = RSEClient()
rse_str = ''.join(choice(ascii_uppercase) for x in range(10))
shr_rse = 'SHR_%s' % rse_str
mock_protocol = {'scheme': 'MOCK',
'hostname': 'localhost',
'port': 123,
'prefix': '/test/automatix',
'impl': 'rucio.rse.protocols.mock.Default',
'domains': {
'lan': {'read': 1,
'write': 1,
'delete': 1},
'wan': {'read': 1,
'write': 1,
'delete': 1}}}
rse_client.add_rse(shr_rse)
rse_client.add_rse_attribute(rse=shr_rse, key='verify_checksum', value=False)
rse_client.add_protocol(shr_rse, mock_protocol)
add_rse(shr_rse, 'root', **self.new_vo)
add_rse_attribute(rse=shr_rse, key='verify_checksum', value=False, issuer='root', **self.new_vo)
add_protocol(rse=shr_rse, data=mock_protocol, issuer='root', **self.new_vo)
automatix(sites=[shr_rse], inputfile='/opt/rucio/etc/automatix.json', sleep_time=30, account='root', once=True, scope=shr_scope)
did_list_tst = list(DIDClient().list_dids(shr_scope, {}))
did_list_new = list(list_dids(shr_scope, {}, **self.new_vo))
assert_not_equal(len(did_list_tst), 0)
assert_equal(len(did_list_new), 0)
did_dicts = [{'scope': shr_scope, 'name': n} for n in did_list_tst]
replicas_tst = list(ReplicaClient().list_replicas(did_dicts, rse_expression=shr_rse))
replicas_new = list(list_replicas(did_dicts, rse_expression=shr_rse, **self.new_vo))
assert_not_equal(len(replicas_tst), 0)
assert_equal(len(replicas_new), 0)
def test_did_set_metadata_bulk_multi_client(testdid):
""" DID (CLIENT) : Test setting metadata in bulk with multiple key-values """
skip_without_json()
testkeys = list(map(lambda i: 'testkey' + str(i), range(3)))
testmeta = {key: key + 'value' for key in testkeys}
# let two keys have the same value
testmeta[testkeys[1]] = testmeta[testkeys[0]]
didclient = DIDClient()
external_testdid = testdid.copy()
external_testdid['scope'] = testdid['scope'].external
result = didclient.set_metadata_bulk(meta=testmeta, recursive=False, **external_testdid)
assert result is True
meta = get_metadata(plugin="ALL", **testdid)
print('Metadata:', meta)
for testkey in testkeys:
assert testkey in meta and meta[testkey] == testmeta[testkey]
