def test_delete_mgr_SourceNotFound_single(self):
"""(RSE/PROTOCOLS): Delete a single file from storage (SourceNotFound)"""
mgr.delete(self.rse_settings, {
'name': 'not_existing_data.raw',
'scope': 'user.%s' % self.user
},
impl=self.impl)
def test_delete_mgr_ok_single(self):
"""(RSE/PROTOCOLS): Delete a single file from storage (Success)"""
mgr.delete(self.rse_settings, {
'name': '3_rse_remote_delete.raw',
'scope': 'user.%s' % self.user
},
impl=self.impl)
def test_delete_mgr_ok_dir(self):
"""(RSE/PROTOCOLS): Delete a directory from storage (Success)"""
mgr.delete(
self.rse_settings, {
'path': 'user/%s' % self.user,
'name': 'user.%s' % self.user,
'scope': 'user.%s' % self.user
})
def test_delete_mgr_SourceNotFound_multi(self):
"""(RSE/PROTOCOLS): Delete multiple files from storage (SourceNotFound)"""
status, details = mgr.delete(self.rse_settings, [{'name': 'not_existing_data.raw', 'scope': 'user.%s' % self.user}, {'name': '4_rse_remote_delete.raw', 'scope': 'user.%s' % self.user}])
if details['user.%s:4_rse_remote_delete.raw' % self.user]:
raise details['user.%s:not_existing_data.raw' % self.user]
else:
raise Exception('Return not as expected: %s, %s' % (status, details))
def test_delete_mgr_SourceNotFound_multi(self):
"""(RSE/PROTOCOLS): Delete multiple files from storage (SourceNotFound)"""
status, details = mgr.delete(self.rse_settings, [{'name': 'not_existing_data.raw', 'scope': 'user.%s' % self.user}, {'name': '4_rse_remote_delete.raw', 'scope': 'user.%s' % self.user}])
if details['user.%s:4_rse_remote_delete.raw' % self.user]:
raise details['user.%s:not_existing_data.raw' % self.user]
else:
raise Exception('Return not as expected: %s, %s' % (status, details))
def test_delete_mgr_ok_multi(self):
"""(RSE/PROTOCOLS): Delete multiple files from storage (Success)"""
status, details = mgr.delete(self.rse_settings, [{'name': '1_rse_remote_delete.raw', 'scope': 'user.%s' % self.user}, {'name': '2_rse_remote_delete.raw', 'scope': 'user.%s' % self.user}])
if not (status and details['user.%s:1_rse_remote_delete.raw' % self.user] and details['user.%s:2_rse_remote_delete.raw' % self.user]):
if isinstance(details['user.%s:1_rse_remote_delete.raw' % self.user], NotImplementedError) and isinstance(details['user.%s:2_rse_remote_delete.raw' % self.user], NotImplementedError):
pass
else:
raise Exception('Return not as expected: %s, %s' % (status, details))
def upload(files, scope, metadata, rse, account, source_dir, worker_number, total_workers, dataset_lifetime, did=None, set_metadata=False):
logging.debug('In upload')
dsn = None
if did:
dsn = {'scope': did.split(':')[0], 'name': did.split(':')[1]}
client = Client()
list_files = []
lfns = []
prepend_str = 'Thread [%i/%i] : ' % (worker_number, total_workers)
logging.debug(prepend_str + 'Looping over the files')
for filename in files:
fullpath = '%s/%s' % (source_dir, filename)
size = stat(fullpath).st_size
checksum = adler32(fullpath)
logging.info(prepend_str + 'File %s : Size %s , adler32 %s' % (fullpath, str(size), checksum))
list_files.append({'scope': scope, 'name': filename, 'bytes': size, 'adler32': checksum, 'meta': {'guid': generate_uuid()}})
lfns.append({'name': filename, 'scope': scope, 'filesize': size, 'adler32': checksum, 'filename': filename})
# Physical upload
logging.info(prepend_str + 'Uploading physically the files %s on %s' % (str(lfns), rse))
rse_info = rsemgr.get_rse_info(rse)
try:
success_upload = True
for cnt in xrange(0, 3):
global_status, ret = rsemgr.upload(rse_info, lfns=lfns, source_dir=source_dir)
logging.info(prepend_str + 'Returned global status : %s, Returned : %s' % (str(global_status), str(ret)))
if not global_status:
for item in ret:
if (not isinstance(ret[item], FileReplicaAlreadyExists)) and ret[item] is not True:
sleep(exp(cnt))
success_upload = False
logging.error(prepend_str + 'Problem to upload file %s with error %s' % (item, str(ret[item])))
break
else:
break
if not success_upload:
logging.error(prepend_str + 'Upload operation to %s failed, removing leftovers' % (rse))
rsemgr.delete(rse_info, lfns=lfns)
return False
except Exception, error:
logging.error(prepend_str + '%s' % (str(error)))
return False
def upload(files, scope, metadata, rse, account, source_dir, worker_number, total_workers, dataset_lifetime, did=None):
logging.debug('In upload')
dsn = None
if did:
dsn = {'scope': did.split(':')[0], 'name': did.split(':')[1]}
client = Client()
list_files = []
lfns = []
logging.debug('Thread [%i/%i] : Looping over the files' % (worker_number, total_workers))
for filename in files:
fullpath = '%s/%s' % (source_dir, filename)
size = stat(fullpath).st_size
checksum = adler32(fullpath)
logging.info('Thread [%i/%i] : File %s : Size %s , adler32 %s' % (worker_number, total_workers, fullpath, str(size), checksum))
list_files.append({'scope': scope, 'name': filename, 'bytes': size, 'adler32': checksum, 'meta': {'guid': generate_uuid()}})
lfns.append({'name': filename, 'scope': scope, 'filesize': size, 'adler32': checksum})
# Physical upload
logging.info('Thread [%i/%i] : Uploading physically the files %s on %s' % (worker_number, total_workers, str(lfns), rse))
rse_info = rsemgr.get_rse_info(rse)
try:
success_upload = True
for i in xrange(0, 3):
gs, ret = rsemgr.upload(rse_info, lfns=lfns, source_dir=source_dir)
logging.info('Returned global status : %s, Returned : %s' % (str(gs), str(ret)))
if not gs:
for x in ret:
if (not isinstance(ret[x], FileReplicaAlreadyExists)) and ret[x] is not True:
sleep(exp(i))
success_upload = False
logging.error('Problem to upload file %s with error %s' % (x, str(ret[x])))
break
else:
break
if not success_upload:
logging.error('Thread [%i/%i] : Upload operation to %s failed, removing leftovers' % (worker_number, total_workers, rse))
rsemgr.delete(rse_info, lfns=lfns)
return False
except Exception, e:
return False
def upload(files,
scope,
metadata,
rse,
account,
source_dir,
dataset_lifetime,
did=None,
set_metadata=False,
logger=logging.log):
logger(logging.DEBUG, 'In upload')
dsn = None
if did:
dsn = {'scope': did.split(':')[0], 'name': did.split(':')[1]}
client = Client()
list_files = []
lfns = []
for filename in files:
physical_fname = filename
if physical_fname.find('/') > -1:
physical_fname = "".join(filename.split('/'))
fullpath = '%s/%s' % (source_dir, physical_fname)
size = stat(fullpath).st_size
checksum = adler32(fullpath)
logger(logging.INFO, 'File %s : Size %s , adler32 %s', fullpath,
str(size), checksum)
list_files.append({
'scope': scope,
'name': filename,
'bytes': size,
'adler32': checksum,
'meta': {
'guid': generate_uuid()
}
})
lfns.append({
'name': filename,
'scope': scope,
'filesize': size,
'adler32': checksum,
'filename': physical_fname
})
# Physical upload
logger(logging.INFO, 'Uploading physically the files %s on %s', str(lfns),
rse)
rse_info = rsemgr.get_rse_info(rse, vo=client.vo)
try:
success_upload = True
for cnt in range(0, 3):
rows = rsemgr.upload(rse_info,
lfns=lfns,
source_dir=source_dir,
logger=logger)
# temporary hack
global_status, ret = rows['success'], rows[1]
logger(logging.INFO, 'Returned global status : %s, Returned : %s',
str(global_status), str(ret))
if not global_status:
for item in ret:
if (not isinstance(ret[item], FileReplicaAlreadyExists)
) and ret[item] is not True:
sleep(exp(cnt))
success_upload = False
logger(logging.ERROR,
'Problem to upload file %s with error %s', item,
str(ret[item]))
break
else:
break
if not success_upload:
logger(logging.ERROR,
'Upload operation to %s failed, removing leftovers', rse)
rsemgr.delete(rse_info, lfns=lfns)
return False
except Exception as error:
logger(logging.DEBUG, "Exception", exc_info=True)
logger(logging.ERROR, '%s', str(error))
return False
logger(logging.INFO, 'Files successfully copied on %s', rse)
# Registering DIDs and replicas in Rucio
logger(logging.INFO, 'Registering DIDs and replicas in Rucio')
meta = metadata
if not set_metadata:
meta = None
if dsn:
try:
client.add_dataset(scope=dsn['scope'],
name=dsn['name'],
rules=[{
'account': account,
'copies': 1,
'rse_expression': rse,
'grouping': 'DATASET',
'activity': 'Functional Test'
}],
meta=meta,
lifetime=dataset_lifetime)
client.add_files_to_dataset(scope=dsn['scope'],
name=dsn['name'],
files=list_files,
rse=rse)
logger(logging.INFO, 'Upload operation for %s:%s done',
dsn['scope'], dsn['name'])
except Exception as error:
logger(logging.DEBUG, "Exception", exc_info=True)
logger(logging.ERROR, 'Failed to upload %s', str(list_files))
logger(logging.ERROR, '%s', str(error))
logger(logging.ERROR, 'removing files from the Storage')
rsemgr.delete(rse_info, lfns=lfns)
return False
else:
logger(logging.WARNING, 'No dsn is specified')
try:
client.add_replicas(files=list_files, rse=rse)
client.add_replication_rule(list_files,
copies=1,
rse_expression=rse,
activity='Functional Test')
logger(logging.INFO, 'Upload operation for %s done',
str(list_files))
except Exception as error:
logger(logging.DEBUG, "Exception", exc_info=True)
logger(logging.ERROR, 'Failed to upload %s', str(list_files))
logger(logging.ERROR, '%s', str(error))
logger(logging.ERROR, 'Removing files from the Storage')
rsemgr.delete(rse_info, lfns=lfns)
return False
return True
except Exception, e:
return False
logging.info('Thread [%i/%i] : Files successfully copied on %s' % (worker_number, total_workers, rse))
# Registering DIDs and replicas in Rucio
logging.info('Thread [%i/%i] : Registering DIDs and replicas in Rucio' % (worker_number, total_workers))
if dsn:
try:
client.add_dataset(scope=dsn['scope'], name=dsn['name'], rules=[{'account': account, 'copies': 1, 'rse_expression': rse, 'grouping': 'DATASET'}], meta=metadata, lifetime=dataset_lifetime)
client.add_files_to_dataset(scope=dsn['scope'], name=dsn['name'], files=list_files, rse=rse)
logging.info('Thread [%i/%i] : Upload operation for %s:%s done' % (worker_number, total_workers, dsn['scope'], dsn['name']))
except Exception, e:
logging.error('Thread [%(worker_number)s/%(total_workers)s] : Failed to upload %(files)s' % locals())
logging.error('Thread [%i/%i] : %s' % (worker_number, total_workers, str(e)))
logging.error('Removing files from the Storage')
rsemgr.delete(rse_info, lfns=lfns)
return False
else:
logging.warning('Thread [%i/%i] : No dsn is specified' % (worker_number, total_workers))
try:
client.add_replicas(files=list_files, rse=rse)
client.add_replication_rule(list_files, copies=1, rse_expression=rse)
logging.info('Thread [%i/%i] : Upload operation for %s done' % (worker_number, total_workers, str(list_files)))
except Exception, e:
logging.error('Thread [%(worker_number)s/%(total_workers)s] : Failed to upload %(files)s' % locals())
logging.error('Thread [%i/%i] : %s' % (worker_number, total_workers, str(e)))
logging.error('Removing files from the Storage')
rsemgr.delete(rse_info, lfns=lfns)
return False
return True
def upload(files, scope, metadata, rse, account, source_dir, worker_number, total_workers, dataset_lifetime, did=None, set_metadata=False):
logging.debug('In upload')
dsn = None
if did:
dsn = {'scope': did.split(':')[0], 'name': did.split(':')[1]}
client = Client()
list_files = []
lfns = []
prepend_str = 'Thread [%i/%i] : ' % (worker_number, total_workers)
logging.debug(prepend_str + 'Looping over the files')
for filename in files:
fullpath = '%s/%s' % (source_dir, filename)
size = stat(fullpath).st_size
checksum = adler32(fullpath)
logging.info(prepend_str + 'File %s : Size %s , adler32 %s' % (fullpath, str(size), checksum))
list_files.append({'scope': scope, 'name': filename, 'bytes': size, 'adler32': checksum, 'meta': {'guid': generate_uuid()}})
lfns.append({'name': filename, 'scope': scope, 'filesize': size, 'adler32': checksum, 'filename': filename})
# Physical upload
logging.info(prepend_str + 'Uploading physically the files %s on %s' % (str(lfns), rse))
rse_info = rsemgr.get_rse_info(rse)
try:
success_upload = True
for cnt in range(0, 3):
rows = rsemgr.upload(rse_info, lfns=lfns, source_dir=source_dir)
# temporary hack
global_status, ret = rows['success'], rows[1]
logging.info(prepend_str + 'Returned global status : %s, Returned : %s' % (str(global_status), str(ret)))
if not global_status:
for item in ret:
if (not isinstance(ret[item], FileReplicaAlreadyExists)) and ret[item] is not True:
sleep(exp(cnt))
success_upload = False
logging.error(prepend_str + 'Problem to upload file %s with error %s' % (item, str(ret[item])))
break
else:
break
if not success_upload:
logging.error(prepend_str + 'Upload operation to %s failed, removing leftovers' % (rse))
rsemgr.delete(rse_info, lfns=lfns)
return False
except Exception as error:
logging.debug(traceback.format_exc())
logging.error(prepend_str + '%s' % (str(error)))
return False
logging.info(prepend_str + 'Files successfully copied on %s' % (rse))
# Registering DIDs and replicas in Rucio
logging.info(prepend_str + 'Registering DIDs and replicas in Rucio')
meta = metadata
if not set_metadata:
meta = None
if dsn:
try:
client.add_dataset(scope=dsn['scope'], name=dsn['name'], rules=[{'account': account, 'copies': 1, 'rse_expression': rse, 'grouping': 'DATASET', 'activity': 'Functional Test'}], meta=meta, lifetime=dataset_lifetime)
client.add_files_to_dataset(scope=dsn['scope'], name=dsn['name'], files=list_files, rse=rse)
logging.info(prepend_str + 'Upload operation for %s:%s done' % (dsn['scope'], dsn['name']))
except Exception as error:
logging.debug(traceback.format_exc())
logging.error(prepend_str + 'Failed to upload %(files)s' % locals())
logging.error(prepend_str + '%s' % (str(error)))
logging.error(prepend_str + 'Removing files from the Storage')
rsemgr.delete(rse_info, lfns=lfns)
return False
else:
logging.warning(prepend_str + 'No dsn is specified')
try:
client.add_replicas(files=list_files, rse=rse)
client.add_replication_rule(list_files, copies=1, rse_expression=rse, activity='Functional Test')
logging.info(prepend_str + 'Upload operation for %s done' % (str(list_files)))
except Exception as error:
logging.debug(traceback.format_exc())
logging.error(prepend_str + 'Failed to upload %(files)s' % locals())
logging.error(prepend_str + '%s' % (str(error)))
logging.error(prepend_str + 'Removing files from the Storage')
rsemgr.delete(rse_info, lfns=lfns)
return False
return True
'activity': 'Functional Test'
}],
meta=meta,
lifetime=dataset_lifetime)
client.add_files_to_dataset(scope=dsn['scope'],
name=dsn['name'],
files=list_files,
rse=rse)
logging.info(prepend_str + 'Upload operation for %s:%s done' %
(dsn['scope'], dsn['name']))
except Exception, error:
logging.error(prepend_str +
'Failed to upload %(files)s' % locals())
logging.error(prepend_str + '%s' % (str(error)))
logging.error(prepend_str + 'Removing files from the Storage')
rsemgr.delete(rse_info, lfns=lfns)
return False
else:
logging.warning(prepend_str + 'No dsn is specified')
try:
client.add_replicas(files=list_files, rse=rse)
client.add_replication_rule(list_files,
copies=1,
rse_expression=rse,
activity='Functional Test')
logging.info(prepend_str + 'Upload operation for %s done' %
(str(list_files)))
except Exception, error:
logging.error(prepend_str +
'Failed to upload %(files)s' % locals())
logging.error(prepend_str + '%s' % (str(error)))
def test_delete_mgr_SourceNotFound_single(self):
"""(RSE/PROTOCOLS): Delete a single file from storage (SourceNotFound)"""
mgr.delete(self.rse_settings, {'name': 'not_existing_data.raw', 'scope': 'user.%s' % self.user})
def test_delete_mgr_ok_single(self):
"""(RSE/PROTOCOLS): Delete a single file from storage (Success)"""
mgr.delete(self.rse_settings, {'name': '3_rse_remote_delete.raw', 'scope': 'user.%s' % self.user})
def test_delete_mgr_ok_multi(self):
"""(RSE/PROTOCOLS): Delete multiple files from storage (Success)"""
status, details = mgr.delete(self.rse_settings, [{'name': '1_rse_remote_delete.raw', 'scope': 'user.%s' % self.user}, {'name': '2_rse_remote_delete.raw', 'scope': 'user.%s' % self.user}])
if not (status and details['user.%s:1_rse_remote_delete.raw' % self.user] and details['user.%s:2_rse_remote_delete.raw' % self.user]):
raise Exception('Return not as expected: %s, %s' % (status, details))
