def test_delete_mgr_SourceNotFound_single(self): """(RSE/PROTOCOLS): Delete a single file from storage (SourceNotFound)""" mgr.delete(self.rse_settings, { 'name': 'not_existing_data.raw', 'scope': 'user.%s' % self.user }, impl=self.impl)
def test_delete_mgr_ok_single(self): """(RSE/PROTOCOLS): Delete a single file from storage (Success)""" mgr.delete(self.rse_settings, { 'name': '3_rse_remote_delete.raw', 'scope': 'user.%s' % self.user }, impl=self.impl)
def test_delete_mgr_ok_dir(self): """(RSE/PROTOCOLS): Delete a directory from storage (Success)""" mgr.delete( self.rse_settings, { 'path': 'user/%s' % self.user, 'name': 'user.%s' % self.user, 'scope': 'user.%s' % self.user })
def test_delete_mgr_SourceNotFound_multi(self): """(RSE/PROTOCOLS): Delete multiple files from storage (SourceNotFound)""" status, details = mgr.delete(self.rse_settings, [{'name': 'not_existing_data.raw', 'scope': 'user.%s' % self.user}, {'name': '4_rse_remote_delete.raw', 'scope': 'user.%s' % self.user}]) if details['user.%s:4_rse_remote_delete.raw' % self.user]: raise details['user.%s:not_existing_data.raw' % self.user] else: raise Exception('Return not as expected: %s, %s' % (status, details))
def test_delete_mgr_ok_multi(self): """(RSE/PROTOCOLS): Delete multiple files from storage (Success)""" status, details = mgr.delete(self.rse_settings, [{'name': '1_rse_remote_delete.raw', 'scope': 'user.%s' % self.user}, {'name': '2_rse_remote_delete.raw', 'scope': 'user.%s' % self.user}]) if not (status and details['user.%s:1_rse_remote_delete.raw' % self.user] and details['user.%s:2_rse_remote_delete.raw' % self.user]): if isinstance(details['user.%s:1_rse_remote_delete.raw' % self.user], NotImplementedError) and isinstance(details['user.%s:2_rse_remote_delete.raw' % self.user], NotImplementedError): pass else: raise Exception('Return not as expected: %s, %s' % (status, details))
def upload(files, scope, metadata, rse, account, source_dir, worker_number, total_workers, dataset_lifetime, did=None, set_metadata=False): logging.debug('In upload') dsn = None if did: dsn = {'scope': did.split(':')[0], 'name': did.split(':')[1]} client = Client() list_files = [] lfns = [] prepend_str = 'Thread [%i/%i] : ' % (worker_number, total_workers) logging.debug(prepend_str + 'Looping over the files') for filename in files: fullpath = '%s/%s' % (source_dir, filename) size = stat(fullpath).st_size checksum = adler32(fullpath) logging.info(prepend_str + 'File %s : Size %s , adler32 %s' % (fullpath, str(size), checksum)) list_files.append({'scope': scope, 'name': filename, 'bytes': size, 'adler32': checksum, 'meta': {'guid': generate_uuid()}}) lfns.append({'name': filename, 'scope': scope, 'filesize': size, 'adler32': checksum, 'filename': filename}) # Physical upload logging.info(prepend_str + 'Uploading physically the files %s on %s' % (str(lfns), rse)) rse_info = rsemgr.get_rse_info(rse) try: success_upload = True for cnt in xrange(0, 3): global_status, ret = rsemgr.upload(rse_info, lfns=lfns, source_dir=source_dir) logging.info(prepend_str + 'Returned global status : %s, Returned : %s' % (str(global_status), str(ret))) if not global_status: for item in ret: if (not isinstance(ret[item], FileReplicaAlreadyExists)) and ret[item] is not True: sleep(exp(cnt)) success_upload = False logging.error(prepend_str + 'Problem to upload file %s with error %s' % (item, str(ret[item]))) break else: break if not success_upload: logging.error(prepend_str + 'Upload operation to %s failed, removing leftovers' % (rse)) rsemgr.delete(rse_info, lfns=lfns) return False except Exception, error: logging.error(prepend_str + '%s' % (str(error))) return False
def upload(files, scope, metadata, rse, account, source_dir, worker_number, total_workers, dataset_lifetime, did=None): logging.debug('In upload') dsn = None if did: dsn = {'scope': did.split(':')[0], 'name': did.split(':')[1]} client = Client() list_files = [] lfns = [] logging.debug('Thread [%i/%i] : Looping over the files' % (worker_number, total_workers)) for filename in files: fullpath = '%s/%s' % (source_dir, filename) size = stat(fullpath).st_size checksum = adler32(fullpath) logging.info('Thread [%i/%i] : File %s : Size %s , adler32 %s' % (worker_number, total_workers, fullpath, str(size), checksum)) list_files.append({'scope': scope, 'name': filename, 'bytes': size, 'adler32': checksum, 'meta': {'guid': generate_uuid()}}) lfns.append({'name': filename, 'scope': scope, 'filesize': size, 'adler32': checksum}) # Physical upload logging.info('Thread [%i/%i] : Uploading physically the files %s on %s' % (worker_number, total_workers, str(lfns), rse)) rse_info = rsemgr.get_rse_info(rse) try: success_upload = True for i in xrange(0, 3): gs, ret = rsemgr.upload(rse_info, lfns=lfns, source_dir=source_dir) logging.info('Returned global status : %s, Returned : %s' % (str(gs), str(ret))) if not gs: for x in ret: if (not isinstance(ret[x], FileReplicaAlreadyExists)) and ret[x] is not True: sleep(exp(i)) success_upload = False logging.error('Problem to upload file %s with error %s' % (x, str(ret[x]))) break else: break if not success_upload: logging.error('Thread [%i/%i] : Upload operation to %s failed, removing leftovers' % (worker_number, total_workers, rse)) rsemgr.delete(rse_info, lfns=lfns) return False except Exception, e: return False
def upload(files, scope, metadata, rse, account, source_dir, dataset_lifetime, did=None, set_metadata=False, logger=logging.log): logger(logging.DEBUG, 'In upload') dsn = None if did: dsn = {'scope': did.split(':')[0], 'name': did.split(':')[1]} client = Client() list_files = [] lfns = [] for filename in files: physical_fname = filename if physical_fname.find('/') > -1: physical_fname = "".join(filename.split('/')) fullpath = '%s/%s' % (source_dir, physical_fname) size = stat(fullpath).st_size checksum = adler32(fullpath) logger(logging.INFO, 'File %s : Size %s , adler32 %s', fullpath, str(size), checksum) list_files.append({ 'scope': scope, 'name': filename, 'bytes': size, 'adler32': checksum, 'meta': { 'guid': generate_uuid() } }) lfns.append({ 'name': filename, 'scope': scope, 'filesize': size, 'adler32': checksum, 'filename': physical_fname }) # Physical upload logger(logging.INFO, 'Uploading physically the files %s on %s', str(lfns), rse) rse_info = rsemgr.get_rse_info(rse, vo=client.vo) try: success_upload = True for cnt in range(0, 3): rows = rsemgr.upload(rse_info, lfns=lfns, source_dir=source_dir, logger=logger) # temporary hack global_status, ret = rows['success'], rows[1] logger(logging.INFO, 'Returned global status : %s, Returned : %s', str(global_status), str(ret)) if not global_status: for item in ret: if (not isinstance(ret[item], FileReplicaAlreadyExists) ) and ret[item] is not True: sleep(exp(cnt)) success_upload = False logger(logging.ERROR, 'Problem to upload file %s with error %s', item, str(ret[item])) break else: break if not success_upload: logger(logging.ERROR, 'Upload operation to %s failed, removing leftovers', rse) rsemgr.delete(rse_info, lfns=lfns) return False except Exception as error: logger(logging.DEBUG, "Exception", exc_info=True) logger(logging.ERROR, '%s', str(error)) return False logger(logging.INFO, 'Files successfully copied on %s', rse) # Registering DIDs and replicas in Rucio logger(logging.INFO, 'Registering DIDs and replicas in Rucio') meta = metadata if not set_metadata: meta = None if dsn: try: client.add_dataset(scope=dsn['scope'], name=dsn['name'], rules=[{ 'account': account, 'copies': 1, 'rse_expression': rse, 'grouping': 'DATASET', 'activity': 'Functional Test' }], meta=meta, lifetime=dataset_lifetime) client.add_files_to_dataset(scope=dsn['scope'], name=dsn['name'], files=list_files, rse=rse) logger(logging.INFO, 'Upload operation for %s:%s done', dsn['scope'], dsn['name']) except Exception as error: logger(logging.DEBUG, "Exception", exc_info=True) logger(logging.ERROR, 'Failed to upload %s', str(list_files)) logger(logging.ERROR, '%s', str(error)) logger(logging.ERROR, 'removing files from the Storage') rsemgr.delete(rse_info, lfns=lfns) return False else: logger(logging.WARNING, 'No dsn is specified') try: client.add_replicas(files=list_files, rse=rse) client.add_replication_rule(list_files, copies=1, rse_expression=rse, activity='Functional Test') logger(logging.INFO, 'Upload operation for %s done', str(list_files)) except Exception as error: logger(logging.DEBUG, "Exception", exc_info=True) logger(logging.ERROR, 'Failed to upload %s', str(list_files)) logger(logging.ERROR, '%s', str(error)) logger(logging.ERROR, 'Removing files from the Storage') rsemgr.delete(rse_info, lfns=lfns) return False return True
except Exception, e: return False logging.info('Thread [%i/%i] : Files successfully copied on %s' % (worker_number, total_workers, rse)) # Registering DIDs and replicas in Rucio logging.info('Thread [%i/%i] : Registering DIDs and replicas in Rucio' % (worker_number, total_workers)) if dsn: try: client.add_dataset(scope=dsn['scope'], name=dsn['name'], rules=[{'account': account, 'copies': 1, 'rse_expression': rse, 'grouping': 'DATASET'}], meta=metadata, lifetime=dataset_lifetime) client.add_files_to_dataset(scope=dsn['scope'], name=dsn['name'], files=list_files, rse=rse) logging.info('Thread [%i/%i] : Upload operation for %s:%s done' % (worker_number, total_workers, dsn['scope'], dsn['name'])) except Exception, e: logging.error('Thread [%(worker_number)s/%(total_workers)s] : Failed to upload %(files)s' % locals()) logging.error('Thread [%i/%i] : %s' % (worker_number, total_workers, str(e))) logging.error('Removing files from the Storage') rsemgr.delete(rse_info, lfns=lfns) return False else: logging.warning('Thread [%i/%i] : No dsn is specified' % (worker_number, total_workers)) try: client.add_replicas(files=list_files, rse=rse) client.add_replication_rule(list_files, copies=1, rse_expression=rse) logging.info('Thread [%i/%i] : Upload operation for %s done' % (worker_number, total_workers, str(list_files))) except Exception, e: logging.error('Thread [%(worker_number)s/%(total_workers)s] : Failed to upload %(files)s' % locals()) logging.error('Thread [%i/%i] : %s' % (worker_number, total_workers, str(e))) logging.error('Removing files from the Storage') rsemgr.delete(rse_info, lfns=lfns) return False return True
def upload(files, scope, metadata, rse, account, source_dir, worker_number, total_workers, dataset_lifetime, did=None, set_metadata=False): logging.debug('In upload') dsn = None if did: dsn = {'scope': did.split(':')[0], 'name': did.split(':')[1]} client = Client() list_files = [] lfns = [] prepend_str = 'Thread [%i/%i] : ' % (worker_number, total_workers) logging.debug(prepend_str + 'Looping over the files') for filename in files: fullpath = '%s/%s' % (source_dir, filename) size = stat(fullpath).st_size checksum = adler32(fullpath) logging.info(prepend_str + 'File %s : Size %s , adler32 %s' % (fullpath, str(size), checksum)) list_files.append({'scope': scope, 'name': filename, 'bytes': size, 'adler32': checksum, 'meta': {'guid': generate_uuid()}}) lfns.append({'name': filename, 'scope': scope, 'filesize': size, 'adler32': checksum, 'filename': filename}) # Physical upload logging.info(prepend_str + 'Uploading physically the files %s on %s' % (str(lfns), rse)) rse_info = rsemgr.get_rse_info(rse) try: success_upload = True for cnt in range(0, 3): rows = rsemgr.upload(rse_info, lfns=lfns, source_dir=source_dir) # temporary hack global_status, ret = rows['success'], rows[1] logging.info(prepend_str + 'Returned global status : %s, Returned : %s' % (str(global_status), str(ret))) if not global_status: for item in ret: if (not isinstance(ret[item], FileReplicaAlreadyExists)) and ret[item] is not True: sleep(exp(cnt)) success_upload = False logging.error(prepend_str + 'Problem to upload file %s with error %s' % (item, str(ret[item]))) break else: break if not success_upload: logging.error(prepend_str + 'Upload operation to %s failed, removing leftovers' % (rse)) rsemgr.delete(rse_info, lfns=lfns) return False except Exception as error: logging.debug(traceback.format_exc()) logging.error(prepend_str + '%s' % (str(error))) return False logging.info(prepend_str + 'Files successfully copied on %s' % (rse)) # Registering DIDs and replicas in Rucio logging.info(prepend_str + 'Registering DIDs and replicas in Rucio') meta = metadata if not set_metadata: meta = None if dsn: try: client.add_dataset(scope=dsn['scope'], name=dsn['name'], rules=[{'account': account, 'copies': 1, 'rse_expression': rse, 'grouping': 'DATASET', 'activity': 'Functional Test'}], meta=meta, lifetime=dataset_lifetime) client.add_files_to_dataset(scope=dsn['scope'], name=dsn['name'], files=list_files, rse=rse) logging.info(prepend_str + 'Upload operation for %s:%s done' % (dsn['scope'], dsn['name'])) except Exception as error: logging.debug(traceback.format_exc()) logging.error(prepend_str + 'Failed to upload %(files)s' % locals()) logging.error(prepend_str + '%s' % (str(error))) logging.error(prepend_str + 'Removing files from the Storage') rsemgr.delete(rse_info, lfns=lfns) return False else: logging.warning(prepend_str + 'No dsn is specified') try: client.add_replicas(files=list_files, rse=rse) client.add_replication_rule(list_files, copies=1, rse_expression=rse, activity='Functional Test') logging.info(prepend_str + 'Upload operation for %s done' % (str(list_files))) except Exception as error: logging.debug(traceback.format_exc()) logging.error(prepend_str + 'Failed to upload %(files)s' % locals()) logging.error(prepend_str + '%s' % (str(error))) logging.error(prepend_str + 'Removing files from the Storage') rsemgr.delete(rse_info, lfns=lfns) return False return True
'activity': 'Functional Test' }], meta=meta, lifetime=dataset_lifetime) client.add_files_to_dataset(scope=dsn['scope'], name=dsn['name'], files=list_files, rse=rse) logging.info(prepend_str + 'Upload operation for %s:%s done' % (dsn['scope'], dsn['name'])) except Exception, error: logging.error(prepend_str + 'Failed to upload %(files)s' % locals()) logging.error(prepend_str + '%s' % (str(error))) logging.error(prepend_str + 'Removing files from the Storage') rsemgr.delete(rse_info, lfns=lfns) return False else: logging.warning(prepend_str + 'No dsn is specified') try: client.add_replicas(files=list_files, rse=rse) client.add_replication_rule(list_files, copies=1, rse_expression=rse, activity='Functional Test') logging.info(prepend_str + 'Upload operation for %s done' % (str(list_files))) except Exception, error: logging.error(prepend_str + 'Failed to upload %(files)s' % locals()) logging.error(prepend_str + '%s' % (str(error)))
def test_delete_mgr_SourceNotFound_single(self): """(RSE/PROTOCOLS): Delete a single file from storage (SourceNotFound)""" mgr.delete(self.rse_settings, {'name': 'not_existing_data.raw', 'scope': 'user.%s' % self.user})
def test_delete_mgr_ok_single(self): """(RSE/PROTOCOLS): Delete a single file from storage (Success)""" mgr.delete(self.rse_settings, {'name': '3_rse_remote_delete.raw', 'scope': 'user.%s' % self.user})
def test_delete_mgr_ok_multi(self): """(RSE/PROTOCOLS): Delete multiple files from storage (Success)""" status, details = mgr.delete(self.rse_settings, [{'name': '1_rse_remote_delete.raw', 'scope': 'user.%s' % self.user}, {'name': '2_rse_remote_delete.raw', 'scope': 'user.%s' % self.user}]) if not (status and details['user.%s:1_rse_remote_delete.raw' % self.user] and details['user.%s:2_rse_remote_delete.raw' % self.user]): raise Exception('Return not as expected: %s, %s' % (status, details))