def __init__(self,
dataset,
site,
rse=None,
scope=DEFAULT_SCOPE,
uuid=None,
check=True,
lifetime=None,
dry_run=False):
self.dataset = dataset
self.site = site
if rse is None:
rse = site
self.rse = rse
self.scope = scope
self.uuid = uuid
self.check = check
self.lifetime = lifetime
self.dry_run = dry_run
self.blocks = []
self.url = ''
self.getmetadata()
self.get_global_url()
self.didc = DIDClient()
self.repc = ReplicaClient()
self.gfal = Gfal2Context()
def __init__(self,
dataset,
site,
rse=None,
scope=DEFAULT_SCOPE,
uuid=None,
check=True,
lifetime=None,
dry_run=False):
# Working with default DAS Go client path
super(DatasetInjector, self).__init__(account=None,
auth_type=None,
scope=scope,
dry_run=dry_run)
self.dataset = dataset
self.site = site
if rse is None:
rse = site
self.rse = rse
self.uuid = uuid
self.check = check
self.lifetime = lifetime
self.url = ''
self.blocks = self.get_phedex_metadata(self.dataset, self.site)
self.get_global_url()
self.gfal = Gfal2Context()
def __init__(self, account, auth_type, creds=None, scope='cms', dry_run=False,
das_go_path=DEFAULT_DASGOCLIENT, check=False):
self.account = account
self.auth_type = auth_type
self.creds = creds
self.scope = scope
self.dry_run = dry_run
self.dasgoclient = das_go_path
self.check = check
self.cli = Client(account=self.account, auth_type=self.auth_type, creds=self.creds)
self.gfal = Gfal2Context()
def check_storage(filepath):
"""
Check size and checksum of a file on storage
"""
# FIXME: Gfal2Context cannot be pickled so we have to instantiate here to
# use multiprocessing()
gfal = Gfal2Context()
logging.info("Checking url %s" % filepath)
try:
size = gfal.stat(str(filepath)).st_size
checksum = gfal.checksum(str(filepath), 'adler32')
logging.info("Got size and checksum of file: %s size=%s checksum=%s" %
(filepath, size, checksum))
except GError:
logging.warning("no file found at %s" % filepath)
return False
return size, checksum
def __init__(self, myscope, orgRse, destRse, account='bruzzese', working_folder=None):
self.myscope = myscope
self.orgRse = orgRse
self.destRse = destRse
self.working_folder = working_folder
self.gfal = Gfal2Context()
self.didc = DIDClient()
self.repc = ReplicaClient()
self.rulesClient = RuleClient()
# Configuration
self.account = account
# account=account
self.client = Client(account=self.account)
def __init__(self,
run,
originrse=DEFAULT_ORIGIN_RSE,
destrse=None,
scope=DEFAULT_SCOPE,
check=True,
lifetime=None,
dry_run=False,
container=None):
"""
:param dataset: Name of the PhEDEx dataset to synchronize with Rucio.
:param pnn: PhEDEx node name to filter on for replica information.
"""
self.run = run
self.originrse = originrse
self.destrse = destrse
self.scope = scope
self.check = check
self.lifetime = lifetime
self.dry_run = dry_run
self.container = container
self.rucio_datasets = {}
self.run_files = {}
self.existent_replica_files = {}
self.url = ''
self.gfal = Gfal2Context()
self.run_Number = None
self.get_run_Number()
self.files_storage = {}
self.get_global_url()
self.didc = DIDClient()
self.repc = ReplicaClient()
self.rulesClient = RuleClient()
# Right now obtaining the Metadata from the storage at WIPAC
# Hopefully in the future from JADE # TODO
self.get_run_Files()
self.get_rucio_metadata()
self.update_run_Files()
self.get_files_metadata()
downloadClient = DownloadClient()
rulesClient = RuleClient()
## user scope
Default_Scope = 'test-root'
# Get list of all RSEs
default_rses = list(client.list_rses())
rses_lists = []
for single_rse in default_rses:
rses_lists.append(single_rse['rse'])
print(rses_lists)
# Gfal settings
gfal = Gfal2Context()
print(json.dumps(client.whoami(), indent=4, sort_keys=True))
print(json.dumps(client.ping(), indent=4, sort_keys=True))
sys.path.append("/usr/lib64/python3.6/site-packages/")
import gfal2
from gfal2 import Gfal2Context, GError
gfal2.set_verbose(gfal2.verbose_level.debug)
gfal = Gfal2Context()
# In[3]:
# Specific Functions for Rucio
def __init__(self):
self.gfal = Gfal2Context()
RSE = rsemgr.get_rse_info(OPTIONS.rse)
# Use the first protocol
PROTO = RSE['protocols'][0]
# Get the replica url
SCHEMA = PROTO['scheme']
PREFIX = PROTO['prefix'] + '/' + OPTIONS.scope.replace('.', '/')
if SCHEMA == 'srm':
PREFIX = PROTO['extended_attributes']['web_service_path'] + PREFIX
URL = SCHEMA + '://' + PROTO['hostname']
if PROTO['port'] != 0:
URL = URL + ':' + str(PROTO['port'])
URL = URL + PREFIX + '/' + OPTIONS.name
GFAL = Gfal2Context()
try:
SIZE = GFAL.stat(str(URL)).st_size
CHECKSUM = GFAL.checksum(str(URL), 'adler32')
print("Registering file: pfn=%s size=%s checksum=%s" %
(URL, SIZE, CHECKSUM))
except GError:
print("no file found at %s" % URL)
exit()
R = ReplicaClient()
REPLICAS = list(
R.list_replicas([{
'scope': OPTIONS.scope,
