def __init__(self, dataset, site, rse=None, scope=DEFAULT_SCOPE, uuid=None, check=True, lifetime=None, dry_run=False): self.dataset = dataset self.site = site if rse is None: rse = site self.rse = rse self.scope = scope self.uuid = uuid self.check = check self.lifetime = lifetime self.dry_run = dry_run self.blocks = [] self.url = '' self.getmetadata() self.get_global_url() self.didc = DIDClient() self.repc = ReplicaClient() self.gfal = Gfal2Context()
def __init__(self, dataset, site, rse=None, scope=DEFAULT_SCOPE, uuid=None, check=True, lifetime=None, dry_run=False): # Working with default DAS Go client path super(DatasetInjector, self).__init__(account=None, auth_type=None, scope=scope, dry_run=dry_run) self.dataset = dataset self.site = site if rse is None: rse = site self.rse = rse self.uuid = uuid self.check = check self.lifetime = lifetime self.url = '' self.blocks = self.get_phedex_metadata(self.dataset, self.site) self.get_global_url() self.gfal = Gfal2Context()
def __init__(self, account, auth_type, creds=None, scope='cms', dry_run=False, das_go_path=DEFAULT_DASGOCLIENT, check=False): self.account = account self.auth_type = auth_type self.creds = creds self.scope = scope self.dry_run = dry_run self.dasgoclient = das_go_path self.check = check self.cli = Client(account=self.account, auth_type=self.auth_type, creds=self.creds) self.gfal = Gfal2Context()
def check_storage(filepath): """ Check size and checksum of a file on storage """ # FIXME: Gfal2Context cannot be pickled so we have to instantiate here to # use multiprocessing() gfal = Gfal2Context() logging.info("Checking url %s" % filepath) try: size = gfal.stat(str(filepath)).st_size checksum = gfal.checksum(str(filepath), 'adler32') logging.info("Got size and checksum of file: %s size=%s checksum=%s" % (filepath, size, checksum)) except GError: logging.warning("no file found at %s" % filepath) return False return size, checksum
def __init__(self, myscope, orgRse, destRse, account='bruzzese', working_folder=None): self.myscope = myscope self.orgRse = orgRse self.destRse = destRse self.working_folder = working_folder self.gfal = Gfal2Context() self.didc = DIDClient() self.repc = ReplicaClient() self.rulesClient = RuleClient() # Configuration self.account = account # account=account self.client = Client(account=self.account)
def __init__(self, run, originrse=DEFAULT_ORIGIN_RSE, destrse=None, scope=DEFAULT_SCOPE, check=True, lifetime=None, dry_run=False, container=None): """ :param dataset: Name of the PhEDEx dataset to synchronize with Rucio. :param pnn: PhEDEx node name to filter on for replica information. """ self.run = run self.originrse = originrse self.destrse = destrse self.scope = scope self.check = check self.lifetime = lifetime self.dry_run = dry_run self.container = container self.rucio_datasets = {} self.run_files = {} self.existent_replica_files = {} self.url = '' self.gfal = Gfal2Context() self.run_Number = None self.get_run_Number() self.files_storage = {} self.get_global_url() self.didc = DIDClient() self.repc = ReplicaClient() self.rulesClient = RuleClient() # Right now obtaining the Metadata from the storage at WIPAC # Hopefully in the future from JADE # TODO self.get_run_Files() self.get_rucio_metadata() self.update_run_Files() self.get_files_metadata()
downloadClient = DownloadClient() rulesClient = RuleClient() ## user scope Default_Scope = 'test-root' # Get list of all RSEs default_rses = list(client.list_rses()) rses_lists = [] for single_rse in default_rses: rses_lists.append(single_rse['rse']) print(rses_lists) # Gfal settings gfal = Gfal2Context() print(json.dumps(client.whoami(), indent=4, sort_keys=True)) print(json.dumps(client.ping(), indent=4, sort_keys=True)) sys.path.append("/usr/lib64/python3.6/site-packages/") import gfal2 from gfal2 import Gfal2Context, GError gfal2.set_verbose(gfal2.verbose_level.debug) gfal = Gfal2Context() # In[3]: # Specific Functions for Rucio
def __init__(self): self.gfal = Gfal2Context()
RSE = rsemgr.get_rse_info(OPTIONS.rse) # Use the first protocol PROTO = RSE['protocols'][0] # Get the replica url SCHEMA = PROTO['scheme'] PREFIX = PROTO['prefix'] + '/' + OPTIONS.scope.replace('.', '/') if SCHEMA == 'srm': PREFIX = PROTO['extended_attributes']['web_service_path'] + PREFIX URL = SCHEMA + '://' + PROTO['hostname'] if PROTO['port'] != 0: URL = URL + ':' + str(PROTO['port']) URL = URL + PREFIX + '/' + OPTIONS.name GFAL = Gfal2Context() try: SIZE = GFAL.stat(str(URL)).st_size CHECKSUM = GFAL.checksum(str(URL), 'adler32') print("Registering file: pfn=%s size=%s checksum=%s" % (URL, SIZE, CHECKSUM)) except GError: print("no file found at %s" % URL) exit() R = ReplicaClient() REPLICAS = list( R.list_replicas([{ 'scope': OPTIONS.scope,