def __init__(self, pnn, account, auth_type=None, rsetype=DEFAULT_RSETYPE, suffix=None,
dry=False, fts=None, tier=None, lfn2pfn_algorithm=None, country=None,
attrs=None, seinfo=None, tfc=None, tfc_exclude=EXCLUDE_TFC, domains=None,
space_token=None, add_prefix=None, proto=DEFAULT_PROTOCOL,
instance=DEFAULT_PHEDEX_INST, dasgoclient=DEFAULT_DASGOCLIENT,
datasvc=DEFAULT_DATASVC_URL):
attrs = attrs or []
self.pnn = pnn
self.rsetype = rsetype
if suffix is None:
suffix = DEFAULT_SUFFIXES[rsetype]
self.suffix = suffix
self.rsename = pnn + self.suffix
if tfc and os.path.isdir(tfc):
self.tfc = tfc + '/' + pnn + '/PhEDEx/storage.xml'
else:
self.tfc = tfc
self.pcli = PhEDEx(instance=instance, dasgoclient=dasgoclient, datasvc=datasvc)
self.rcli = Client(account=account, auth_type=auth_type)
self.dry = dry
self._get_attributes(fts, tier, lfn2pfn_algorithm, country, attrs)
self._get_settings()
self._get_protocol(seinfo, add_prefix, tfc_exclude, domains, space_token, proto)
def __init__(self,
pnn,
account,
auth_type=None,
rsetype=DEFAULT_RSETYPE,
suffix=None,
dry=False,
fts=None,
tier=None,
lfn2pfn_algorithm=None,
country=None,
attrs=None,
seinfo=None,
tfc=None,
tfc_exclude=EXCLUDE_TFC,
domains=None,
space_token=None,
add_prefix=None,
proto=DEFAULT_PROTOCOL,
instance=DEFAULT_PHEDEX_INST,
dasgoclient=DEFAULT_DASGOCLIENT,
datasvc=DEFAULT_DATASVC_URL):
attrs = attrs or []
self.pnn = pnn
self.rsetype = rsetype
if suffix is None:
suffix = DEFAULT_SUFFIXES[rsetype]
self.suffix = suffix
if pnn.endswith('_MSS'):
raise ValueError(
'Please import PhEDEx _Buffer pnns rather than _MSS for tape endpoints'
)
elif pnn.endswith('_Buffer'):
self.rsename = pnn.replace('_Buffer', '_Tape') + self.suffix
self.rucio_rse_type = 'TAPE'
else:
self.rsename = pnn + self.suffix
self.rucio_rse_type = 'DISK'
if tfc and os.path.isdir(tfc):
self.tfc = tfc + '/' + pnn + '/PhEDEx/storage.xml'
else:
self.tfc = tfc
self.pcli = PhEDEx(instance=instance,
dasgoclient=dasgoclient,
datasvc=datasvc)
self.rcli = Client(account=account, auth_type=auth_type)
self.dry = dry
self._get_attributes(fts, tier, lfn2pfn_algorithm, country, attrs)
self._get_settings()
self._get_protocol(seinfo, add_prefix, tfc_exclude, domains,
space_token, proto)
def __init__(self, options):
self.options = options
self.config = load_config(options.config)
self.last_synced = {} # load_last_synced()
self.phedex_svc = PhEDEx()
pass
def _get_pcli(self, pcli):
if pcli is None:
pcli = {}
if isinstance(pcli, dict):
self.pcli = PhEDEx(**pcli)
elif isinstance(pcli, PhEDEx):
# pylint: disable=redefined-variable-type
self.pcli = pcli
else:
raise Exception("wrong type for pcli parameter %s" % type(pcli))
def __init__(self,
block_name,
pnn,
rse=None,
lifetime=None,
dry_run=False):
"""
Get the status of replica of pditem at pnn
considering only closed blocks completely replicated at site.
:rds: PhEDEx block name.
:pnn: PhEDEx node name.
:rse: Rucio RSE. If None (default) inferred by the pnn using DEFAULT_RSE_FMT.
:scope: Scope. Default: DEFAULT_SCOPE.
"""
self.phedex_svc = PhEDEx()
self.dry_run = dry_run
self.pnn = pnn
if rse is None:
self.rse = list_rses('cms_type=real&pnn=%s' % self.pnn)[0]['rse']
else:
self.rse = rse
rse_details = get_rse(self.rse)
self.rse_id = rse_details['id']
self.account = (SYNC_ACCOUNT_FMT % self.rse.lower())[:25]
self.container = self.phedex_svc.check_data_item(
pditem=block_name)['pds']
self.scope = DEFAULT_SCOPE
self.block_name = block_name
self.lifetime = lifetime
self.group, self.custodial, self.is_at_pnn = self.phedex_svc.block_at_pnn_phedex(
block=self.block_name, pnn=self.pnn)
self.block_in_phedex = self.phedex_svc.block_exists(
block=self.block_name)
self.block_known = self.phedex_svc.block_known(block=self.block_name)
if self.is_at_pnn:
self.replicas = self.phedex_svc.fileblock_files_phedex(
pnn=pnn, pfb=block_name)
else:
self.replicas = {}
self.container_exists = None
self.block_exists = None
self.rule_exists = None
touch(text=self.rse)
def __init__(self,
account,
auth_type=None,
exclude=DEFAULT_EXCLUDE_LINKS,
distance=None,
phedex_links=False,
rselist=None,
instance=DEFAULT_PHEDEX_INST,
datasvc=DEFAULT_DATASVC_URL):
if distance is None:
distance = DEFAULT_DISTANCE_RULES
self.pcli = PhEDEx(instance=instance, datasvc=datasvc)
self.rcli = Client(account=account, auth_type=auth_type)
self._get_rselist(rselist)
self._get_matrix(distance, phedex_links, exclude)
def sync(config, logs):
"""
Main Sync process
"""
logging.my_logfile(logs=logs)
logging.my_fmt(label='main_sync')
starttime = datetime.now()
modify = {}
workers = {} # this is the array of running pnns
pnns = None # this is the array of pnn to be launched
pool = None
pcli = PhEDEx()
install_mp_handler()
conf = _load_config(config, modify, starttime)
pnns = []
size = conf['main']['pool']
logging.summary('Starting')
while conf['main']['run']:
if pool is None:
logging.notice('Started pool of size %d', size)
pool = multiprocessing.NDPool(size)
add = [
pnn for pnn, sec in conf.items() if pnn != 'main' if sec['run']
if pnn not in workers if pnn not in pnns
]
pnns += add
random.shuffle(pnns)
if not _ping():
logging.warning('Cannot ping, not launching workers')
else:
_launch_workers(pool, workers, pnns, pcli)
pnns = []
_poll_workers(workers, pnns)
conf = _load_config(config, modify, starttime)
if not conf['main']['run'] or\
conf['main']['pool'] != size:
# trigger draining of all workers, close the pool and wait
# for the task to be over
conf = _load_config(config, {'default': {'run': False}}, starttime)
_drain_up(workers, pnns)
workers = {}
pool.close()
pool = None
size = conf['main']['pool']
else:
time.sleep(conf['main']['sleep'])
logging.summary('Exiting.')
return config
help='dataset to be updates. Can have wildcard and can be multiple')
PARSER.add_argument('--pool',
dest='pool',
default=1,
help='number of parallel threads. Default 1.')
OPTIONS = PARSER.parse_args()
logging.my_lvl(OPTIONS.verbosity)
# logging.summary('DBP1')
install_mp_handler()
POOL = multiprocessing.Pool(int(OPTIONS.pool))
PCLI = PhEDEx()
PNNS = PCLI.pnns(select=OPTIONS.pnn)
TIMING = {}
WILDCARD = re.compile(r'\S*[*]\S*')
DATASETS = get_timing(_get_dset_list(PCLI, OPTIONS.dataset), TIMING)
PROCS = get_timing(_launch_workers(PNNS, DATASETS, POOL, OPTIONS, PCLI),
TIMING)
get_timing(_get_workers(POOL, PROCS), TIMING)
logging.summary(
OPTIONS = PARSER.parse_args()
if OPTIONS.debug:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.INFO)
if OPTIONS.domains:
OPTIONS.domains = json.loads(OPTIONS.domains.replace("'", '"'))
if OPTIONS.seinfo is not None:
OPTIONS.seinfo = json.loads(OPTIONS.seinfo.replace("'", '"'))
if 'all' in OPTIONS.pnn:
OPTIONS.pnn = PhEDEx(instance=OPTIONS.instance).pnns(
select=OPTIONS.select, exclude=OPTIONS.exclude)
CHANGED = []
TOT = []
for node_name in OPTIONS.pnn:
for rse_type in OPTIONS.type:
logging.info('Starting pnn %s and type %s', node_name, rse_type)
RSE = CMSRSE(pnn=node_name,
rsetype=rse_type,
account=OPTIONS.account,
dry=OPTIONS.dry,
suffix=OPTIONS.suffix,
fts=OPTIONS.fts,
tier=OPTIONS.tier,
lfn2pfn_algorithm=OPTIONS.lfn2pfn,