예제 #1
0
    def __init__(self, pnn, account, auth_type=None, rsetype=DEFAULT_RSETYPE, suffix=None,
                 dry=False, fts=None, tier=None, lfn2pfn_algorithm=None, country=None,
                 attrs=None, seinfo=None, tfc=None, tfc_exclude=EXCLUDE_TFC, domains=None,
                 space_token=None, add_prefix=None, proto=DEFAULT_PROTOCOL,
                 instance=DEFAULT_PHEDEX_INST, dasgoclient=DEFAULT_DASGOCLIENT,
                 datasvc=DEFAULT_DATASVC_URL):

        attrs = attrs or []

        self.pnn = pnn
        self.rsetype = rsetype
        if suffix is None:
            suffix = DEFAULT_SUFFIXES[rsetype]

        self.suffix = suffix
        self.rsename = pnn + self.suffix

        if tfc and os.path.isdir(tfc):
            self.tfc = tfc + '/' + pnn + '/PhEDEx/storage.xml'
        else:
            self.tfc = tfc

        self.pcli = PhEDEx(instance=instance, dasgoclient=dasgoclient, datasvc=datasvc)
        self.rcli = Client(account=account, auth_type=auth_type)

        self.dry = dry

        self._get_attributes(fts, tier, lfn2pfn_algorithm, country, attrs)

        self._get_settings()

        self._get_protocol(seinfo, add_prefix, tfc_exclude, domains, space_token, proto)
예제 #2
0
    def __init__(self,
                 pnn,
                 account,
                 auth_type=None,
                 rsetype=DEFAULT_RSETYPE,
                 suffix=None,
                 dry=False,
                 fts=None,
                 tier=None,
                 lfn2pfn_algorithm=None,
                 country=None,
                 attrs=None,
                 seinfo=None,
                 tfc=None,
                 tfc_exclude=EXCLUDE_TFC,
                 domains=None,
                 space_token=None,
                 add_prefix=None,
                 proto=DEFAULT_PROTOCOL,
                 instance=DEFAULT_PHEDEX_INST,
                 dasgoclient=DEFAULT_DASGOCLIENT,
                 datasvc=DEFAULT_DATASVC_URL):

        attrs = attrs or []

        self.pnn = pnn
        self.rsetype = rsetype
        if suffix is None:
            suffix = DEFAULT_SUFFIXES[rsetype]

        self.suffix = suffix
        if pnn.endswith('_MSS'):
            raise ValueError(
                'Please import PhEDEx _Buffer pnns rather than _MSS for tape endpoints'
            )
        elif pnn.endswith('_Buffer'):
            self.rsename = pnn.replace('_Buffer', '_Tape') + self.suffix
            self.rucio_rse_type = 'TAPE'
        else:
            self.rsename = pnn + self.suffix
            self.rucio_rse_type = 'DISK'

        if tfc and os.path.isdir(tfc):
            self.tfc = tfc + '/' + pnn + '/PhEDEx/storage.xml'
        else:
            self.tfc = tfc

        self.pcli = PhEDEx(instance=instance,
                           dasgoclient=dasgoclient,
                           datasvc=datasvc)
        self.rcli = Client(account=account, auth_type=auth_type)

        self.dry = dry

        self._get_attributes(fts, tier, lfn2pfn_algorithm, country, attrs)

        self._get_settings()

        self._get_protocol(seinfo, add_prefix, tfc_exclude, domains,
                           space_token, proto)
예제 #3
0
    def __init__(self, options):
        self.options = options

        self.config = load_config(options.config)
        self.last_synced = {}  # load_last_synced()
        self.phedex_svc = PhEDEx()

        pass
예제 #4
0
    def _get_pcli(self, pcli):
        if pcli is None:
            pcli = {}

        if isinstance(pcli, dict):
            self.pcli = PhEDEx(**pcli)
        elif isinstance(pcli, PhEDEx):
            # pylint: disable=redefined-variable-type
            self.pcli = pcli
        else:
            raise Exception("wrong type for pcli parameter %s" % type(pcli))
예제 #5
0
    def __init__(self,
                 block_name,
                 pnn,
                 rse=None,
                 lifetime=None,
                 dry_run=False):
        """
        Get the status of replica of pditem at pnn
        considering only closed blocks completely replicated at site.

        :rds:    PhEDEx block name.
        :pnn:    PhEDEx node name.
        :rse:    Rucio RSE. If None (default) inferred by the pnn using DEFAULT_RSE_FMT.
        :scope:  Scope. Default: DEFAULT_SCOPE.
        """

        self.phedex_svc = PhEDEx()
        self.dry_run = dry_run

        self.pnn = pnn
        if rse is None:
            self.rse = list_rses('cms_type=real&pnn=%s' % self.pnn)[0]['rse']
        else:
            self.rse = rse
        rse_details = get_rse(self.rse)
        self.rse_id = rse_details['id']

        self.account = (SYNC_ACCOUNT_FMT % self.rse.lower())[:25]
        self.container = self.phedex_svc.check_data_item(
            pditem=block_name)['pds']
        self.scope = DEFAULT_SCOPE
        self.block_name = block_name
        self.lifetime = lifetime

        self.group, self.custodial, self.is_at_pnn = self.phedex_svc.block_at_pnn_phedex(
            block=self.block_name, pnn=self.pnn)
        self.block_in_phedex = self.phedex_svc.block_exists(
            block=self.block_name)
        self.block_known = self.phedex_svc.block_known(block=self.block_name)

        if self.is_at_pnn:
            self.replicas = self.phedex_svc.fileblock_files_phedex(
                pnn=pnn, pfb=block_name)
        else:
            self.replicas = {}

        self.container_exists = None
        self.block_exists = None
        self.rule_exists = None

        touch(text=self.rse)
예제 #6
0
    def __init__(self,
                 account,
                 auth_type=None,
                 exclude=DEFAULT_EXCLUDE_LINKS,
                 distance=None,
                 phedex_links=False,
                 rselist=None,
                 instance=DEFAULT_PHEDEX_INST,
                 datasvc=DEFAULT_DATASVC_URL):

        if distance is None:
            distance = DEFAULT_DISTANCE_RULES

        self.pcli = PhEDEx(instance=instance, datasvc=datasvc)
        self.rcli = Client(account=account, auth_type=auth_type)

        self._get_rselist(rselist)

        self._get_matrix(distance, phedex_links, exclude)
예제 #7
0
def sync(config, logs):
    """
    Main Sync process
    """

    logging.my_logfile(logs=logs)
    logging.my_fmt(label='main_sync')
    starttime = datetime.now()
    modify = {}
    workers = {}  # this is the array of running pnns
    pnns = None  # this is the array of pnn to be launched
    pool = None

    pcli = PhEDEx()

    install_mp_handler()

    conf = _load_config(config, modify, starttime)

    pnns = []

    size = conf['main']['pool']

    logging.summary('Starting')

    while conf['main']['run']:

        if pool is None:
            logging.notice('Started pool of size %d', size)
            pool = multiprocessing.NDPool(size)

        add = [
            pnn for pnn, sec in conf.items() if pnn != 'main' if sec['run']
            if pnn not in workers if pnn not in pnns
        ]

        pnns += add

        random.shuffle(pnns)

        if not _ping():
            logging.warning('Cannot ping, not launching workers')
        else:
            _launch_workers(pool, workers, pnns, pcli)
            pnns = []

        _poll_workers(workers, pnns)

        conf = _load_config(config, modify, starttime)

        if not conf['main']['run'] or\
            conf['main']['pool'] != size:

            # trigger draining of all workers, close the pool and wait
            # for the task to be over
            conf = _load_config(config, {'default': {'run': False}}, starttime)
            _drain_up(workers, pnns)
            workers = {}
            pool.close()
            pool = None
            size = conf['main']['pool']

        else:
            time.sleep(conf['main']['sleep'])

    logging.summary('Exiting.')

    return config
예제 #8
0
        help='dataset to be updates. Can have wildcard and can be multiple')
    PARSER.add_argument('--pool',
                        dest='pool',
                        default=1,
                        help='number of parallel threads. Default 1.')

    OPTIONS = PARSER.parse_args()

    logging.my_lvl(OPTIONS.verbosity)

    #    logging.summary('DBP1')

    install_mp_handler()
    POOL = multiprocessing.Pool(int(OPTIONS.pool))

    PCLI = PhEDEx()

    PNNS = PCLI.pnns(select=OPTIONS.pnn)

    TIMING = {}

    WILDCARD = re.compile(r'\S*[*]\S*')

    DATASETS = get_timing(_get_dset_list(PCLI, OPTIONS.dataset), TIMING)

    PROCS = get_timing(_launch_workers(PNNS, DATASETS, POOL, OPTIONS, PCLI),
                       TIMING)

    get_timing(_get_workers(POOL, PROCS), TIMING)

    logging.summary(
예제 #9
0
    OPTIONS = PARSER.parse_args()

    if OPTIONS.debug:
        logging.basicConfig(level=logging.DEBUG)
    else:
        logging.basicConfig(level=logging.INFO)

    if OPTIONS.domains:
        OPTIONS.domains = json.loads(OPTIONS.domains.replace("'", '"'))

    if OPTIONS.seinfo is not None:
        OPTIONS.seinfo = json.loads(OPTIONS.seinfo.replace("'", '"'))

    if 'all' in OPTIONS.pnn:
        OPTIONS.pnn = PhEDEx(instance=OPTIONS.instance).pnns(
            select=OPTIONS.select, exclude=OPTIONS.exclude)

    CHANGED = []
    TOT = []

    for node_name in OPTIONS.pnn:
        for rse_type in OPTIONS.type:
            logging.info('Starting pnn %s and type %s', node_name, rse_type)
            RSE = CMSRSE(pnn=node_name,
                         rsetype=rse_type,
                         account=OPTIONS.account,
                         dry=OPTIONS.dry,
                         suffix=OPTIONS.suffix,
                         fts=OPTIONS.fts,
                         tier=OPTIONS.tier,
                         lfn2pfn_algorithm=OPTIONS.lfn2pfn,