Beispiel #1
0
def get_pfns(lfn, verbose=None):
    """
    Look-up LFN in Phedex and get corresponding list of PFNs
    """
    pfnlist = []
    selist = []
    params = {'se': '*', 'lfn': lfn}
    json_dict = get_data(phedex_url('fileReplicas'), params)
    ddict = DotDict(json_dict)
    if not json_dict['phedex']['block']:
        return pfnlist, selist
    for fname in ddict.get('phedex.block.file'):
        for replica in fname['replica']:
            cmsname = replica['node']
            se = replica['se']
            if se not in selist:
                selist.append(se)
            # query Phedex for PFN
            params = {'protocol': 'srmv2', 'lfn': lfn, 'node': cmsname}
            result = get_data(phedex_url('lfn2pfn'), params)
            try:
                for item in result['phedex']['mapping']:
                    pfn = item['pfn']
                    if pfn not in pfnlist:
                        pfnlist.append(pfn)
            except:
                msg = "Fail to look-up PFNs in Phedex\n" + str(result)
                print msg
                continue
    return pfnlist, selist
Beispiel #2
0
def get_pfns(lfn, verbose=None):
    """
    Look-up LFN in Phedex and get corresponding list of PFNs
    """
    pfnlist   = []
    selist    = []
    params    = {'se':'*', 'lfn':lfn}
    json_dict = get_data(phedex_url('fileReplicas'), params)
    ddict     = DotDict(json_dict)
    if  not json_dict['phedex']['block']:
        return pfnlist, selist
    for fname in ddict.get('phedex.block.file'):
        for replica in fname['replica']:
            cmsname = replica['node']
            se      = replica['se']
            if  se not in selist:
                selist.append(se)
            # query Phedex for PFN
            params = {'protocol':'srmv2', 'lfn':lfn, 'node':cmsname}
            result = get_data(phedex_url('lfn2pfn'), params)
            try:
                for item in result['phedex']['mapping']:
                    pfn = item['pfn']
                    if  pfn not in pfnlist:
                        pfnlist.append(pfn)
            except:
                msg = "Fail to look-up PFNs in Phedex\n" + str(result)
                print msg
                continue
    return pfnlist, selist
Beispiel #3
0
def pfn_dst(lfn, dst, verbose=None):
    """
    Look-up LFN in Phedex and return pfn dst for further processing
    """
    dstfname = None
    pat = re.compile('^T[0-9]_[A-Z]+(_)[A-Z]+')
    if pat.match(dst):
        dst_split = dst.split(':')
        dst = dst_split[0]
        if len(dst_split) == 1:  # copy to the node
            local_path = dst_split[1]
            for srm_path, lfn_match in resolve_srm_path(dst, verbose):
                lfn_pat = re.compile(lfn_match)
                if lfn_pat.match(lfn):
                    srm_path = srm_path.replace('\?',
                                                '?').replace('$1', local_path)
                    if verbose:
                        print "Resolve %s into %s" % (dst, srm_path)
                    dst = srm_path
        else:
            paths = [p for p in resolve_user_srm_path(dst, verbose=verbose)]
            dst = '%s/%s' % (paths[0], get_username())
        check_permission(dst, verbose)
    else:
        if dst.find('file:///') == -1:
            dstfname = dst.split('/')[-1]
            if dstfname == '.':
                dstfname = None
            if dst[0] == '/':  # absolute path
                if os.path.isdir(dst):
                    ddir = dst
                    dstfname = None
                else:
                    ddir = '/'.join(dst.split('/')[:-1])
                if not os.path.isdir(ddir):
                    msg = 'Provided destination directory %s does not exists' % ddir
                    raise Exception(msg)
                dst = 'file:///%s' % ddir
            else:
                ddir = '/'.join(dst.split('/')[:-1]).replace(
                    '$PWD', os.getcwd())
                if os.path.isdir(ddir):
                    dst = 'file:///%s' % os.path.join(os.getcwd(), ddir)
                else:
                    dst = 'file:///%s' % os.getcwd()
    pfnlist = []
    if os.path.isfile(lfn) or lfn.find('file:///') != -1:  # local file
        pfn = lfn.replace('file:///', '')
        if pfn[0] != '/':
            pfn = 'file:///%s' % os.path.join(os.getcwd(), pfn)
        else:
            pfn = 'file:///%s' % pfn
        pfnlist = [pfn]
    else:
        if lfn.find(':') != -1:
            node, lfn = lfn.split(':')
            params = {'node': node, 'lfn': lfn, 'protocol': 'srmv2'}
            method = 'lfn2pfn'
        else:
            params = {'se': '*', 'lfn': lfn}
            method = 'fileReplicas'
        json_dict = get_data(phedex_url(method), params)
        ddict = DotDict(json_dict)
        if verbose:
            print "Look-up LFN:"
            print lfn
        phedex = json_dict['phedex']
        if phedex.has_key('mapping'):
            if not phedex['mapping']:
                msg = "LFN: %s\n" % lfn
                msg += 'No replicas found\n'
                msg += str(json_dict)
                raise Exception(msg)
            filelist = ddict.get('phedex.mapping.pfn')
            if not filelist:
                filelist = []
            if isinstance(filelist, basestring):
                filelist = [filelist]
            for fname in filelist:
                pfnlist.append(fname)
        elif phedex.has_key('block') and not phedex['block']:
            msg = 'No replicas found in PhEDEx, will try to get original SE from DBS'
            print_warning(msg)
            sename = get_dbs_se(lfn)
            msg = 'Orignal LFN site %s' % sename
            print_info(msg)
            mgr = SiteDBManager()
            pfnlist = lfn2pfn(lfn, sename, mgr)
        filelist = ddict.get('phedex.block.file')
        if not filelist:
            filelist = []
        for fname in filelist:
            for replica in fname['replica']:
                cmsname = replica['node']
                se = replica['se']
                if verbose:
                    print "found LFN on node=%s, se=%s" % (cmsname, se)
                if cmsname.count('T0', 0, 2) == 1:
                    continue  # skip T0's
                # query Phedex for PFN
                params = {'protocol': 'srmv2', 'lfn': lfn, 'node': cmsname}
                result = get_data(phedex_url('lfn2pfn'), params)
                try:
                    for item in result['phedex']['mapping']:
                        pfn = item['pfn']
                        if pfn not in pfnlist:
                            pfnlist.append(pfn)
                except:
                    msg = "Fail to look-up PFNs in Phedex\n" + str(result)
                    print msg
                    continue
    if verbose > 1:
        print "PFN list:"
        for pfn in pfnlist:
            print pfn

    # finally return pfn and dst paths w/ file for further processing
    for item in pfnlist:
        ifile = item.split("/")[-1] if not dstfname else dstfname
        yield item, '%s/%s' % (dst, ifile)
Beispiel #4
0
def pfn_dst(lfn, dst, verbose=None):
    """
    Look-up LFN in Phedex and return pfn dst for further processing
    """
    dstfname = None
    pat = re.compile('^T[0-9]_[A-Z]+(_)[A-Z]+')
    if  pat.match(dst):
        dst_split = dst.split(':')
        dst = dst_split[0]
        if  len(dst_split) == 1: # copy to the node
            local_path = dst_split[1]
            for srm_path, lfn_match in resolve_srm_path(dst, verbose):
                lfn_pat = re.compile(lfn_match)
                if  lfn_pat.match(lfn):
                    srm_path = srm_path.replace('\?', '?').replace('$1', local_path)
                    if  verbose:
                        print "Resolve %s into %s" % (dst, srm_path)
                    dst = srm_path
        else:
            paths = [p for p in resolve_user_srm_path(dst, verbose=verbose)]
            dst = '%s/%s' % (paths[0], get_username())
        check_permission(dst, verbose)
    else:
        if  dst.find('file:///') == -1:
            dstfname = dst.split('/')[-1]
            if  dstfname == '.':
                dstfname = None
            if  dst[0] == '/': # absolute path
                if  os.path.isdir(dst):
                    ddir = dst
                    dstfname = None
                else:
                    ddir =  '/'.join(dst.split('/')[:-1])
                if  not os.path.isdir(ddir):
                    msg = 'Provided destination directory %s does not exists' % ddir
                    raise Exception(msg)
                dst = 'file:///%s' % ddir
            else:
                ddir = '/'.join(dst.split('/')[:-1]).replace('$PWD', os.getcwd())
                if  os.path.isdir(ddir):
                    dst = 'file:///%s' % os.path.join(os.getcwd(), ddir)
                else:
                    dst = 'file:///%s' % os.getcwd()
    pfnlist   = []
    if  os.path.isfile(lfn) or lfn.find('file:///') != -1: # local file
        pfn = lfn.replace('file:///', '')
        if  pfn[0] != '/':
            pfn = 'file:///%s' % os.path.join(os.getcwd(), pfn)
        else:
            pfn = 'file:///%s' % pfn
        pfnlist   = [pfn]
    else:
        if  lfn.find(':') != -1:
            node, lfn = lfn.split(':')
            params    = {'node':node, 'lfn':lfn, 'protocol':'srmv2'}
            method    = 'lfn2pfn'
        else:
            params    = {'se':'*', 'lfn':lfn}
            method    = 'fileReplicas'
        json_dict = get_data(phedex_url(method), params)
        ddict     = DotDict(json_dict)
        if  verbose:
            print "Look-up LFN:"
            print lfn
        phedex = json_dict['phedex']
        if  phedex.has_key('mapping'):
            if  not phedex['mapping']:
                msg  = "LFN: %s\n" % lfn
                msg += 'No replicas found\n'
                msg += str(json_dict)
                raise Exception(msg)
            filelist = ddict.get('phedex.mapping.pfn')
            if  not filelist:
                filelist = []
            if  isinstance(filelist, basestring):
                filelist = [filelist]
            for fname in filelist:
                pfnlist.append(fname)
        elif  phedex.has_key('block') and not phedex['block']:
            msg = 'No replicas found in PhEDEx, will try to get original SE from DBS'
            print_warning(msg)
            sename = get_dbs_se(lfn)
            msg = 'Orignal LFN site %s' % sename
            print_info(msg)
            mgr = SiteDBManager()
            pfnlist = lfn2pfn(lfn, sename, mgr)
        filelist = ddict.get('phedex.block.file')
        if  not filelist:
            filelist = []
        for fname in filelist:
            for replica in fname['replica']:
                cmsname = replica['node']
                se      = replica['se']
                if  verbose:
                    print "found LFN on node=%s, se=%s" % (cmsname, se)
                if  cmsname.count('T0', 0, 2) == 1:
                    continue # skip T0's
                # query Phedex for PFN
                params = {'protocol':'srmv2', 'lfn':lfn, 'node':cmsname}
                result = get_data(phedex_url('lfn2pfn'), params)
                try:
                    for item in result['phedex']['mapping']:
                        pfn = item['pfn']
                        if  pfn not in pfnlist:
                            pfnlist.append(pfn)
                except:
                    msg = "Fail to look-up PFNs in Phedex\n" + str(result)
                    print msg
                    continue
    if  verbose > 1:
        print "PFN list:"
        for pfn in pfnlist:
            print pfn

    # finally return pfn and dst paths w/ file for further processing
    for item in pfnlist:
        ifile = item.split("/")[-1] if not dstfname else dstfname
        yield item, '%s/%s' % (dst, ifile)