Esempio n. 1
0
def lfn2pfn(lfn, sename, mgr=None):
    "Find PFN for given LFN and SE"
    pfnlist = []
    if not mgr:
        mgr = SiteDBManager()
    cmsname = mgr.get_name(sename)
    if cmsname:
        params = {'protocol': 'srmv2', 'lfn': lfn, 'node': cmsname}
        result = get_data(phedex_url('lfn2pfn'), params)
        try:
            for item in result['phedex']['mapping']:
                pfn = item['pfn']
                if pfn not in pfnlist:
                    pfnlist.append(pfn)
        except:
            msg = "Fail to look-up PFNs in Phedex\n" + str(result)
            print msg
    return pfnlist
Esempio n. 2
0
def lfn2pfn(lfn, sename, mgr=None):
    "Find PFN for given LFN and SE"
    pfnlist = []
    if  not mgr:
        mgr = SiteDBManager()
    cmsname = mgr.get_name(sename)
    if  cmsname:
        params = {'protocol':'srmv2', 'lfn':lfn, 'node':cmsname}
        result = get_data(phedex_url('lfn2pfn'), params)
        try:
            for item in result['phedex']['mapping']:
                pfn = item['pfn']
                if  pfn not in pfnlist:
                    pfnlist.append(pfn)
        except:
            msg = "Fail to look-up PFNs in Phedex\n" + str(result)
            print msg
    return pfnlist
Esempio n. 3
0
def get_username(verbose=None):
    """
    Get user name from provided DN
    """
    # get DN from grid-proxy-info
    cmd = 'grid-proxy-info'
    stdout, stderr = execmd(cmd)
    if stderr.find('command not found') != -1:
        raise Exception(stderr)
    userdn = None
    try:
        for line in stdout.split('\n'):
            if line.find('issuer') != -1:
                issuer, userdn = line.split(' : ')
    except:
        raise Exception('Unable to parse grid-proxy-info:\n%s' % stdout)
    if verbose:
        print "userdn :", userdn
    if not userdn:
        msg = 'Unable to determine your DN, please run grid-proxy-init'
        raise Exception(msg)
    mgr = SiteDBManager()
    user = mgr.get_user(userdn)
    return user
Esempio n. 4
0
def get_username(verbose=None):
    """
    Get user name from provided DN
    """
    # get DN from grid-proxy-info
    cmd    = 'grid-proxy-info'
    stdout, stderr = execmd(cmd)
    if  stderr.find('command not found') != -1:
        raise Exception(stderr)
    userdn = None
    try:
        for line in stdout.split('\n'):
            if  line.find('issuer') != -1:
                issuer, userdn = line.split(' : ')
    except:
        raise Exception('Unable to parse grid-proxy-info:\n%s' % stdout)
    if  verbose:
        print "userdn :", userdn
    if  not userdn:
        msg = 'Unable to determine your DN, please run grid-proxy-init'
        raise Exception(msg)
    mgr  = SiteDBManager()
    user = mgr.get_user(userdn)
    return user
Esempio n. 5
0
def pfn_dst(lfn, dst, verbose=None):
    """
    Look-up LFN in Phedex and return pfn dst for further processing
    """
    dstfname = None
    pat = re.compile('^T[0-9]_[A-Z]+(_)[A-Z]+')
    if pat.match(dst):
        dst_split = dst.split(':')
        dst = dst_split[0]
        if len(dst_split) == 1:  # copy to the node
            local_path = dst_split[1]
            for srm_path, lfn_match in resolve_srm_path(dst, verbose):
                lfn_pat = re.compile(lfn_match)
                if lfn_pat.match(lfn):
                    srm_path = srm_path.replace('\?',
                                                '?').replace('$1', local_path)
                    if verbose:
                        print "Resolve %s into %s" % (dst, srm_path)
                    dst = srm_path
        else:
            paths = [p for p in resolve_user_srm_path(dst, verbose=verbose)]
            dst = '%s/%s' % (paths[0], get_username())
        check_permission(dst, verbose)
    else:
        if dst.find('file:///') == -1:
            dstfname = dst.split('/')[-1]
            if dstfname == '.':
                dstfname = None
            if dst[0] == '/':  # absolute path
                if os.path.isdir(dst):
                    ddir = dst
                    dstfname = None
                else:
                    ddir = '/'.join(dst.split('/')[:-1])
                if not os.path.isdir(ddir):
                    msg = 'Provided destination directory %s does not exists' % ddir
                    raise Exception(msg)
                dst = 'file:///%s' % ddir
            else:
                ddir = '/'.join(dst.split('/')[:-1]).replace(
                    '$PWD', os.getcwd())
                if os.path.isdir(ddir):
                    dst = 'file:///%s' % os.path.join(os.getcwd(), ddir)
                else:
                    dst = 'file:///%s' % os.getcwd()
    pfnlist = []
    if os.path.isfile(lfn) or lfn.find('file:///') != -1:  # local file
        pfn = lfn.replace('file:///', '')
        if pfn[0] != '/':
            pfn = 'file:///%s' % os.path.join(os.getcwd(), pfn)
        else:
            pfn = 'file:///%s' % pfn
        pfnlist = [pfn]
    else:
        if lfn.find(':') != -1:
            node, lfn = lfn.split(':')
            params = {'node': node, 'lfn': lfn, 'protocol': 'srmv2'}
            method = 'lfn2pfn'
        else:
            params = {'se': '*', 'lfn': lfn}
            method = 'fileReplicas'
        json_dict = get_data(phedex_url(method), params)
        ddict = DotDict(json_dict)
        if verbose:
            print "Look-up LFN:"
            print lfn
        phedex = json_dict['phedex']
        if phedex.has_key('mapping'):
            if not phedex['mapping']:
                msg = "LFN: %s\n" % lfn
                msg += 'No replicas found\n'
                msg += str(json_dict)
                raise Exception(msg)
            filelist = ddict.get('phedex.mapping.pfn')
            if not filelist:
                filelist = []
            if isinstance(filelist, basestring):
                filelist = [filelist]
            for fname in filelist:
                pfnlist.append(fname)
        elif phedex.has_key('block') and not phedex['block']:
            msg = 'No replicas found in PhEDEx, will try to get original SE from DBS'
            print_warning(msg)
            sename = get_dbs_se(lfn)
            msg = 'Orignal LFN site %s' % sename
            print_info(msg)
            mgr = SiteDBManager()
            pfnlist = lfn2pfn(lfn, sename, mgr)
        filelist = ddict.get('phedex.block.file')
        if not filelist:
            filelist = []
        for fname in filelist:
            for replica in fname['replica']:
                cmsname = replica['node']
                se = replica['se']
                if verbose:
                    print "found LFN on node=%s, se=%s" % (cmsname, se)
                if cmsname.count('T0', 0, 2) == 1:
                    continue  # skip T0's
                # query Phedex for PFN
                params = {'protocol': 'srmv2', 'lfn': lfn, 'node': cmsname}
                result = get_data(phedex_url('lfn2pfn'), params)
                try:
                    for item in result['phedex']['mapping']:
                        pfn = item['pfn']
                        if pfn not in pfnlist:
                            pfnlist.append(pfn)
                except:
                    msg = "Fail to look-up PFNs in Phedex\n" + str(result)
                    print msg
                    continue
    if verbose > 1:
        print "PFN list:"
        for pfn in pfnlist:
            print pfn

    # finally return pfn and dst paths w/ file for further processing
    for item in pfnlist:
        ifile = item.split("/")[-1] if not dstfname else dstfname
        yield item, '%s/%s' % (dst, ifile)