示例#1
0
                print '(R)emove incomplete file sets / (s)kip?'
                response = sys.stdin.readline().strip()
                if response == 'R' or response == 's' or response == 'l': break
        else:
            response = 's'

        if response == 'R':
            while True:
                try:
                    suffix = incompletes.pop()
                except IndexError:
                    break

                for file in filter(lambda s: suffix in s, allFiles):
                    print 'dsrm', path + '/' + file
                    rmlink(path + '/' + file)

                resolved = []
                for iD in range(len(duplicates)):
                    if suffix in duplicates[iD]:
                        duplicates[iD].remove(suffix)
                        if len(duplicates[iD]) < 2:
                            resolved.append(duplicates[iD])

                for suffices in resolved:
                    duplicates.remove(suffices)

    if len(duplicates):
        if not options.output:
            while True:
                print '(R)emove duplicates individually / remove (All) duplicates / (s)kip / (l)ist?'
示例#2
0
def check_dataset(path, names, exclude = [], removeEmpty = False):

    patterns = []
    for name in names:
        patterns.append(re.compile(name[0:name.rfind('.')] + '_(([0-9]+)(?:_[0-9]+_[a-zA-Z0-9]{3}|))[.]' + name[name.rfind('.') + 1:]))
    
    lists = {} # {jobNumber: {suffix-1: [pat1exists, pat2exists, ..], suffix-2: [pat1exists, pat2exists, ..]}}
        
    for file in os.listdir(path):
        for iP in range(len(patterns)):
            pattern = patterns[iP]
            matches = pattern.match(file)
            if matches:
                jobNumber = int(matches.group(2))
        
                if jobNumber in exclude:
                    break
        
                if removeEmpty:
                    pfn = os.readlink(path + '/' + file)
                    try:
                        size = os.stat(pfn).st_size
                    except OSError:
                        size = 0
        
                    if size == 0:
                        print 'dsrm', path + '/' + file
                        rmlink(path + '/' + file)
                        break
                
                if jobNumber not in lists:
                    lists[jobNumber] = {}
        
                suffix = matches.group(1)
                if suffix not in lists[jobNumber]:
                    lists[jobNumber][suffix] = [False] * len(patterns)
        
                lists[jobNumber][suffix][iP] = True

                break
        else:
            print 'File', file, 'does not match any given patterns'
            continue


    # check if we had non-matching patterns

    iP = 0
    while iP != len(patterns):
        for combinations in lists.values():
            for existenceList in combinations.values():
                if existenceList[iP]: break
            else:
                continue
            break
        else:
            # no match found for the pattern
            for combinations in lists.values():
                for existenceList in combinations.values():
                    existenceList.pop(iP)

            patterns.pop(iP)
            iP -= 1

        iP += 1

    incompletes = [] # list of suffices
    duplicates = [] # list of list of suffices
    
    for jobNumber, combinations in lists.items():
        for suffix, existenceList in combinations.items():
            if not reduce(lambda x, y: x and y, existenceList):
                incompletes.append(suffix)

        if len(combinations) > 1:
            duplicates.append(combinations.keys())

    absents = sorted(set(range(1, max(lists.keys()) + 1)) - set(lists.keys()))

    return len(lists), incompletes, duplicates, absents
示例#3
0
文件: dscp.py 项目: CMUCMS/dcmutools
def dscp(source, lfn, logfunc = print, force = False):
    logfunc('dscp', source, lfn)
    
    if os.path.islink(lfn):
        if force:
            try:
                dsrm.rmlink(lfn)
            except:
                logfunc('Failed to remove', lfn)
                return False
        else:
            logfunc(lfn, 'already exists as LFN')
            return False
    
    if lfn[0:7] != '/store/':
        logfunc('LFN must start with /store/')
        return False

    lfdir = os.path.dirname(lfn)

    if os.path.exists(lfdir):
        if not os.path.isdir(lfdir):
            logfunc('Cannot make', lfdir, 'into a directory')
            return False
    else:
        os.makedirs(lfdir)

    target = random.choice(disks)

    pfn = lfn.replace('/store', target)
    
    pfdir = os.path.dirname(pfn)
    
    if os.path.exists(pfdir):
        if not os.path.isdir(pfdir):
            logfunc('Cannot make', pfdir, 'into a directory')
            return False
    else:
        os.makedirs(pfdir)

    if ':' in source:
        proc = subprocess.Popen(['scp', '-oStrictHostKeyChecking=no', source, pfn], stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
        while proc.poll() is None: pass
        if proc.returncode != 0:
            response = ''
            while True:
                line = subprocess.stdout.readline()
                if not line: break
                response += line

            logfunc('Copy failed:', response)
            return False
    else:
        try:
            shutil.copyfile(source, pfn)
        except:
            logfunc('Copy failed:\n', excDump())
            return False
    
    os.symlink(pfn, lfn)

    return True