Ejemplo n.º 1
0
def walkAndCalcReps(root_dir):

    """Clasify files in a given directory to prortrait or not.
    Traverses all sub directories of the given root
    Saves to db if portrait
    """
    global processedFname
    global repFname, failFname

    processedFname = os.path.join(dataPath,processedFname)
    repFname = os.path.join(dataPath,repFname)
    failFname = os.path.join(dataPath,failFname)

    processedList = []
    if os.path.isfile(processedFname):
        processedList = pickle.load(open(processedFname,'r'))

    repDict = {}
    if os.path.isfile(repFname):
        repDict = pickle.load(open(repFname,'r'))

    failDict = {}
    if os.path.isfile(failFname):
        failDict = pickle.load(open(failFname,'r'))



    # print repFname
    # print

    dirs = [s for s in os.listdir(root_dir) if s.lower().startswith('ie')]

    # print '********'
    # print dirs
    # print '********'

    nDirs = len(dirs)
    iDir = 0

    ts = lt.resetTimer(nDirs,'Going over NLI dirs!')

    for root, dirs, files in os.walk(root_dir, topdown=True):

        # print "***" + root + "***"
        # print files

        ent = root.split('/')[-1]

        if not ent.lower().startswith(r'ie'):

            continue

        print "-----"  + ent + "-----" #+ str(ent.lower().startswith(r'ie'))

        for name in files:


            key = os.path.join(ent, name)
            # print key

            if key in processedList:

                continue

            # print "=====calculating rep====="

            fname = os.path.join(root, name)

            if not fname.endswith(('.jpg', '.png', '.tif')):

                continue

            rep, err = getImgRep(fname)

            if rep is not None:

                repDict[key]=rep

            else:

                failDict[key]=err

            processedList.append(key)

        pickle.dump(repDict,open(repFname,'w'),pickle.HIGHEST_PROTOCOL)

        pickle.dump(failDict,open(failFname,'w'))

        pickle.dump(processedList,open(processedFname,'w'))

        lt.sampleTimer(iDir,ts)

        iDir = iDir + 1
Ejemplo n.º 2
0
def walkAndCalcReps(root_dir):

    """Clasify files in a given directory to prortrait or not.
    Traverses all sub directories of the given root
    Saves to db if portrait
    """
    processedFname = os.path.join(root_dir,'processed.pkl')
    repFname = os.path.join(root_dir,'repsDBase.pkl')
    repFnameOldFmt = os.path.join(root_dir,'repsDBaseOldFmt.pkl')
    failFname = os.path.join(root_dir,'failDBase.pkl')
    repFnameJson = os.path.join(root_dir,'repsDBase.json')

    startTime = None

    # processedFname = os.path.join(root_dir,processedFname)
    # repFname = os.path.join(root_dir,repFname)
    # failFname = os.path.join(root_dir,failFname)

    processedList = []
    if os.path.isfile(processedFname):
        processedList = pickle.load(open(processedFname,'r'))

    repDict = {}
    if os.path.isfile(repFname):
        repDict = pickle.load(open(repFname,'r'))

    failDict = {}
    if os.path.isfile(failFname):
        failDict = pickle.load(open(failFname,'r'))



    # print repFname
    # print

    # dirs = [s for s in os.listdir(root_dir) if s.lower().startswith('ie')]

    # print '********'
    # print dirs
    # print '********'

    # nDirs = len(dirs)
    # iDir = 0

    # first walk - to get the number of files
    nFiles = 0
    for root, dirs, files in os.walk(root_dir, topdown=True):
        nFiles += len([f for f in files if f.lower().endswith(('.jpg', '.png', '.tif'))])


    ts = lt.resetTimer(nFiles,'Analyzing images!', percentile=1.0)

    iFile = 0

    for root, dirs, files in os.walk(root_dir, topdown=True):

        # print "***" + root + "***"
        # print files

        ent = root.split(root_dir)[-1]
        if ent.startswith(r'/'):
            ent = ent[1:]

        print "-----"  + ent + "-----" #+ str(ent.lower().startswith(r'ie'))

        for name in files:


            key = os.path.join(ent, name)
            # print key

            # if key in processedList:
            #
            #     continue

            # print "=====calculating rep====="

            fname = os.path.join(root, name)

            if not fname.endswith(('.jpg', '.png', '.tif')):

                continue

            rep, err = getImgRep(fname)

            iFile += 1

            lt.sampleTimer(iFile ,ts)


            if rep is not None:

                repDict[key]=rep

            else:

                failDict[key]=err

            processedList.append(key)

        pickle.dump(repDict,open(repFname,'w'),pickle.HIGHEST_PROTOCOL)
        pickle.dump(repDict,open(repFnameOldFmt, 'w'))

        pickle.dump(failDict,open(failFname,'w'))

        pickle.dump(processedList,open(processedFname,'w'))

    repDictJson = {k:list(v) for k,v in repDict.iteritems()}

    f = open(repFnameJson,'w')
    f.write(json.dumps(repDictJson))
    f.close()