예제 #1
0
def funkRunner3(root_img_dir='', single_style='', update=''):
    import multiprocessing
    #import Queue
    import threading
    import os
    from glob import glob
    from magicColorspaceModAspctLoadFaster2 import rename_retouched_file, sort_files_by_values
    destdir = os.environ.get('DESTDIR', '/mnt/Post_Complete/ImageDrop')
    print 'Starting Funkrunner2 Pools'

    ## Get all modules on poython exec path
    import sys
    sys.path.append('/usr/local/batchRunScripts/python/jbmodules/image_processing/marketplace_dev')

    ########## One ##########
    #
    # 1A
    # List of images to run through processing as glob of the root_img_dir
    #print root_img_dir, ' <-- Rootimgdir FunkR2'

    if update:
        root_img_dir = '/mnt/Post_Complete/Complete_Archive/MARKETPLACE'
        imagesGlob = os.path.join(root_img_dir, '*/999999/*.??[gG]')
        images = [f for f in glob(imagesGlob) if f is not None]
    elif single_style:
        imagesGlob = glob(os.path.join(root_img_dir, '*/*/{0}*.??g'.format(single_style))) + glob(os.path.join(root_img_dir, '*/*/*/{0}*.??g'.format(single_style)))
        images = [f for f in imagesGlob if f is not None]
    elif os.environ.get('ROOT_IMG_DIR') or root_img_dir:
        root_img_dir = os.environ.get('ROOT_IMG_DIR', root_img_dir)
        imagesGlob = os.path.join(root_img_dir, '*/*/*.??[gG]')
        images = [f for f in glob(imagesGlob) if f is not None]
    else:
        root_img_dir = '/mnt/Post_Complete/Complete_Archive/MARKETPLACE'
        imagesGlob = os.path.join(root_img_dir, '*/*/*.??[gG]')
        images = [f for f in glob(imagesGlob) if f is not None]

    print imagesGlob, "GLOBB11"
    # 1B
    # Rename files using Multiproc pool
    poolRename = multiprocessing.Pool(8)

    while len(images) == 0:
        print len(images), '  <-- Length of the Images to Rename,Process etc. Now the Renamer'
        break

    resrename = poolRename.map(rename_retouched_file, images)
    poolRename.close()
    poolRename.join()
    print 'Images Renamed'

    ## replace the glob pattern for single styles after the rename
    #imagesGlob = imagesGlob.replace('_[1-6]', '*')
    print imagesGlob, "GLOBB22"
    ########## Two ##########
    #
    # 2
    # Extract image pixel data for enhancements. As list of tuples, [<url>, {rgbdata} ].. ithink
    if not single_style:
        imagesGlob =  [ f for f in glob(os.path.join(root_img_dir, '*/*/*.??[gG]')) + glob(os.path.join(root_img_dir, '*/*/*/*.??[gG]')) if f is not None ]
        img_list = [f for f in imagesGlob if f is not None]
        print type(img_list), '\tLen ImageList preThreaded', destdir
        img_dict = run_threaded_imgdict(argslist=(img_list,))
        print type(img_dict), '\tLen ImageDict postThreaded', destdir
    else:
        imagesGlob = glob(os.path.join(root_img_dir, '*/*/{0}*.??g'.format(single_style))) + glob(os.path.join(root_img_dir, '*/*/*/{0}*.??g'.format(single_style)))
        img_list = [f for f in imagesGlob if f is not None]
        img_dict = run_threaded_imgdict(argslist=(img_list,))
    ########## Three ##########
    #
    # 3A
    # Init Task and Results Queues
    from ConsumerManager import Starter

    s = Starter()
    s.img_dict = img_dict
    s.destdir  = destdir
    s.start()

    #
    # if single_style and len(imagesGlob) <= 7:
    #     settest = list(set([ f.split('/')[:9] for f in glob(imagesGlob) if f is not None ] ))
    #     print settest, len(settest), ' <=====END FLAG TEST'
    #     for img in imagesGlob:
    #         os.remove(img)
    #         print 'Deleted {0} after uploading'.format(img)
    # else:
    print 'DUNNNN --> multproc2', single_style, len(imagesGlob), '<-- LENGlob'

    if img_dict and type(img_dict) == dict:
        return img_dict
    else:
        return img_list
예제 #2
0
def funkRunner3(root_img_dir=None, single_flag=None):
    import multiprocessing
    #import Queue
    import threading
    import glob, os
    #from os import os.path
    import jbmodules
    from jbmodules import image_processing
    from jbmodules.image_processing.marketplace.magicColorspaceModAspctLoadFaster2 import rename_retouched_file, sort_files_by_values
    destdir = '/mnt/Post_Complete/ImageDrop'
    print 'Starting Funkrunner2 Pools'

    ## Get all modules on poython exec path
    import sys
    sys.path.append('/usr/local/batchRunScripts/python/jbmodules/image_processing/magick_tweaks')
    sys.path.append('/usr/local/batchRunScripts/mozu')
    sys.path.append('/usr/local/batchRunScripts/python')
    sys.path.append('/usr/local/batchRunScripts/python/jbmodules')
    sys.path.append('/usr/local/batchRunScripts/python/jbmodules/mongo_tools')
    sys.path.append('/usr/local/batchRunScripts/python/jbmodules/image_processing')
    sys.path.append('/usr/local/batchRunScripts/python/jbmodules/image_processing/marketplace')

    ########## One ##########
    #
    # 1A
    # List of images to run through processing as glob of the root_img_dir
    #print root_img_dir, ' <-- Rootimgdir FunkR2'
    if root_img_dir == '/mnt/Post_Complete/Complete_Archive/MARKETPLACE' or root_img_dir is None:
        if not single_flag:
            imagesGlob = os.path.join(root_img_dir, '*/*/*.??[gG]')
        else:
            imagesGlob = os.path.join(root_img_dir, '*/*/*/{0}_[1-6].??[gG]'.format(single_flag))

    else:
        imagesGlob = os.path.join(root_img_dir, '*.??[gG]')

    print imagesGlob, "GLOBB"
    # 1B
    # Rename files using Multiproc pool
    poolRename = multiprocessing.Pool(8)
    images = [ f for f in glob.glob(imagesGlob) if f is not None ]
    while len(images) == 0:
        print len(images), '  <-- Length of the Images to Rename,Process etc. Now the Renamer'
        break

    resrename = poolRename.map(rename_retouched_file, images)
    poolRename.close()
    poolRename.join()
    print 'Images Renamed'


    ########## Two ##########
    #
    # 2
    # Extract image pixel data for enhancements. As list of tuples, [<url>, {rgbdata} ].. ithink
    img_list =  [ f for f in glob.glob(imagesGlob) if f is not None ]
    print type(img_list), '\tLen ImageList preThreaded', destdir
    img_dict = run_threaded_imgdict(argslist=(img_list,))
    print type(img_dict), '\tLen ImageDict postThreaded', destdir

    ########## Three ##########
    #
    # 3A
    # Init Task and Results Queues
    from ConsumerManager import Starter

    s = Starter()
    s.img_dict = img_dict
    s.destdir  = destdir
    s.start()



    if single_flag and len(imagesGlob) <= 7:
        settest = list(set([ f.split('/')[:9] for f in glob.glob(imagesGlob) if f is not None ] ))
        print settest, len(settest), ' <=====END FLAG TEST'
        for img in imagesGlob:
            os.remove(img)
            print 'Deleted {0} after uploading'.format(img)
    else:
        print 'multproc2', single_flag, len(imagesGlob), '<-- LENGlob'