def funkRunner3(root_img_dir='', single_style='', update=''):
import multiprocessing
#import Queue
import threading
import os
from glob import glob
from magicColorspaceModAspctLoadFaster2 import rename_retouched_file, sort_files_by_values
destdir = os.environ.get('DESTDIR', '/mnt/Post_Complete/ImageDrop')
print 'Starting Funkrunner2 Pools'
## Get all modules on poython exec path
import sys
sys.path.append('/usr/local/batchRunScripts/python/jbmodules/image_processing/marketplace_dev')
########## One ##########
#
# 1A
# List of images to run through processing as glob of the root_img_dir
#print root_img_dir, ' <-- Rootimgdir FunkR2'
if update:
root_img_dir = '/mnt/Post_Complete/Complete_Archive/MARKETPLACE'
imagesGlob = os.path.join(root_img_dir, '*/999999/*.??[gG]')
images = [f for f in glob(imagesGlob) if f is not None]
elif single_style:
imagesGlob = glob(os.path.join(root_img_dir, '*/*/{0}*.??g'.format(single_style))) + glob(os.path.join(root_img_dir, '*/*/*/{0}*.??g'.format(single_style)))
images = [f for f in imagesGlob if f is not None]
elif os.environ.get('ROOT_IMG_DIR') or root_img_dir:
root_img_dir = os.environ.get('ROOT_IMG_DIR', root_img_dir)
imagesGlob = os.path.join(root_img_dir, '*/*/*.??[gG]')
images = [f for f in glob(imagesGlob) if f is not None]
else:
root_img_dir = '/mnt/Post_Complete/Complete_Archive/MARKETPLACE'
imagesGlob = os.path.join(root_img_dir, '*/*/*.??[gG]')
images = [f for f in glob(imagesGlob) if f is not None]
print imagesGlob, "GLOBB11"
# 1B
# Rename files using Multiproc pool
poolRename = multiprocessing.Pool(8)
while len(images) == 0:
print len(images), ' <-- Length of the Images to Rename,Process etc. Now the Renamer'
break
resrename = poolRename.map(rename_retouched_file, images)
poolRename.close()
poolRename.join()
print 'Images Renamed'
## replace the glob pattern for single styles after the rename
#imagesGlob = imagesGlob.replace('_[1-6]', '*')
print imagesGlob, "GLOBB22"
########## Two ##########
#
# 2
# Extract image pixel data for enhancements. As list of tuples, [<url>, {rgbdata} ].. ithink
if not single_style:
imagesGlob = [ f for f in glob(os.path.join(root_img_dir, '*/*/*.??[gG]')) + glob(os.path.join(root_img_dir, '*/*/*/*.??[gG]')) if f is not None ]
img_list = [f for f in imagesGlob if f is not None]
print type(img_list), '\tLen ImageList preThreaded', destdir
img_dict = run_threaded_imgdict(argslist=(img_list,))
print type(img_dict), '\tLen ImageDict postThreaded', destdir
else:
imagesGlob = glob(os.path.join(root_img_dir, '*/*/{0}*.??g'.format(single_style))) + glob(os.path.join(root_img_dir, '*/*/*/{0}*.??g'.format(single_style)))
img_list = [f for f in imagesGlob if f is not None]
img_dict = run_threaded_imgdict(argslist=(img_list,))
########## Three ##########
#
# 3A
# Init Task and Results Queues
from ConsumerManager import Starter
s = Starter()
s.img_dict = img_dict
s.destdir = destdir
s.start()
#
# if single_style and len(imagesGlob) <= 7:
# settest = list(set([ f.split('/')[:9] for f in glob(imagesGlob) if f is not None ] ))
# print settest, len(settest), ' <=====END FLAG TEST'
# for img in imagesGlob:
# os.remove(img)
# print 'Deleted {0} after uploading'.format(img)
# else:
print 'DUNNNN --> multproc2', single_style, len(imagesGlob), '<-- LENGlob'
if img_dict and type(img_dict) == dict:
return img_dict
else:
return img_list
def funkRunner3(root_img_dir=None, single_flag=None):
import multiprocessing
#import Queue
import threading
import glob, os
#from os import os.path
import jbmodules
from jbmodules import image_processing
from jbmodules.image_processing.marketplace.magicColorspaceModAspctLoadFaster2 import rename_retouched_file, sort_files_by_values
destdir = '/mnt/Post_Complete/ImageDrop'
print 'Starting Funkrunner2 Pools'
## Get all modules on poython exec path
import sys
sys.path.append('/usr/local/batchRunScripts/python/jbmodules/image_processing/magick_tweaks')
sys.path.append('/usr/local/batchRunScripts/mozu')
sys.path.append('/usr/local/batchRunScripts/python')
sys.path.append('/usr/local/batchRunScripts/python/jbmodules')
sys.path.append('/usr/local/batchRunScripts/python/jbmodules/mongo_tools')
sys.path.append('/usr/local/batchRunScripts/python/jbmodules/image_processing')
sys.path.append('/usr/local/batchRunScripts/python/jbmodules/image_processing/marketplace')
########## One ##########
#
# 1A
# List of images to run through processing as glob of the root_img_dir
#print root_img_dir, ' <-- Rootimgdir FunkR2'
if root_img_dir == '/mnt/Post_Complete/Complete_Archive/MARKETPLACE' or root_img_dir is None:
if not single_flag:
imagesGlob = os.path.join(root_img_dir, '*/*/*.??[gG]')
else:
imagesGlob = os.path.join(root_img_dir, '*/*/*/{0}_[1-6].??[gG]'.format(single_flag))
else:
imagesGlob = os.path.join(root_img_dir, '*.??[gG]')
print imagesGlob, "GLOBB"
# 1B
# Rename files using Multiproc pool
poolRename = multiprocessing.Pool(8)
images = [ f for f in glob.glob(imagesGlob) if f is not None ]
while len(images) == 0:
print len(images), ' <-- Length of the Images to Rename,Process etc. Now the Renamer'
break
resrename = poolRename.map(rename_retouched_file, images)
poolRename.close()
poolRename.join()
print 'Images Renamed'
########## Two ##########
#
# 2
# Extract image pixel data for enhancements. As list of tuples, [<url>, {rgbdata} ].. ithink
img_list = [ f for f in glob.glob(imagesGlob) if f is not None ]
print type(img_list), '\tLen ImageList preThreaded', destdir
img_dict = run_threaded_imgdict(argslist=(img_list,))
print type(img_dict), '\tLen ImageDict postThreaded', destdir
########## Three ##########
#
# 3A
# Init Task and Results Queues
from ConsumerManager import Starter
s = Starter()
s.img_dict = img_dict
s.destdir = destdir
s.start()
if single_flag and len(imagesGlob) <= 7:
settest = list(set([ f.split('/')[:9] for f in glob.glob(imagesGlob) if f is not None ] ))
print settest, len(settest), ' <=====END FLAG TEST'
for img in imagesGlob:
os.remove(img)
print 'Deleted {0} after uploading'.format(img)
else:
print 'multproc2', single_flag, len(imagesGlob), '<-- LENGlob'