def worker(): count = 0 while True: item = q.get() #print item[0] imgdata = sort_files_by_values(item) #print imgdata img_dict_list.append(imgdata) # Can add functions to adjust based on imgdict params or store image data or delete etc. # insertres = insert_gridfs_extract_metadata(item[0]) count += 1 print count, '\n\t ImageDict Threade'#, imgdata q.task_done()
def funkRunner(root_img_dir=None): import multiprocessing import Queue import threading import glob, os from magicColorspaceModAspctLoadFaster2 import rename_retouched_file, subproc_magick_png, subproc_magick_large_jpg, subproc_magick_medium_jpg, sort_files_by_values import convert_img_srgb import mongo_gridfs_insert_file from jbmodules.legacy_mongo_img_prep import insert_gridfs_extract_metadata imgs_renamed = [rename_retouched_file(f) for f in (glob.glob(os.path.join(root_img_dir,'*.??[gG]')))] img_dict = sort_files_by_values(glob.glob(os.path.join(root_img_dir,'*.??[gG]'))) q = Queue.Queue() for k,v in img_dict.items(): try: img_rgbmean = k, v.items() q.put(img_rgbmean) except AttributeError: print 'SOMETHING IS WRONG WITH THE IMAGE Error {}'.format(img) pass def worker(): count = 0 destdir = '/mnt/Post_Complete/ImageDrop/' while True: img, rgbmean = q.get() convert_img_srgb.main(image_file=img) ## Add to Mongo DB try: jbmodules.mongo_image_prep.update_gridfs_extract_metadata(image_file, db_name='gridfs_mrktplce') except: jbmodules.mongo_image_prep.insert_gridfs_extract_metadata(image_file, db_name='gridfs_mrktplce') ## Generate png from source then jpgs from png pngout = magickProc2.subproc_magick_png(img, rgbmean=dict(rgbmean), destdir=destdir) magickProc2.subproc_magick_large_jpg(pngout, destdir=destdir) magickProc2.subproc_magick_medium_jpg(pngout, destdir=destdir) count += 1 print count q.task_done() cpus=multiprocessing.cpu_count() #detect number of cores print("Creating %d threads" % cpus) for i in xrange(cpus*2): t = threading.Thread(target=worker) t.daemon = True t.start() q.join() #block until all tasks are done