def main(): import sys,glob,shutil,os,re,datetime sys.path.append('/usr/local/batchRunScripts/python/jbmodules/image_processing/magick_tweaks') sys.path.append('/usr/local/batchRunScripts/mozu') sys.path.append('/usr/local/batchRunScripts/python') sys.path.append('/usr/local/batchRunScripts/python/jbmodules') sys.path.append('/usr/local/batchRunScripts/python/jbmodules/mongo_tools') sys.path.append('/usr/local/batchRunScripts/python/jbmodules/image_processing') sys.path.append('/usr/local/batchRunScripts/python/jbmodules/image_processing/marketplace') regex_coded = re.compile(r'^.+?/[1-9][0-9]{8}_[1-6]\.jpg$') regex_alt = re.compile(r'^.+?/[1-9][0-9]{8}_\w+?0[1-6]\.[JjPpNnGg]{3}$') regex_valid_style = re.compile(r'^.+?/[1-9][0-9]{8}_?.*?\.[JjPpNnGg]{3}$') ### Date Defs todaysdatefullsecs = '{:%Y%m%d%H%M%S}'.format(datetime.datetime.now()) todaysdatefull = todaysdatefullsecs[:12] todaysdate = todaysdatefull[:8] # '{:%Y,%m,%d}'.format(datetime.datetime.now()) todaysdatearch = todaysdatefull # '{:%Y,%m,%d,%H,%M}'.format(datetime.datetime.now()) ### Define tmp and archive paths prior to Creating tmp_processing = os.path.join("/mnt/Post_Complete/Complete_to_Load/.tmp_processing" , "tmp_" + str(todaysdatefullsecs).replace(",", "")) tmp_processing_l = os.path.join(tmp_processing, "largejpg") tmp_processing_m = os.path.join(tmp_processing, "mediumjpg") tmp_loading = os.path.join("/mnt/Post_Complete/Complete_Archive/.tmp_loading" , "tmp_" + str(todaysdatefullsecs).replace(",", "")) tmp_mozu_loading = os.path.join("/mnt/Post_Complete/Complete_Archive/.tmp_mozu_loading" , "tmp_" + str(todaysdatefullsecs).replace(",", "")) ## Define for Creating Archive dirs archive = '/mnt/Post_Complete/Complete_Archive/Uploaded' archive_uploaded = os.path.join(archive, "dateloaded_" + str(todaysdate).replace(",", ""), "uploaded_" + str(todaysdatefullsecs).replace(",", "")) imgdest_jpg_mozu = os.path.join(archive, "dateloaded_" + str(todaysdate), 'JPG_MOZU_LOAD') #imgdest_jpg_final = os.path.join(archive_uploaded, 'JPG_RETOUCHED_ORIG') imgdest_png_final = os.path.join(archive_uploaded, 'PNG') ################### ## Create Lock File ################### #locker = os.path.join(rootdir, 'LOCKED.lock') #if os.path.isfile(locker): # break #else: # with open(locker, 'wb') as f: # f.write(todaysdatefull) # f.close() try: root_img_dir = os.path.abspath(sys.argv[1]) destdir = os.path.abspath(sys.argv[2]) #'/Users/johnb/Pictures' tmp_processing = root_img_dir tmp_loading = destdir except IndexError: root_img_dir = '/mnt/Post_Complete/Complete_to_Load/Drop_FinalFilesOnly' destdir = tmp_loading if os.path.isdir(root_img_dir): pass else: root_img_dir = '/mnt/Post_Complete/Complete_to_Load/Drop_FinalFilesOnly' destdir = tmp_loading ########### ## Test for ex #root_img_dir = tmp_processing walkedout_tmp = glob.glob(os.path.join(root_img_dir, '*/*.*g')) image_ct = len(walkedout_tmp) starttime = todaysdatefull if image_ct == 0: print "Nothing to Process" else: ### Make Tmp Folders for Processing And Uploading -- tmp_dirs are dated with time(hr:min)to prevent collisions print 'Processing -- %s Files \nStarted --> %s \n' % (image_ct, starttime) try: os.makedirs(archive_uploaded, 16877) except: pass try: os.makedirs(tmp_processing, 16877) except: pass try: os.makedirs(tmp_processing_l, 16877) except: pass try: os.makedirs(tmp_processing_m, 16877) except: pass try: os.makedirs(tmp_loading, 16877) except: pass try: os.makedirs(imgdest_jpg_mozu, 16877) except: pass try: os.makedirs(imgdest_png_final, 16877) except: pass # try: # os.makedirs(imgdest_jpg_final, 16877) # except: # pass #################################################### ## Begin Processing and compiling images for Loading #################################################### ## Move All DropFinal Files from Retouchers dirs to tmp_processing from drop folders Then Mogrify to create pngs copy to load and arch dirs walkedout_tmp = glob.glob(os.path.join(root_img_dir, '*/*.*g')) [ shutil.move(file, os.path.join(tmp_processing, os.path.basename(file))) for file in walkedout_tmp ] ### Rename Files moved into Temp Processing Floder walkedout_tmp = glob.glob(os.path.join(tmp_processing, '*.jpg')) [ rename_retouched_file(file) for file in walkedout_tmp ] mz_converted_jpgs = [] if os.path.isdir(tmp_processing): for img in glob.glob(os.path.join(tmp_processing,'*.??[gG]')): if regex_coded.findall(img): img = rename_retouched_file(img) try: pngout = subproc_magick_png(img, destdir=tmp_processing) subproc_magick_large_jpg(pngout, destdir=tmp_loading) subproc_magick_medium_jpg(pngout, destdir=tmp_loading) ############################################## ##### BEGIN MOZU pt 1 of 2 ################## ############################################## ## Make Jpegs and collect Finals in mozu loading dir ####### import sys from os import chdir, path #chdir('/usr/local/batchRunScripts/mozu') import mozu_exec, mozu_image_util_functions ## Compress and convert to jpg and store in separate dir for concurrent xfers #if path.isfile(pngout): try: print ' Is file PNGOUT', pngout, img jpgout = mozu_image_util_functions.magick_convert_to_jpeg(pngout,destdir=imgdest_jpg_mozu) except IOError: jpgout = mozu_image_util_functions.magick_convert_to_jpeg(img,destdir=imgdest_jpg_mozu) pass # Try and make a list without needing globbing below mz_converted_jpgs.append(jpgout) except AttributeError: # from shutil import move print "AttributeError at 780 in magickLoad_PrdCmp.py --> ", img #move(img, "/mnt/Post_Complete/Complete_to_Load/") ########################################### ###### END mozu ########################### ########################################### #metadict = metadata_info_dict(img) #dimens = get_imagesize_variables(img) #test_img = get_image_color_minmax(img) ########################################################## ### Glob created PNGs and copy to Load Dir then Store in Arch dir tmp_png = glob.glob(os.path.join(tmp_processing, '*.png')) [ shutil.copy2(file, os.path.join(tmp_loading, os.path.basename(file))) for file in tmp_png ] [ shutil.move(file, os.path.join(imgdest_png_final, os.path.basename(file))) for file in tmp_png ] ## ARCHIVED Backup ## All JPGs in Root dir Only of tmp_processing will be now Archived as all Conversions are completed ###### All PNGs Created and moved to Archive plus Copy sent to Load Directory #### All Files Converted for Upload, Now glob search and move large and medium named jpgs to tmp loading import time upload_tmp_loading = glob.glob(os.path.join(tmp_loading, '*.*g')) # upload_tmp_loading_mozuall = glob.glob(os.path.join(imgdest_jpg_mozu, '*.jpg')) for upload_file in upload_tmp_loading: #### UPLOAD upload_file via NFS to imagedrop ## Then rm loading tmp dir try: result = copy_to_imagedrop_upload(upload_file, destdir='/mnt/Post_Complete/ImageDrop') if result: print "Uploaded {}".format(upload_file) time.sleep(float(.1)) shutil.move(upload_file, archive_uploaded) else: print result, upload_file pass ##time.sleep(float(3)) # try: # ftpload_to_imagedrop(upload_file) # print "Uploaded {}".format(upload_file) # time.sleep(float(.3)) # shutil.move(upload_file, archive_uploaded) # except: # pass except: print "Error moving Finals to Arch {}".format(file) ###################################################### ### ##### BEGIN MOZU pt 2 of 2 ###################### ###################################################### ## TODO: Fix so all go through funcs. Glob Mozu Jpegs to tmp Originals dir for now by passing productions sharpening funcs ###################################################### ## Mozu Xfer #### import mozu_exec, mozu_image_util_functions, glob # upload_imgdest_jpg_mozu_glob = glob.glob(os.path.join(imgdest_jpg_mozu, '*.*g')) # # print 'MozuExec', '\n\n\n\n\v\v\t\t\t\tMozuExec\t', ' <-- Now MozuExec' # mozu_exec.main(upload_imgdest_jpg_mozu_glob) ####################################################### ############### END MOZU ############################## ####################################################### ### Check for okb files and send to Uploader via email zerobytefiles = glob.glob(os.path.join('/mnt/Post_Complete/Complete_to_Load/Drop_FinalFilesOnly/zero_byte_errors', '*/*.*g')) ## After completed Process and Load to imagedrop ### Finally Remove the 2 tmp folder trees for process and load if Empty try: upload_tmp_loading_remainder = glob.glob(os.path.join(tmp_loading, '*.*g')) if len(upload_tmp_loading_remainder) == 0 and os.path.isdir(tmp_loading): shutil.rmtree(tmp_loading) except: pass ## remove the processing tmp folder if os.path.isdir(tmp_processing): shutil.rmtree(tmp_processing) else: print tmp_processing, ' tmp_processing Does not Exist, tmp_loading-->', tmp_loading
def main(fileslist): import sqlalchemy from db import mozu_image_table_instance from mozu_image_util_functions import compile_todict_for_class_instance_variables, magick_convert_to_jpeg, netsrv101_path_maker # Compiles Data Payload and other Vars per Doc -- Including src_filepath -- **values keys set per instance # print type(fileslist), '<--Type\tLenLoFilepaths', len(fileslist), '\t', fileslist ### Date Defs from os import path # chdir , curdir import datetime #, glob, shutil todaysdatefullsecs = '{:%Y%m%d%H%M%S}'.format(datetime.datetime.now()) todaysdatefull = todaysdatefullsecs[:12] todaysdate = todaysdatefull[:8] # '{:%Y,%m,%d}'.format(datetime.datetime.now()) # Define for Creating Archive dirs archive = '/mnt/Post_Complete/Complete_Archive/Uploaded' # archive_uploaded = path.join(archive, "dateloaded_" + str(todaysdate).replace(",", ""), "uploaded_" + str(todaysdatefullsecs).replace(",", "")) archive_uploaded_day = path.join(archive, "dateloaded_" + str(todaysdate).replace(",", "")) imgdest_jpg_mozu = path.join(archive_uploaded_day, 'JPG_MOZU_LOAD') # imgdest_jpg_mozu_loaded = path.join(imgdest_jpg_mozu, 'LOADED') if path.dirname(fileslist[0]).split('/')[-1] == 'JPG_MOZU_LOAD': # fileslistX= [magick_convert_to_jpeg(f) for f in fileslist if f.split('.')[-1] == 'png'] fileslist = [magick_convert_to_jpeg(f, destdir=imgdest_jpg_mozu) for f in fileslist if f] else: fileslist = fileslist # [ path.abspath(f) for f in fileslist if f ] if not path.isfile(fileslist[0]): fileslist = netsrv101_path_maker(fileslist) compiled_instance_vars = compile_todict_for_class_instance_variables(fileslist=fileslist) # print type(compiled_instance_vars), '<--Type\tLenCompiledInsVars', len(compiled_instance_vars), '\tKeys: ', compiled_instance_vars.keys() # print compiled_instance_vars, "186-MZEXECY" styles_incr_media_version = [] for key,values in compiled_instance_vars.iteritems(): # v = include_keys(values, __mozu_image_table_valid_keys__) # print "IncludedKeys: {}\n\tkey:\t{}\n\tvalues:\t{}".format(v.items(), key , values.popitem()) if not values.get('mz_imageid'): # ### --> src_filepath = k # will need src_filepath in order to perfom any image manipulation # ## ---> before loading(would actually need to redo the md5checksum from compiler) # Insert -- Then try Update if Insert to DB fails or Create NewDoc Fails to Mozu try: values['mz_imageid'], response = upload_new(**values) create_resource_resp = upload_new(**values) mozu_image_table = mozu_image_table_instance() print 'Initial Post to Mozu {0}'.format(create_resource_resp.keys()[0]) if int(create_resource_resp.keys()[0]) == int(201): table_args = include_keys(values, __mozu_image_table_valid_keys__) insert_db = mozu_image_table.insert(values=dict(**table_args)) res_insrt = insert_db.execute() print 'Inserted --> ', values.items(), ' <-- ', dir(res_insrt) styles_incr_media_version.append(table_args.get('bf_imageid', locals().get('bf_imageid'))) elif int(create_resource_resp.keys()[0]) <= int(409): table_args = include_keys(values, __mozu_image_table_valid_keys__) mz_imageid = mozu_image_table.select( whereclause=( (mozu_image_table.c.bf_imageid == table_args['bf_imageid']) ) ).execute().fetchone()['mz_imageid'] bf_imageid = mozu_image_table.select( whereclause=( (mozu_image_table.c.mz_imageid == table_args['mz_imageid']) ) ).execute().fetchone()['bf_imageid'] table_args['bf_imageid'] = values['bf_imageid'] = bf_imageid table_args['mz_imageid'] = values['mz_imageid'] = mz_imageid resp = update_content_mz_image(**values) print('RESP 207 mzexec: {}'.format(resp)) #upsert_content_resp = upsert_data_mz_image(**values) # ,dict(**values)) if resp.http_status_code < 400: update_db = mozu_image_table.update(values=dict(**table_args),whereclause=mozu_image_table.c.bf_imageid == table_args['bf_imageid']) res = update_db.execute() print res, 'Updated--> ', values.items(), ' <-- ', update_db styles_incr_media_version.append(table_args.get('bf_imageid', locals().get('bf_imageid'))) else: print "HTTP Status: {}\n Raising Integrity Error".format(create_resource_resp.http_status_code) raise sqlalchemy.exc.IntegrityError() except ValueError as ve: #sqlalchemy.exc.IntegrityError: print 'VALUE Error and everything is or will be commented out below because it is in the db already', ve #return 'IntegrityError' except KeyError as ke: # sqlalchemy.exc.IntegrityError: print 'Key Error and everything is or will be commented out below because it is in the db already', ke #return 'IntegrityError' #pass # except IOError: # print "ENDING ERROR...", values elif values.get('mz_imageid'): print "KWARGS has MZID: {}".format(values.get('mz_imageid')) styles_incr_media_version.append(values.get('bf_imageid', locals().get('bf_imageid'))) print 'styles list ', styles_incr_media_version return list(set(sorted(styles_incr_media_version)))
def main(root_img_dir=None): import sys,glob,shutil,os,re from image_processing.magick_tweaks import convert_img_srgb regex_coded = re.compile(r'^.+?/[1-9][0-9]{8}_[1-6]\.[JjPpNnGg]{3}$') regex_alt = re.compile(r'^.+?/[1-9][0-9]{8}_\w+?0[1-6]\.[JjPpNnGg]{3}$') regex_valid_style = re.compile(r'^.+?/[1-9][0-9]{8}_?.*?\.[JjPpNnGg]{3}$') if not root_img_dir: try: root_img_dir = sys.argv[1] if root_img_dir == 'jblocal': root_img_dir = os.path.abspath('/mnt/Post_Ready/Retouchers/JohnBragato/MARKETPLACE_LOCAL') else: root_img_dir = os.path.abspath('/mnt/Post_Complete/Complete_Archive/MARKETPLACE') except IndexError: root_img_dir = os.path.abspath('/mnt/Post_Complete/Complete_Archive/MARKETPLACE') pass else: pass try: destdir = os.path.abspath(sys.argv[2]) if not os.path.isdir(destdir): os.makedirs(destdir, 16877) except IndexError: destdir = '/mnt/Post_Complete/ImageDrop/' ##os.path.join(root_img_dir, 'output') # try: # os.makedirs(destdir, 16877) # except OSError: # pass if not type(root_img_dir) == list and os.path.isdir(root_img_dir): #import md5_unique_dup_files #duplicates = md5_unique_dup_files.find_duplicate_imgs(root_img_dir)[1] #[ os.remove(f) for f in duplicates if f ] imgs_renamed = [rename_retouched_file(f) for f in (glob.glob(os.path.join(root_img_dir,'*.??[gG]')))] img_dict = sort_files_by_values(glob.glob(os.path.join(root_img_dir,'*.??[gG]'))) for k,v in img_dict.items(): try: img = k ## Convert profile of source img if CMYK ignores if RGB convert_img_srgb.main(image_file=img) ## Get color pixel values from source img rgbmean = v.items() ## Generate png from source then jpgs from png pngout = subproc_magick_png(img, rgbmean=dict(rgbmean), destdir=destdir) ## TODO: pngout will be the image to POST to mozu returned from mozu # ie. response = send_to_mozu(pngout) then save response in postgres --> and push the id to mozu again ############################################## ## Make Jpegs and collect Finals in mozu loading dir ####### from os import path # chdir , curdir import datetime #, glob, shutil todaysdatefullsecs = '{:%Y%m%d%H%M%S}'.format(datetime.datetime.now()) todaysdatefull = todaysdatefullsecs[:12] todaysdate = todaysdatefull[:8] # '{:%Y,%m,%d}'.format(datetime.datetime.now()) # Define for Creating Archive dirs archive = '/mnt/Post_Complete/Complete_Archive/Uploaded' # archive_uploaded = path.join(archive, "dateloaded_" + str(todaysdate).replace(",", ""), "uploaded_" + str(todaysdatefullsecs).replace(",", "")) archive_uploaded_day = path.join(archive, "dateloaded_" + str(todaysdate).replace(",", "")) imgdest_jpg_mozu = path.join(archive_uploaded_day, 'JPG_MOZU_LOAD') import sys from os import chdir, path import sys,glob,shutil,os,re,datetime sys.path.append('/usr/local/batchRunScripts/python/jbmodules/image_processing/magick_tweaks') sys.path.append('/usr/local/batchRunScripts/mozu') sys.path.append('/usr/local/batchRunScripts/python') sys.path.append('/usr/local/batchRunScripts/python/jbmodules') sys.path.append('/usr/local/batchRunScripts/python/jbmodules/mongo_tools') sys.path.append('/usr/local/batchRunScripts/python/jbmodules/image_processing') sys.path.append('/usr/local/batchRunScripts/python/jbmodules/image_processing/marketplace') #chdir('/usr/local/batchRunScripts/mozu') import mozu_exec, mozu_image_util_functions ## Compress and convert to jpg and store in separate dir for concurrent xfers #if path.isfile(pngout): try: print ' Is file PNGOUT', pngout, img jpgout = mozu_image_util_functions.magick_convert_to_jpeg(pngout,destdir=imgdest_jpg_mozu) except IOError: jpgout = mozu_image_util_functions.magick_convert_to_jpeg(img,destdir=imgdest_jpg_mozu) pass # Try and make a list without needing globbing below mz_converted_jpgs.append(jpgout) ########################################### # from mozu import mozu_exec # mz_res = mozu_exec.main(pngout) # print mz_res, ' <-- MOZU-Done' subproc_magick_large_jpg(pngout, destdir=destdir) subproc_magick_medium_jpg(pngout, destdir=destdir) except AttributeError: print 'SOMETHING IS WRONG WITH THE IMAGE Error {}'.format(img) pass else: 'Image is ', root_img_dir img = root_img_dir if regex_coded.findall(img): img = rename_retouched_file(img) pngout = subproc_magick_png(img, destdir=destdir) #from mozu import mozu_exec as mozu_exec #mz_res = mozu_exec.main(pngout) #print mz_res, ' <-- MOZU-Done' subproc_magick_large_jpg(pngout, destdir=destdir) subproc_magick_medium_jpg(pngout, destdir=destdir)