def main():
import sys,glob,shutil,os,re,datetime
sys.path.append('/usr/local/batchRunScripts/python/jbmodules/image_processing/magick_tweaks')
sys.path.append('/usr/local/batchRunScripts/mozu')
sys.path.append('/usr/local/batchRunScripts/python')
sys.path.append('/usr/local/batchRunScripts/python/jbmodules')
sys.path.append('/usr/local/batchRunScripts/python/jbmodules/mongo_tools')
sys.path.append('/usr/local/batchRunScripts/python/jbmodules/image_processing')
sys.path.append('/usr/local/batchRunScripts/python/jbmodules/image_processing/marketplace')
regex_coded = re.compile(r'^.+?/[1-9][0-9]{8}_[1-6]\.jpg$')
regex_alt = re.compile(r'^.+?/[1-9][0-9]{8}_\w+?0[1-6]\.[JjPpNnGg]{3}$')
regex_valid_style = re.compile(r'^.+?/[1-9][0-9]{8}_?.*?\.[JjPpNnGg]{3}$')
### Date Defs
todaysdatefullsecs = '{:%Y%m%d%H%M%S}'.format(datetime.datetime.now())
todaysdatefull = todaysdatefullsecs[:12]
todaysdate = todaysdatefull[:8] # '{:%Y,%m,%d}'.format(datetime.datetime.now())
todaysdatearch = todaysdatefull # '{:%Y,%m,%d,%H,%M}'.format(datetime.datetime.now())
### Define tmp and archive paths prior to Creating
tmp_processing = os.path.join("/mnt/Post_Complete/Complete_to_Load/.tmp_processing" , "tmp_" + str(todaysdatefullsecs).replace(",", ""))
tmp_processing_l = os.path.join(tmp_processing, "largejpg")
tmp_processing_m = os.path.join(tmp_processing, "mediumjpg")
tmp_loading = os.path.join("/mnt/Post_Complete/Complete_Archive/.tmp_loading" , "tmp_" + str(todaysdatefullsecs).replace(",", ""))
tmp_mozu_loading = os.path.join("/mnt/Post_Complete/Complete_Archive/.tmp_mozu_loading" , "tmp_" + str(todaysdatefullsecs).replace(",", ""))
## Define for Creating Archive dirs
archive = '/mnt/Post_Complete/Complete_Archive/Uploaded'
archive_uploaded = os.path.join(archive, "dateloaded_" + str(todaysdate).replace(",", ""), "uploaded_" + str(todaysdatefullsecs).replace(",", ""))
imgdest_jpg_mozu = os.path.join(archive, "dateloaded_" + str(todaysdate), 'JPG_MOZU_LOAD')
#imgdest_jpg_final = os.path.join(archive_uploaded, 'JPG_RETOUCHED_ORIG')
imgdest_png_final = os.path.join(archive_uploaded, 'PNG')
###################
## Create Lock File
###################
#locker = os.path.join(rootdir, 'LOCKED.lock')
#if os.path.isfile(locker):
# break
#else:
# with open(locker, 'wb') as f:
# f.write(todaysdatefull)
# f.close()
try:
root_img_dir = os.path.abspath(sys.argv[1])
destdir = os.path.abspath(sys.argv[2]) #'/Users/johnb/Pictures'
tmp_processing = root_img_dir
tmp_loading = destdir
except IndexError:
root_img_dir = '/mnt/Post_Complete/Complete_to_Load/Drop_FinalFilesOnly'
destdir = tmp_loading
if os.path.isdir(root_img_dir): pass
else:
root_img_dir = '/mnt/Post_Complete/Complete_to_Load/Drop_FinalFilesOnly'
destdir = tmp_loading
###########
## Test for ex
#root_img_dir = tmp_processing
walkedout_tmp = glob.glob(os.path.join(root_img_dir, '*/*.*g'))
image_ct = len(walkedout_tmp)
starttime = todaysdatefull
if image_ct == 0:
print "Nothing to Process"
else:
### Make Tmp Folders for Processing And Uploading -- tmp_dirs are dated with time(hr:min)to prevent collisions
print 'Processing -- %s Files \nStarted --> %s \n' % (image_ct, starttime)
try:
os.makedirs(archive_uploaded, 16877)
except:
pass
try:
os.makedirs(tmp_processing, 16877)
except:
pass
try:
os.makedirs(tmp_processing_l, 16877)
except:
pass
try:
os.makedirs(tmp_processing_m, 16877)
except:
pass
try:
os.makedirs(tmp_loading, 16877)
except:
pass
try:
os.makedirs(imgdest_jpg_mozu, 16877)
except:
pass
try:
os.makedirs(imgdest_png_final, 16877)
except:
pass
# try:
# os.makedirs(imgdest_jpg_final, 16877)
# except:
# pass
####################################################
## Begin Processing and compiling images for Loading
####################################################
## Move All DropFinal Files from Retouchers dirs to tmp_processing from drop folders Then Mogrify to create pngs copy to load and arch dirs
walkedout_tmp = glob.glob(os.path.join(root_img_dir, '*/*.*g'))
[ shutil.move(file, os.path.join(tmp_processing, os.path.basename(file))) for file in walkedout_tmp ]
### Rename Files moved into Temp Processing Floder
walkedout_tmp = glob.glob(os.path.join(tmp_processing, '*.jpg'))
[ rename_retouched_file(file) for file in walkedout_tmp ]
mz_converted_jpgs = []
if os.path.isdir(tmp_processing):
for img in glob.glob(os.path.join(tmp_processing,'*.??[gG]')):
if regex_coded.findall(img):
img = rename_retouched_file(img)
try:
pngout = subproc_magick_png(img, destdir=tmp_processing)
subproc_magick_large_jpg(pngout, destdir=tmp_loading)
subproc_magick_medium_jpg(pngout, destdir=tmp_loading)
##############################################
##### BEGIN MOZU pt 1 of 2 ##################
##############################################
## Make Jpegs and collect Finals in mozu loading dir
#######
import sys
from os import chdir, path
#chdir('/usr/local/batchRunScripts/mozu')
import mozu_exec, mozu_image_util_functions
## Compress and convert to jpg and store in separate dir for concurrent xfers
#if path.isfile(pngout):
try:
print ' Is file PNGOUT', pngout, img
jpgout = mozu_image_util_functions.magick_convert_to_jpeg(pngout,destdir=imgdest_jpg_mozu)
except IOError:
jpgout = mozu_image_util_functions.magick_convert_to_jpeg(img,destdir=imgdest_jpg_mozu)
pass
# Try and make a list without needing globbing below
mz_converted_jpgs.append(jpgout)
except AttributeError:
# from shutil import move
print "AttributeError at 780 in magickLoad_PrdCmp.py --> ", img
#move(img, "/mnt/Post_Complete/Complete_to_Load/")
###########################################
###### END mozu ###########################
###########################################
#metadict = metadata_info_dict(img)
#dimens = get_imagesize_variables(img)
#test_img = get_image_color_minmax(img)
##########################################################
### Glob created PNGs and copy to Load Dir then Store in Arch dir
tmp_png = glob.glob(os.path.join(tmp_processing, '*.png'))
[ shutil.copy2(file, os.path.join(tmp_loading, os.path.basename(file))) for file in tmp_png ]
[ shutil.move(file, os.path.join(imgdest_png_final, os.path.basename(file))) for file in tmp_png ]
## ARCHIVED Backup
## All JPGs in Root dir Only of tmp_processing will be now Archived as all Conversions are completed
###### All PNGs Created and moved to Archive plus Copy sent to Load Directory
#### All Files Converted for Upload, Now glob search and move large and medium named jpgs to tmp loading
import time
upload_tmp_loading = glob.glob(os.path.join(tmp_loading, '*.*g'))
# upload_tmp_loading_mozuall = glob.glob(os.path.join(imgdest_jpg_mozu, '*.jpg'))
for upload_file in upload_tmp_loading:
#### UPLOAD upload_file via NFS to imagedrop
## Then rm loading tmp dir
try:
result = copy_to_imagedrop_upload(upload_file, destdir='/mnt/Post_Complete/ImageDrop')
if result:
print "Uploaded {}".format(upload_file)
time.sleep(float(.1))
shutil.move(upload_file, archive_uploaded)
else:
print result, upload_file
pass
##time.sleep(float(3))
# try:
# ftpload_to_imagedrop(upload_file)
# print "Uploaded {}".format(upload_file)
# time.sleep(float(.3))
# shutil.move(upload_file, archive_uploaded)
# except:
# pass
except:
print "Error moving Finals to Arch {}".format(file)
######################################################
### ##### BEGIN MOZU pt 2 of 2 ######################
######################################################
## TODO: Fix so all go through funcs. Glob Mozu Jpegs to tmp Originals dir for now by passing productions sharpening funcs
######################################################
## Mozu Xfer ####
import mozu_exec, mozu_image_util_functions, glob
# upload_imgdest_jpg_mozu_glob = glob.glob(os.path.join(imgdest_jpg_mozu, '*.*g'))
#
# print 'MozuExec', '\n\n\n\n\v\v\t\t\t\tMozuExec\t', ' <-- Now MozuExec'
# mozu_exec.main(upload_imgdest_jpg_mozu_glob)
#######################################################
############### END MOZU ##############################
#######################################################
### Check for okb files and send to Uploader via email
zerobytefiles = glob.glob(os.path.join('/mnt/Post_Complete/Complete_to_Load/Drop_FinalFilesOnly/zero_byte_errors', '*/*.*g'))
## After completed Process and Load to imagedrop
### Finally Remove the 2 tmp folder trees for process and load if Empty
try:
upload_tmp_loading_remainder = glob.glob(os.path.join(tmp_loading, '*.*g'))
if len(upload_tmp_loading_remainder) == 0 and os.path.isdir(tmp_loading):
shutil.rmtree(tmp_loading)
except:
pass
## remove the processing tmp folder
if os.path.isdir(tmp_processing):
shutil.rmtree(tmp_processing)
else:
print tmp_processing, ' tmp_processing Does not Exist, tmp_loading-->', tmp_loading
def main(fileslist):
import sqlalchemy
from db import mozu_image_table_instance
from mozu_image_util_functions import compile_todict_for_class_instance_variables, magick_convert_to_jpeg, netsrv101_path_maker
# Compiles Data Payload and other Vars per Doc -- Including src_filepath -- **values keys set per instance
# print type(fileslist), '<--Type\tLenLoFilepaths', len(fileslist), '\t', fileslist
### Date Defs
from os import path # chdir , curdir
import datetime #, glob, shutil
todaysdatefullsecs = '{:%Y%m%d%H%M%S}'.format(datetime.datetime.now())
todaysdatefull = todaysdatefullsecs[:12]
todaysdate = todaysdatefull[:8] # '{:%Y,%m,%d}'.format(datetime.datetime.now())
# Define for Creating Archive dirs
archive = '/mnt/Post_Complete/Complete_Archive/Uploaded'
# archive_uploaded = path.join(archive, "dateloaded_" + str(todaysdate).replace(",", ""), "uploaded_" + str(todaysdatefullsecs).replace(",", ""))
archive_uploaded_day = path.join(archive, "dateloaded_" + str(todaysdate).replace(",", ""))
imgdest_jpg_mozu = path.join(archive_uploaded_day, 'JPG_MOZU_LOAD')
# imgdest_jpg_mozu_loaded = path.join(imgdest_jpg_mozu, 'LOADED')
if path.dirname(fileslist[0]).split('/')[-1] == 'JPG_MOZU_LOAD':
# fileslistX= [magick_convert_to_jpeg(f) for f in fileslist if f.split('.')[-1] == 'png']
fileslist = [magick_convert_to_jpeg(f, destdir=imgdest_jpg_mozu) for f in fileslist if f]
else:
fileslist = fileslist # [ path.abspath(f) for f in fileslist if f ]
if not path.isfile(fileslist[0]):
fileslist = netsrv101_path_maker(fileslist)
compiled_instance_vars = compile_todict_for_class_instance_variables(fileslist=fileslist)
# print type(compiled_instance_vars), '<--Type\tLenCompiledInsVars', len(compiled_instance_vars), '\tKeys: ', compiled_instance_vars.keys()
# print compiled_instance_vars, "186-MZEXECY"
styles_incr_media_version = []
for key,values in compiled_instance_vars.iteritems():
# v = include_keys(values, __mozu_image_table_valid_keys__)
# print "IncludedKeys: {}\n\tkey:\t{}\n\tvalues:\t{}".format(v.items(), key , values.popitem())
if not values.get('mz_imageid'):
# ### --> src_filepath = k # will need src_filepath in order to perfom any image manipulation
# ## ---> before loading(would actually need to redo the md5checksum from compiler)
# Insert -- Then try Update if Insert to DB fails or Create NewDoc Fails to Mozu
try:
values['mz_imageid'], response = upload_new(**values)
create_resource_resp = upload_new(**values)
mozu_image_table = mozu_image_table_instance()
print 'Initial Post to Mozu {0}'.format(create_resource_resp.keys()[0])
if int(create_resource_resp.keys()[0]) == int(201):
table_args = include_keys(values, __mozu_image_table_valid_keys__)
insert_db = mozu_image_table.insert(values=dict(**table_args))
res_insrt = insert_db.execute()
print 'Inserted --> ', values.items(), ' <-- ', dir(res_insrt)
styles_incr_media_version.append(table_args.get('bf_imageid', locals().get('bf_imageid')))
elif int(create_resource_resp.keys()[0]) <= int(409):
table_args = include_keys(values, __mozu_image_table_valid_keys__)
mz_imageid = mozu_image_table.select( whereclause=( (mozu_image_table.c.bf_imageid == table_args['bf_imageid']) ) ).execute().fetchone()['mz_imageid']
bf_imageid = mozu_image_table.select( whereclause=( (mozu_image_table.c.mz_imageid == table_args['mz_imageid']) ) ).execute().fetchone()['bf_imageid']
table_args['bf_imageid'] = values['bf_imageid'] = bf_imageid
table_args['mz_imageid'] = values['mz_imageid'] = mz_imageid
resp = update_content_mz_image(**values)
print('RESP 207 mzexec: {}'.format(resp))
#upsert_content_resp = upsert_data_mz_image(**values) # ,dict(**values))
if resp.http_status_code < 400:
update_db = mozu_image_table.update(values=dict(**table_args),whereclause=mozu_image_table.c.bf_imageid == table_args['bf_imageid'])
res = update_db.execute()
print res, 'Updated--> ', values.items(), ' <-- ', update_db
styles_incr_media_version.append(table_args.get('bf_imageid', locals().get('bf_imageid')))
else:
print "HTTP Status: {}\n Raising Integrity Error".format(create_resource_resp.http_status_code)
raise sqlalchemy.exc.IntegrityError()
except ValueError as ve: #sqlalchemy.exc.IntegrityError:
print 'VALUE Error and everything is or will be commented out below because it is in the db already', ve
#return 'IntegrityError'
except KeyError as ke: # sqlalchemy.exc.IntegrityError:
print 'Key Error and everything is or will be commented out below because it is in the db already', ke
#return 'IntegrityError'
#pass
# except IOError:
# print "ENDING ERROR...", values
elif values.get('mz_imageid'):
print "KWARGS has MZID: {}".format(values.get('mz_imageid'))
styles_incr_media_version.append(values.get('bf_imageid', locals().get('bf_imageid')))
print 'styles list ', styles_incr_media_version
return list(set(sorted(styles_incr_media_version)))
def main(root_img_dir=None):
import sys,glob,shutil,os,re
from image_processing.magick_tweaks import convert_img_srgb
regex_coded = re.compile(r'^.+?/[1-9][0-9]{8}_[1-6]\.[JjPpNnGg]{3}$')
regex_alt = re.compile(r'^.+?/[1-9][0-9]{8}_\w+?0[1-6]\.[JjPpNnGg]{3}$')
regex_valid_style = re.compile(r'^.+?/[1-9][0-9]{8}_?.*?\.[JjPpNnGg]{3}$')
if not root_img_dir:
try:
root_img_dir = sys.argv[1]
if root_img_dir == 'jblocal':
root_img_dir = os.path.abspath('/mnt/Post_Ready/Retouchers/JohnBragato/MARKETPLACE_LOCAL')
else:
root_img_dir = os.path.abspath('/mnt/Post_Complete/Complete_Archive/MARKETPLACE')
except IndexError:
root_img_dir = os.path.abspath('/mnt/Post_Complete/Complete_Archive/MARKETPLACE')
pass
else:
pass
try:
destdir = os.path.abspath(sys.argv[2])
if not os.path.isdir(destdir):
os.makedirs(destdir, 16877)
except IndexError:
destdir = '/mnt/Post_Complete/ImageDrop/' ##os.path.join(root_img_dir, 'output')
# try:
# os.makedirs(destdir, 16877)
# except OSError:
# pass
if not type(root_img_dir) == list and os.path.isdir(root_img_dir):
#import md5_unique_dup_files
#duplicates = md5_unique_dup_files.find_duplicate_imgs(root_img_dir)[1]
#[ os.remove(f) for f in duplicates if f ]
imgs_renamed = [rename_retouched_file(f) for f in (glob.glob(os.path.join(root_img_dir,'*.??[gG]')))]
img_dict = sort_files_by_values(glob.glob(os.path.join(root_img_dir,'*.??[gG]')))
for k,v in img_dict.items():
try:
img = k
## Convert profile of source img if CMYK ignores if RGB
convert_img_srgb.main(image_file=img)
## Get color pixel values from source img
rgbmean = v.items()
## Generate png from source then jpgs from png
pngout = subproc_magick_png(img, rgbmean=dict(rgbmean), destdir=destdir)
## TODO: pngout will be the image to POST to mozu returned from mozu
# ie. response = send_to_mozu(pngout) then save response in postgres --> and push the id to mozu again
##############################################
## Make Jpegs and collect Finals in mozu loading dir
#######
from os import path # chdir , curdir
import datetime #, glob, shutil
todaysdatefullsecs = '{:%Y%m%d%H%M%S}'.format(datetime.datetime.now())
todaysdatefull = todaysdatefullsecs[:12]
todaysdate = todaysdatefull[:8] # '{:%Y,%m,%d}'.format(datetime.datetime.now())
# Define for Creating Archive dirs
archive = '/mnt/Post_Complete/Complete_Archive/Uploaded'
# archive_uploaded = path.join(archive, "dateloaded_" + str(todaysdate).replace(",", ""), "uploaded_" + str(todaysdatefullsecs).replace(",", ""))
archive_uploaded_day = path.join(archive, "dateloaded_" + str(todaysdate).replace(",", ""))
imgdest_jpg_mozu = path.join(archive_uploaded_day, 'JPG_MOZU_LOAD')
import sys
from os import chdir, path
import sys,glob,shutil,os,re,datetime
sys.path.append('/usr/local/batchRunScripts/python/jbmodules/image_processing/magick_tweaks')
sys.path.append('/usr/local/batchRunScripts/mozu')
sys.path.append('/usr/local/batchRunScripts/python')
sys.path.append('/usr/local/batchRunScripts/python/jbmodules')
sys.path.append('/usr/local/batchRunScripts/python/jbmodules/mongo_tools')
sys.path.append('/usr/local/batchRunScripts/python/jbmodules/image_processing')
sys.path.append('/usr/local/batchRunScripts/python/jbmodules/image_processing/marketplace')
#chdir('/usr/local/batchRunScripts/mozu')
import mozu_exec, mozu_image_util_functions
## Compress and convert to jpg and store in separate dir for concurrent xfers
#if path.isfile(pngout):
try:
print ' Is file PNGOUT', pngout, img
jpgout = mozu_image_util_functions.magick_convert_to_jpeg(pngout,destdir=imgdest_jpg_mozu)
except IOError:
jpgout = mozu_image_util_functions.magick_convert_to_jpeg(img,destdir=imgdest_jpg_mozu)
pass
# Try and make a list without needing globbing below
mz_converted_jpgs.append(jpgout)
###########################################
# from mozu import mozu_exec
# mz_res = mozu_exec.main(pngout)
# print mz_res, ' <-- MOZU-Done'
subproc_magick_large_jpg(pngout, destdir=destdir)
subproc_magick_medium_jpg(pngout, destdir=destdir)
except AttributeError:
print 'SOMETHING IS WRONG WITH THE IMAGE Error {}'.format(img)
pass
else:
'Image is ', root_img_dir
img = root_img_dir
if regex_coded.findall(img):
img = rename_retouched_file(img)
pngout = subproc_magick_png(img, destdir=destdir)
#from mozu import mozu_exec as mozu_exec
#mz_res = mozu_exec.main(pngout)
#print mz_res, ' <-- MOZU-Done'
subproc_magick_large_jpg(pngout, destdir=destdir)
subproc_magick_medium_jpg(pngout, destdir=destdir)
