def duplicate_by_md5_mzimg(filepath, **kwargs):
from db import mozu_image_table_instance
from os import stat
mozu_image_table = mozu_image_table_instance()
bf_imageid = filepath.split('/')[-1].split('.')[0]
filepathMD5 = md5_checksumer(filepath)
dbmd5MD5 = ''
dbmd5BFID = ''
dbmd5MD5 = mozu_image_table.select(
whereclause=((mozu_image_table.c.md5checksum == filepathMD5))).execute().fetchone() # ['md5checksum']
dbmd5BFID = mozu_image_table.select(
whereclause=((mozu_image_table.c.bf_imageid == bf_imageid))).execute().fetchone()
if not dbmd5MD5:
if dbmd5BFID:
print 'Updateable Style, diff checksum\nDB:\t\t', dbmd5BFID[
'md5checksum'], '\nFPATHMD5:\t', filepathMD5, '\nPATH:\t', filepath, '\n'
return filepath
elif filepathMD5 == dbmd5MD5['md5checksum']:
if dbmd5BFID and bf_imageid == dbmd5BFID['bf_imageid']:
print 'Duplicate Checksum:\nDB:\t\t', dbmd5BFID[
'md5checksum'], '\nFPATHMD5:\t', filepathMD5, '\nPATH:\t', stat(filepath)
return False
else:
print 'ChecksumMatch Diffstyle:\nDB:\t\t', dbmd5BFID[
'bf_imageid'], '\nbfImageID:\t', bf_imageid, '\nPATH:\t', filepathMD5
return filepath
else:
print 'ElseD', filepathMD5, bf_imageid
return filepath
def upload_new(**kwargs):
from RESTClient import MozuRestClient
from db import mozu_image_table_instance
mzclient = MozuRestClient(**kwargs)
mz_imageid, document_resource = mzclient.create_new_mz_image()
print mz_imageid, document_resource, ' <-- MozuID -- Docresource'
if document_resource == "Keyerror":
print mz_imageid, ' <-- MozuID'
elif document_resource == "documentTree":
bf_imageid = mzclient.bf_imageid
kwargs['bf_imageid'] = bf_imageid
elif document_resource == "documentListDocumentContent":
kwargs['mz_imageid'] = mz_imageid
mozu_image_table = mozu_image_table_instance()
table_args = include_keys(kwargs, __mozu_image_table_valid_keys__)
insert_db = mozu_image_table.insert(values=dict(**table_args))
print "Inserting with, ", insert_db
if kwargs['bf_imageid']:
content_response = update_content_mz_image(**kwargs)
try:
insert_db.execute()
print 'Inserted --> ', kwargs.items(), ' <-- ', insert_db
except sqlalchemy.exc.IntegrityError:
update_db = mozu_image_table.update(values=dict(**table_args),whereclause=mozu_image_table.c.bf_imageid == table_args['bf_imageid'])
print 'Updating after IntegrityERR with args--> ', kwargs # # Insert to mz_imageid + **kwargs to Oracle
update_db.execute()
return content_response
def insert_data_db(**kwargs):
from db import mozu_image_table_instance
from RESTClient import __mozu_image_table_valid_keys__
mozu_image_table = mozu_image_table_instance()
table_args = include_keys(kwargs, __mozu_image_table_valid_keys__)
insert_db = mozu_image_table.insert(values=dict(**table_args))
#update_db = mozu_image_table.update(values=dict(**table_args), whereclause=mozu_image_table.c.bf_imageid == table_args['bf_imageid'])
insert_db.execute()
def update_content_mz_image(**kwargs):
from RESTClient import MozuRestClient
from db import mozu_image_table_instance
mzclient = MozuRestClient(**kwargs)
content_response = mzclient.send_content(**kwargs)
print content_response.headers, "\nUpdate Mozu Content"
mozu_image_table = mozu_image_table_instance()
table_args = include_keys(kwargs, __mozu_image_table_valid_keys__)
update_db = mozu_image_table.update(values=dict(**table_args))
print content_response.headers, "\nUpdate DB MZ_IMAGE"
return content_response
def delete_document_data_content(**kwargs):
from RESTClient import MozuRestClient
from db import mozu_image_table_instance
mzclient = MozuRestClient(**kwargs)
delete_resp = mzclient.delete_mz_image()
mozu_image_table = mozu_image_table_instance()
if kwargs.get('mz_imageid'):
delete_db = mozu_image_table.delete( whereclause=(mozu_image_table.c.mz_imageid == kwargs.get('mz_imageid')) )
else:
delete_db = mozu_image_table.delete( whereclause=(mozu_image_table.c.bf_imageid == kwargs.get('bf_imageid')) )
# res = delete_db.execute()
print delete_resp.headers, "Delete \n", delete_db, "\nMZ CLIENTID in FUNCtion: ", kwargs
return delete_resp
def delete_by_mozuid(**kwargs):
from os import chdir, path, curdir
try:
chdir(path.join('/usr/local/batchRunScripts', 'mozu'))
print 'Executing from ', path.abspath(curdir), kwargs.get('mz_imageid', '')
except:
pass
resp = delete_document_data_content(mz_imageid=kwargs.get('mz_imageid', ''))
from db import mozu_image_table_instance
mozu_image_table = mozu_image_table_instance()
ret_select = mozu_image_table.select(whereclause=((mozu_image_table.c.mz_imageid == kwargs.get('mz_imageid', '')))).execute().fetchone()
ret = mozu_image_table.delete(whereclause=( (mozu_image_table.c.mz_imageid == kwargs.get('mz_imageid', '') ))).execute()
print 'Deleted bf_imageid', ret #ret.fetchone()
return ret_select['bf_imageid']
def delete_by_bflyid(**kwargs):
from os import chdir, path, curdir
try:
chdir(path.join('/usr/local/batchRunScripts', 'mozu'))
print 'Executing from ', path.abspath(curdir), kwargs.get('bf_imageid', '')
except:
pass
try:
resp = delete_document_data_content(**kwargs)
from db import mozu_image_table_instance
mozu_image_table = mozu_image_table_instance()
try:
ret = mozu_image_table.delete(whereclause=((mozu_image_table.c.mz_imageid == kwargs.get('bf_imageid', '')))).execute()
except:
print('Skipping Delete DB data when using delete by BFID')
return kwargs.get('bf_imageid', '')
except TypeError:
'NoneTpe Error in delete by bfid. {} Not in DB'.format(kwargs.get('bf_imageid', ''))
def get_mozu_or_bf_id(mz_imageid=None,bf_imageid=None,styles_list=None):
from os import chdir, path, curdir
try:
chdir(path.join('/usr/local/batchRunScripts', 'mozu'))
print 'Executing from ', path.abspath(curdir), bf_imageid, mz_imageid
except:
print 'Failed from ', path.abspath(curdir)
from db import mozu_image_table_instance
mozu_image_table = mozu_image_table_instance()
if styles_list:
ret = mozu_image_table.select(whereclause=( (mozu_image_table.c.bf_imageid.in_( tuple(styles_list) )))).execute()
print 'FZero'
return ret.fetchall()
elif bf_imageid:
ret = mozu_image_table.select(whereclause=( (mozu_image_table.c.bf_imageid.like("{}".format(bf_imageid) )))).execute()
print 'F1', ret.fetchone()
return ret.fetchone()
elif mz_imageid:
ret = mozu_image_table.select(whereclause=( (mozu_image_table.c.bf_imageid.like("{}".format(bf_imageid) )))).execute()
print 'F2'
return ret.fetchone()
else:
return
def upsert_data_mz_image(**kwargs):
#import pdb
#pdb.set_trace()
from RESTClient import MozuRestClient
from db import mozu_image_table_instance
mozu_image_table = mozu_image_table_instance()
select_result = mozu_image_table.select( whereclause=(mozu_image_table.c.bf_imageid == kwargs.get('bf_imageid')) ).execute().fetchone()
# test = [ row for row in select_result ]
print select_result, '\n\nTEST -->\n', kwargs # , test
if select_result:
try:
if select_result['mz_imageid']:
kwargs['mz_imageid'] = select_result['mz_imageid']
md5checksum = []
kwargs['md5checksum'] = md5checksum
mzclient = MozuRestClient(**kwargs)
update_resp = mzclient.update_mz_image(**kwargs)
table_args = include_keys(kwargs, __mozu_image_table_valid_keys__)
update_db = mozu_image_table.update(values=dict(**table_args), whereclause=mozu_image_table.c.bf_imageid==kwargs.get('bf_imageid'))
print "1\nUpdate Statement: \t", update_db
update_result = update_db.execute()
print update_result, '2-Updated--> ', kwargs.items(), ' <--kwargs.items ', update_db
return update_resp
else:
mzclient = MozuRestClient(**kwargs)
# mz_imageid, document_resource = mzclient.create_new_mz_image()
mz_imageid = mzclient.get_mz_image_document_list(name=kwargs.get('bf_imageid'))['id']
# kwargs['mz_imageid'], kwargs['mozu_url'] = mzclient.create_new_mz_image() #mzclient.create_new_mz_image(**kwargs)
if mz_imageid: #type(post_resp) == dict:
kwargs['mz_imageid'] = mz_imageid # mz_imageid.keys()[0]
table_args = include_keys(kwargs, __mozu_image_table_valid_keys__)
try:
content_response = mzclient.send_content(**kwargs)
insert_db = mozu_image_table.insert(**table_args)
print "3-\nInsert Statement: \t", insert_db
insert_result = insert_db.execute()
print content_response, "Not in DB. Insert Result: ", insert_result.fetchone()['mz_imageid']
return insert_result.fetchone()
except sqlalchemy.exc.IntegrityError:
content_response = mzclient.send_content(**kwargs)
if kwargs.get('bf_imageid'):
kwargs.get('bf_imageid')
update_db = mozu_image_table.update(values=dict(**table_args),
whereclause= # mozu_image_table.c.bf_imageid==kwargs.get('bf_imageid')
(mozu_image_table.c.bf_imageid == table_args['bf_imageid'])
|
(mozu_image_table.c.mz_imageid == table_args['mz_imageid'])
)
print "4-\nUpdate Statement: \t", update_db
update_result = update_db.execute()
print content_response, '4.5-Updated--> Maybe', kwargs.items(), ' <--kwargs.items ', update_db
return update_result.fetchone()
else:
# print 'NO BFID to update ', locals()
insert_db = mozu_image_table.insert(**table_args)
print "5-\nFailed Insert Statement: \t", insert_db
else:
print mz_imageid, ' Failed'
except TypeError:
print ' 128 Type-aka-OldAttribError in Upsert_mz_exec Locals -->', locals()
#pass
else:
res = upload_new(**kwargs)
return res
def main(fileslist):
import sqlalchemy
from db import mozu_image_table_instance
from mozu_image_util_functions import compile_todict_for_class_instance_variables, magick_convert_to_jpeg, netsrv101_path_maker
# Compiles Data Payload and other Vars per Doc -- Including src_filepath -- **values keys set per instance
# print type(fileslist), '<--Type\tLenLoFilepaths', len(fileslist), '\t', fileslist
### Date Defs
from os import path # chdir , curdir
import datetime #, glob, shutil
todaysdatefullsecs = '{:%Y%m%d%H%M%S}'.format(datetime.datetime.now())
todaysdatefull = todaysdatefullsecs[:12]
todaysdate = todaysdatefull[:8] # '{:%Y,%m,%d}'.format(datetime.datetime.now())
# Define for Creating Archive dirs
archive = '/mnt/Post_Complete/Complete_Archive/Uploaded'
# archive_uploaded = path.join(archive, "dateloaded_" + str(todaysdate).replace(",", ""), "uploaded_" + str(todaysdatefullsecs).replace(",", ""))
archive_uploaded_day = path.join(archive, "dateloaded_" + str(todaysdate).replace(",", ""))
imgdest_jpg_mozu = path.join(archive_uploaded_day, 'JPG_MOZU_LOAD')
# imgdest_jpg_mozu_loaded = path.join(imgdest_jpg_mozu, 'LOADED')
if path.dirname(fileslist[0]).split('/')[-1] == 'JPG_MOZU_LOAD':
# fileslistX= [magick_convert_to_jpeg(f) for f in fileslist if f.split('.')[-1] == 'png']
fileslist = [magick_convert_to_jpeg(f, destdir=imgdest_jpg_mozu) for f in fileslist if f]
else:
fileslist = fileslist # [ path.abspath(f) for f in fileslist if f ]
if not path.isfile(fileslist[0]):
fileslist = netsrv101_path_maker(fileslist)
compiled_instance_vars = compile_todict_for_class_instance_variables(fileslist=fileslist)
# print type(compiled_instance_vars), '<--Type\tLenCompiledInsVars', len(compiled_instance_vars), '\tKeys: ', compiled_instance_vars.keys()
# print compiled_instance_vars, "186-MZEXECY"
styles_incr_media_version = []
for key,values in compiled_instance_vars.iteritems():
# v = include_keys(values, __mozu_image_table_valid_keys__)
# print "IncludedKeys: {}\n\tkey:\t{}\n\tvalues:\t{}".format(v.items(), key , values.popitem())
if not values.get('mz_imageid'):
# ### --> src_filepath = k # will need src_filepath in order to perfom any image manipulation
# ## ---> before loading(would actually need to redo the md5checksum from compiler)
# Insert -- Then try Update if Insert to DB fails or Create NewDoc Fails to Mozu
try:
values['mz_imageid'], response = upload_new(**values)
create_resource_resp = upload_new(**values)
mozu_image_table = mozu_image_table_instance()
print 'Initial Post to Mozu {0}'.format(create_resource_resp.keys()[0])
if int(create_resource_resp.keys()[0]) == int(201):
table_args = include_keys(values, __mozu_image_table_valid_keys__)
insert_db = mozu_image_table.insert(values=dict(**table_args))
res_insrt = insert_db.execute()
print 'Inserted --> ', values.items(), ' <-- ', dir(res_insrt)
styles_incr_media_version.append(table_args.get('bf_imageid', locals().get('bf_imageid')))
elif int(create_resource_resp.keys()[0]) <= int(409):
table_args = include_keys(values, __mozu_image_table_valid_keys__)
mz_imageid = mozu_image_table.select( whereclause=( (mozu_image_table.c.bf_imageid == table_args['bf_imageid']) ) ).execute().fetchone()['mz_imageid']
bf_imageid = mozu_image_table.select( whereclause=( (mozu_image_table.c.mz_imageid == table_args['mz_imageid']) ) ).execute().fetchone()['bf_imageid']
table_args['bf_imageid'] = values['bf_imageid'] = bf_imageid
table_args['mz_imageid'] = values['mz_imageid'] = mz_imageid
resp = update_content_mz_image(**values)
print('RESP 207 mzexec: {}'.format(resp))
#upsert_content_resp = upsert_data_mz_image(**values) # ,dict(**values))
if resp.http_status_code < 400:
update_db = mozu_image_table.update(values=dict(**table_args),whereclause=mozu_image_table.c.bf_imageid == table_args['bf_imageid'])
res = update_db.execute()
print res, 'Updated--> ', values.items(), ' <-- ', update_db
styles_incr_media_version.append(table_args.get('bf_imageid', locals().get('bf_imageid')))
else:
print "HTTP Status: {}\n Raising Integrity Error".format(create_resource_resp.http_status_code)
raise sqlalchemy.exc.IntegrityError()
except ValueError as ve: #sqlalchemy.exc.IntegrityError:
print 'VALUE Error and everything is or will be commented out below because it is in the db already', ve
#return 'IntegrityError'
except KeyError as ke: # sqlalchemy.exc.IntegrityError:
print 'Key Error and everything is or will be commented out below because it is in the db already', ke
#return 'IntegrityError'
#pass
# except IOError:
# print "ENDING ERROR...", values
elif values.get('mz_imageid'):
print "KWARGS has MZID: {}".format(values.get('mz_imageid'))
styles_incr_media_version.append(values.get('bf_imageid', locals().get('bf_imageid')))
print 'styles list ', styles_incr_media_version
return list(set(sorted(styles_incr_media_version)))
def get_multi_mzid_by_bf_imageid(bf_imageid):
from db import mozu_image_table_instance
mozu_image_table = mozu_image_table_instance()
ret = mozu_image_table.select(whereclause=( (mozu_image_table.c.bf_imageid.like("%{}".format(bf_imageid) )))).execute()
return ret.fetchall()
