def processfile(df, fn, d):
logger.debug("Updating fn = "+fn+", d = "+d)
columnHeaders = list(df.columns.values)
columnArray = np.asarray(columnHeaders)
# dTyper is a dictionary of Columns and their types to be passed to executeSQL_UPDATE
dTyper = {k: dTypes[k] for k in dTypes.keys() & columnArray}
# kLister is a dictionary of keys to be passed to executeSQL_UPDATE
kLister = {k: kList[k] for k in kList.keys() & columnArray}
aTypesr = {k: aTypes[k] for k in aTypes.keys() & columnArray}
aNamesr = {k: aNames[k] for k in aNames.keys() & columnArray}
typersr = {k: typers[k] for k in typers.keys() & columnArray}
for k, v in list(aNamesr.items()):
if v == None:
del aNamesr[k]
del aTypesr[k]
del typersr[k]
for k, v in list(kLister.items()):
if v != 'K':
del kLister[k]
try:
# export.executeSQL_UPDATE(engine, df, fn, dTyper, kLister, log)
logger.debug("{0} SQL_UPDATE: {1} with {2} rows".format( timestamp(), fn, df.shape[0] ))
export.executeSQL_UPDATE(engine, df, fn, dTyper, kLister, aTypesr, aNamesr, typersr, log)
logger.debug("{0} SQL_UPDATE: {1} with {2} rows [DONE]".format( timestamp(), file, df.shape[0] ))
# Define and create the directory for all the output files
# directory = '{path}/{folder}'.format(path=invalid_path,folder=fn)
# os.makedirs(directory,exist_ok=True)
# df.to_csv('{path}/{fn}_{dt}.csv'.format(path=directory,fn=fn,dt=d), index = False)
#df.to_csv('{path}/{fn}_{dt}.csv'.format(path=invalid_path,fn=fn,dt=d), index = False)
except:
logger.debug("{0} SQL_UPDATE: No updated data for {1}".format( timestamp(), fn ))
logger.error('---Error in file: %s the folder will be skipped' % file)
raise
return
archive_file = pd.read_csv( os.path.join(archive_path, subdir, lastarchive_filename),
encoding='ansi', dtype='str',
na_values=None, keep_default_na=False )
df = export.createDiff( inputFrame, archive_file )
else:
df = inputFrame
if writedb:
#attempts to execute code catching any errors that may arrise then breaks out of loop of folder
if df.shape[0] > 0:
try:
print("\t\t\t{0} SQL_UPDATE: {1} with {2} rows".format( timestamp(), file, df.shape[0] ))
log.write("{0} SQL_UPDATE: {1} with {2} rows\n".format( timestamp(), file, df.shape[0] ))
export.executeSQL_UPDATE(engine, df, sqlName, dTyper, kLister, aTypesr, aNamesr, typersr, log)
print("\t\t\t{0} SQL_UPDATE: {1} with {2} rows [DONE]".format( timestamp(), file, df.shape[0] ))
log.write("{0} SQL_UPDATE: {1} with {2} rows [DONE]\n".format( timestamp(), file, df.shape[0] ))
except:
print('\t\t\t---Error in file: %s the folder will be skipped' % file)
log.write('---Error in file: %s the folder will be skipped\n' % file)
break
else:
print("\t\t\t{0} SQL_UPDATE: No updated data for {1}".format( timestamp(), file ))
log.write("{0} SQL_UPDATE: No updated data for {1}\n".format( timestamp(), file ))
#archives the files in another directory if their are no exceptions
print("\t\t\t{0} Archive: {1}".format( timestamp(), file ))
log.write("{0} Archive: {1}\n".format( timestamp(), file ))