def execQuery(conn, sql, output_file_path=None, global_variable=None):
logger.debug("Starting execQuery Query=%s output_file_path=%s" %
(sql, output_file_path))
rows = None
curs = None
outfile = None
fields = None
logger.debug("Connection to database successful.")
curs = conn.cursor()
logger.debug("Executing query %s" % (sql))
curs.execute(sql)
logger.debug("Fetching results of query.")
rows = curs.fetchall()
row_count = curs.rowcount
if global_variable:
if row_count > 0:
asys.setGV(global_variable, rows[0][0])
else:
asys.setGV(global_variable, "NULL")
logger.debug("%d rows fetched." % (len(rows)))
if row_count == 0:
logger.info("No matching records found.")
else:
field_names = []
count = 0
logger.debug("Collecting field names.")
logger.debug("%d field names are in each record." %
(len(curs.description)))
for record in curs.description:
logger.debug("Field name to add is %s.unt is %d" %
(record[0], count))
field_names.insert(count, record[0])
logger.debug("Added Field named: %s to list" %
(field_names[count]))
count += 1
fields = ','.join(field_names)
curs.close()
# print output to screen
if row_count > 0:
print "%s\n" % (fields)
for row in rows:
record = ','.join(map(str, row))
print record
if output_file_path:
output_file = open(output_file_path, 'w')
output_file.write("%s\n" % (fields))
for row in rows:
record = ','.join(map(str, row))
output_file.write("%s\n" % (record))
output_file.close()
return 0
def execute( conn, sql, global_variable ):
logger.debug( "Executing query %s." % ( sql ))
logger.debug( "Global Variable %s." % ( global_variable ))
logger.debug( "Starting execute. Query %s." % ( sql ))
rows = None
curs = None
fields = None
resultset = None
row_count = 0
curs = conn.cursor()
logger.debug("Executing query %s" % ( sql ) )
curs.execute(sql)
if sql.strip().upper().startswith("SELECT"):
logger.debug("SELECT statement detected. Retrieving recordset." )
records = curs.fetchall()
logger.debug("%d records retrieved." % ( len(records) ) )
if len(records) == 0:
logger.debug("No records returned from query." )
if global_variable:
logger.debug("Setting global variable %s to Null." % ( global_variable ) )
asys.setGV(global_variable, None)
curs.close()
logger.debug( "No records returned. Row count = %d" % ( row_count ) )
return None
elif len(records) > 0:
row_count = len(records)
logger.debug( "%d records returned from query." % ( row_count ) )
if global_variable:
logger.debug( "Settting global vaiiable %s = %s" % ( global_variable, str(records[0][0]) ) )
asys.setGV(global_variable, records[0][0])
logger.info( "Global variable %s set to %s" % ( global_variable, records[0][0] ) )
else:
logger.info( "Global variable not specified so no global variable has been set." )
logger.debug("Collecting field names.")
columns = [column[0] for column in curs.description]
curs.close()
logger.debug("There are %d fields. Field names are %s." % ( len(columns), '.'.join(columns) ) )
resultset = []
resultset.append( columns )
logger.debug( "Appending %d records to the resultset." % ( row_count ) )
for record in records:
resultset.append(record)
else:
logger.debug( "DML operation requested. Not returning any records." )
return resultset
except Exception, err:
logger.error("Failed to initialize logging. Reason: %s" % (str(err)))
traceback.print_exc(file=sys.stdout)
return 2
try:
file_count = countFiles(args.src_dir_path, args.src_file_mask)
except Exception, err:
logger.error(
"Failed to count files with pattern %s in folder %s. Reason: %s" %
(args.src_file_mask, args.src_dir_path, str(err)))
traceback.print_exc(file=sys.stdout)
return 2
try:
if args.gv:
asys.setGV(args.gv, file_count)
logger.info("%d files in %s matched file name pattern %s" %
(file_count, args.src_dir_path, args.src_file_mask))
if args.error:
if file_count == 0:
return 1
else:
return 0
except Exception, err:
logger.error("Failed to set global variable %s to value %s" %
(args.gv, file_count))
traceback.print_exc(file=sys.stdout)
return 2
#return exit code 1 if no files were found and the error flag is set to true
if error and file_count == 0:
return 1
return 2
try:
copied_files = copyFiles(args.src_dir_path, files, args.dst_dir_path,
args.timestamp, args.format, args.error,
args.overwrite, args.global_variable,
args.prepend)
except Exception, err:
logger.error("Exception encountered while copying files. Reason: %s" %
(str(err)))
traceback.print_exc()
return 2
try:
if args.global_variable:
asys.setGV(global_variable, len(copied_files))
except Exception, err:
logger.error(
"Exception encountered setting global variable.Reason: %s" %
(str(err)))
traceback.print_exc()
return 2
if len(copied_files) == 0 and args.error:
logger.error(
"Error: %d files were copied from \"%s\" to \"%s\". and error flag was set."
% (len(copied_files), os.path.realpath(
args.src_dir_path), os.path.realpath(args.dst_dir_path)))
return 1
else:
for file in copied_files:
logger.error("Invalid command line syntax. Reason: %s" % (str(err)))
traceback.print_exc()
return 2
except Exception, err:
logger.error("Failed to parse command line arguments. Reason: %s" %
(str(err)))
traceback.print_exc()
return 2
try:
initLogging(mc['level'])
except Exception, err:
logger.error("Failed to initialize logging. Reason: %s" % (str(err)))
traceback.print_exc()
return 2
try:
total = sum_values(mc["values"])
logger.info("Total is %d" % total)
if mc["global_variable"]:
asys.setGV(mc["global_variable"], str(total))
except Exception, err:
logger.error("Failed to sum values. Reason: %s" % (str(err)))
traceback.print_exc()
return 2
return 0
if __name__ == "__main__":
sys.exit(main())
def monitorFile( src_dir_path, mask, interval, timeout, sort_order, file_min_size, file_max_size, global_variable, access, reverse_condition,file_stability_threshold ):
src_file_path = None
src_dir_path = os.path.realpath(src_dir_path)
start_time = datetime.now()
#ensure the values are in a safe range.
if interval < 3:
interval = 3
if file_stability_threshold < 0:
file_stability_threshold = 0
if file_min_size < 0:
file_min_size = 0
if timeout < 0:
timeout = 0
#check access and existence of the source folder
if os.path.exists(src_dir_path) == False:
raise Exception( "Cannot access path %s. Please check path and permissions. " % ( src_dir_path ) )
elif os.path.isdir(src_dir_path) == False:
raise Exception( logger.error( "Path %s is not a directory. Please check path. " % ( src_dir_path ) ) )
#check for the existence of a file matching name criteria
logger.debug( "Starting MonitorFile at %s. Watching directory %s for a file %s with timeout %d and interval %d" % ( start_time.strftime('%Y/%m/%d %H:%M:%S'), src_dir_path, mask, timeout, interval ) )
src_file_path = None
file_stability_count = 0
#file_stability_threshold = 3
last_file_size = file_size = 0
#File check criteria. The ok means based on what the command indicated is important the file properties are 'ok'
file_exists_ok = False
file_readable = False
file_writable = False
file_executable = False
file_stability_ok = False
file_min_size_ok = False
file_max_size_ok = False
file_read_ok = False
file_write_ok = False
file_execute_ok = False
previous_src_file_path = None
#File criteria checks
while True:
#update the source file path
#check if path was set from previous run and still valid
if src_file_path and os.access( src_file_path, os.F_OK ):
file_exists_ok = True
#get new source file path and reset statisics
else:
file_stability_count = 0
last_file_size = file_size = 0
src_file_path = asys.findFile(src_dir_path, mask, sort_order)
if src_file_path:
file_exists_ok = os.access( src_file_path, os.F_OK )
else:
file_exists_ok = False
#File stats collection block
if file_exists_ok:
logger.debug( "File %s exists. File_Ok: %s .Checking file stats." % ( src_file_path, file_exists_ok ) )
try:
#get file size
file_size = os.path.getsize( src_file_path )
#get file permissions/access
file_readable = os.access( src_file_path, os.R_OK )
file_writable = os.access( src_file_path, os.W_OK )
file_executable = os.access( src_file_path, os.X_OK )
except:
file_exists_ok = False
file_stability_count = 0
pass
#Decision Block
if file_exists_ok:
#File stability check
if timeout == 0:
file_stability_ok = True
elif file_size == last_file_size:
file_stability_count +=1
if file_stability_count >= file_stability_threshold:
file_stability_ok = True
else:
file_stability_ok = False
else:
last_file_size = file_size
file_stability_count = 0
#File minimum size check
if file_min_size == -1:
logger.debug( "File min size value %d disables min size checking." % ( file_min_size ) )
file_min_size_ok = True
elif file_size >= file_min_size:
logger.debug( "File size %d is >= file min size %d" % ( file_size, file_min_size ) )
file_min_size_ok = True
else:
logger.debug( "File size %d is > file min size value %d." % ( file_size, file_min_size ) )
file_min_size_ok = False
#File maximum size check
if file_max_size == -1:
logger.debug( "File max size value %d disables max size checking." % ( file_max_size ) )
file_max_size_ok = True
elif file_size <= file_max_size:
logger.debug( "File size %d is <= file max size %d" % ( file_size, file_max_size ) )
file_max_size_ok = True
else:
logger.debug( "File size %d is greater than max size value %d." % ( file_size, file_max_size ) )
file_max_size_ok = False
#File read access check
if access:
if 'r' in access:
#check the file readability setting
file_read_ok = file_readable
else:
#true if readability check not requested
file_read_ok = True
else:
#true if access parameter not given
file_read_ok = True
#file write access check:
if access:
if 'w' in access:
#check file writability
file_write_ok = file_writable
else:
#true if write check not requested
file_write_ok = True
else:
file_write_ok = True
logger.info( "File Exists:%s| File Stable:%s| Min Size:%s| Max Size:%s| Readable:%s| Writeable:%s" % ( file_exists_ok,file_stability_ok, file_min_size_ok, file_max_size_ok, file_read_ok, file_write_ok ) )
logger.debug("FilePath: %s. Interval: %d. Count: %d. Threshold: %d. last_file_size: %d| Reverse Condition: %s " % ( src_dir_path,interval,file_stability_count,file_stability_threshold,last_file_size, reverse_condition ))
#if looking for the lack of existence of the file
if reverse_condition:
if not file_exists_ok:
logger.info( "File matching %s not found and reverse condition enabled. Returning exit code 0." % ( mask ) )
return 0
#if the file meets the specificied criteria
elif file_stability_ok and file_min_size_ok and file_max_size_ok and file_read_ok and file_write_ok and file_exists_ok:
logger.info( "File %s with file size: %d |Passed checks." % ( src_file_path, last_file_size ) )
#set the global variable if requested
if ( global_variable ):
asys.setGV(global_variable, os.path.basename(src_file_path))
else:
logger.info( "Global variable option not selected. Not setting global variable." )
return 0
if getElapsedTime(start_time ) > timeout:
logger.info( "Timeout was exceeded before file meeting criteria was found. Returning Exit Code 1." )
return 1
#exit conditions not met, sleep and repeat loop
logger.info("Sleeping for %d seconds" % ( interval ) )
time.sleep(interval)
except Exception, err:
logger.error("Exception encountered while deleting files. Reason: %s" %
(str(err)))
traceback.print_exc()
return 2
try:
if deleted_files is None:
raise Exception(
"Exception. Null deleted_file object returned. Fix code. ")
except Exception, err:
traceback.print_exc()
return 2
if args.global_variable:
asys.setGV(args.global_variable, len(deleted_files))
logger.info("Global variable %s set to %d." %
(args.global_variable, len(deleted_files)))
if len(deleted_files) == 0 and args.error:
logger.error("Error: No files were deleted from \"%s\"." %
(os.path.realpath(args.src_dir_path)))
return 1
else:
for file in deleted_files:
logger.info("%s was successfully deleted." % (file))
logger.info("%d files were deleted from \"%s\"." %
(len(deleted_files), os.path.realpath(args.src_dir_path)))
return 0
def setGlobalVariable( glo_name, glo_value ): asys.setGV(glo_name, glo_value) return
logger.info("Writing file list to %s." %
(mc["output_file_path"]))
writeFileList(mc["output_file_path"], file_list)
logger.info("%d files written to list to %s." %
(len(file_list), mc["output_file_path"]))
except Exception, err:
logger.error("Failed to write file list to %s. Reason: %s." %
(mc["output_file_path"], str(err)))
traceback.print_exc()
else:
logger.debug("Output file path is %s. No file will be written." %
(mc["output_file_path"]))
try:
if mc["global_variable"]:
asys.setGV(mc["global_variable"], file_count)
except Exception, err:
print "Failed to set global variable. Reason: %s" % (str(err))
traceback.print_exc()
return 2
if mc["error"] and not len(file_list):
logger.error(
"%d files found and error flag is set. Returning Exit Code 1." %
(len(file_list)))
return 1
else:
return 0
if __name__ == "__main__":