def assign_new_id(connection, table):
"""
must have the connection and table -
all calls from this module should use the
connection and table assigned above, but there
will be a wrapper object to handle that.
will also assume that the id field is name "table_ID"
this is really just a prototype and might be replaced
with more generic methods later.
"""
pass
curs = connection.cursor()
call = "SELECT MAX(%s_ID) FROM %s;" % (table, table)
log("Checking with %s" % (call), log_path)
results = curs.execute(call)
res = results.fetchall()
try:
res = int(res)
log("Identified %s as maximum ID..." % (res), log_path)
except Exception as ECHO:
log("EXCEPTION OCCURRED: %s " % (ECHO), error_path)
res = None
if res is None:
res = 0
else:
res += 1
log("New maximum ID = %s" % (res), log_path)
return res
def find_least_recent_file(search_path,pattern = None):
"""
will attempt to find the least recently modified file in search_path,
matching pattern "pattern". I'll prabably adjust this to do a regex search
later, but for now, only capable of a rudimentary containment search.
"""
pattern = pattern or "";
if type(search_path) != str:
log("TypeError occurred: [search_path] must be a string argument.","error_log.log");
raise TypeError("[search_path] must be a string argument!");
if type(pattern) != str:
log("TypeError occurred: <pattern> must be a string argument.","error_log.log");
raise TypeError("<pattern> must be a string argument!");
if not os.path.isdir(search_path): raise Exception("input must be a path!");
if not os.path.isabs(search_path):
print("Warning: you may encounter errors if the pat provided isn't local, or complete.\n\n%s"%(search_path))
path_items = [];
for a in os.listdir(search_path):
if os.path.isabs(search_path):
a = search_path+os.sep+a;
local_item = []
print(a);
if pattern not in a: continue;
local_item.append(a); local_item.append(os.path.getmtime(a));
log("Adding items to consider: %s"%(str(local_item)));
path_items.append(tuple(local_item))
def find_most_recent_file(search_path,pattern = None):
"""
will attempt to find the most recently modified file in search_path,
matching pattern "pattern". I'll probably adjust this to do a regex search
later, but for now, it is only capable of a rudimentary contaiment search.
Successfully locates the most updated file in a directory matching pattern -
I'll add in a user interface of some sort for this, but for now it isn't necessary.
"""
pattern = pattern or "";
if type(search_path) != str:
log("TypeError occurred: [search_path] must be a string argument.","error_log.log");
raise TypeError("[search_path] must be a string argument!");
if type(pattern)!= str:
log("TypeError occurred: <pattern> must be a string argument.","error_log.log");
raise TypeError("<pattern> must be a string argument!");
if not os.path.isdir(search_path): raise Exception("input must be a path!");
if not os.path.isabs(search_path):
print("Warning: you may encounter errors if the path provided isn't local, or complete.\n\n%s"%(search_path))
path_items = []
for a in os.listdir(search_path):
if os.path.isabs(search_path):
a = search_path+os.sep+a;
local_item = []
#a #= os.path.realpath(a)
print(a)
if pattern not in a: continue
# it appears that real path is causing an issue - I'll have to construct an absolute path.
local_item.append(a); local_item.append(os.path.getmtime(a)); # tuple[1] will be the M time.
log("Adding items to consider: %s"%(str(local_item))) # I could take these and store them... maybe in future development.
path_items.append(tuple(local_item))
most_recent = max_tup(path_items,1)
return most_recent;
def main(path, table, columns):
"""
make and commit a table - try to consolidate instructions for import.
"""
log("Initializing self:")
# now - time for a main loop.
my_self = ich_will()
my_self.path = path
my_self.table = table
my_self.columns = columns
# generate the values, then build the table.
log("launching creation process:")
my_self.create()
log("attempt to create the runcounter file:")
with open(os.path.dirname(my_self.path) + os.sep + "runcounter",
'w') as out:
out.write('1')
log("appears to have worked.")
print('done')
def create(self):
"""
check the values of the attributes,
then generate the file and update the runcounter.
"""
# run type checks - keep breaking until they get it right.
if type(self.columns) not in [list, tuple]:
message = "columns must be of type list or tuple!"
log(message, './activity_log.log')
log(message, './error_log.log')
raise TypeError("columns must be of type list!")
if type(self.path) != str:
message = "path must be of type str!"
log(message, './activity_log.log')
log(message, './error_log.log')
raise TypeError("path must be of type str!")
if type(self.table) != str:
message = "table must be of type str!"
log(message, "./activity_log.log")
log(message, "./error_log.log")
raise TypeError(message)
conn = sqlite3.connect(self.path)
curs = conn.cursor()
# make query:
comm = ich_will.create_table_query(self.table, self.columns)
curs.execute(comm)
log("Generated Query- Executed.")
conn.commit()
log("Generated table and committed")
return
def create_table_query(table_name, columns):
"""
will assign one column to be the index,
but all subsequent columns will be of varchar 255.
this should be adequate for most of what we'll be trying to accomplish.
add support for column typing later - for now treat everything as a string.
returns a valid sql query to create a table (might extract this later for use.)
"""
if type(columns) not in [list, tuple]:
message = "columns must be list or tuple!"
log(message, "activity_log.log")
log(message, "error_log.log")
raise ValueError(message)
if type(table_name) not in [str]:
message = "table_name must be a string!"
log(message, "activity_log.log")
log(message, "error_log.log")
raise ValueError(message)
# going to try something here -
query_base = "CREATE TABLE %s ( %s_ID INT AUTO INCREMENT " % (
table_name, table_name)
log("Generated query base: %s" % (query_base), "./activity_log.log")
for a in columns:
if type(a) == tuple:
query_base += ", %s %s" % (a[0], a[1])
log("added: ,%s %s" % (a[0], a[1]))
if type(a) == str:
query_base += ", %s VARCHAR(255)" % (a)
log("added: , %s VARCHAR(255)" % (a))
query_base += ");"
log("QUERY: %s" % (query_base))
return query_base
def check_for_record(connection, seek_value, columns=None, table=None):
"""
see if an item appears in the exclusionary table.
if it exists, count should be > 1 otherwise it isn't.
I just want to return a true/false, then handle what
happens afterward in whatever the calling process is.
RETURNS LIST OF TUPLES WITH COLUMN_NAME, # OF HITS.
"""
pass
# setting up to handle some overrides or defaults.
columns = columns or exclusion_columns
table = table or exclusion_table
if type(columns) != list:
message = "Columns must be list type, got %s for %s." % (type(columns),
str(columns))
log(message, log_path)
log(message, error_path)
raise Exception(message)
# this should provide a cursory check - might be better with specialized queries.
# alternatively, I could return an array with the hits found in each column.
col_count = []
base = "SELECT COUNT(%s) FROM %s WHERE %s LIKE '%s'" # we'll assume that we're reading strings
for a in columns:
#
match = [a]
log("Checking for references in column: %s" % (a), log_path)
try:
curs = connection.cursor()
tst = curs.execute(base % (a, table, a, seek_value))
match.append(int(tst.fetchall()[0][0]))
log("Found %s references." % (match[1]), log_path)
except Exception as eddy:
message = "Unhandled exception has occurred: %s inputs: column: %s | table: %s | seek_value: %s" % (
eddy, a, table, seek_value)
log(message, log_path)
log(message, error_path)
continue
# using this to prevent appending invalid data.
col_count.append(tuple(match))
return col_count
def insert_new_record(connection, values, columns=None, table=None):
"""
I'll need to set up a method that handles the variable nature of my mechanisms. - some of them
might need to map to values in different tables.
fair warning - it will assume that your insert statement will match the definition for columns
in the configuration section.
Still running into issues - if the last value in the list h
"""
# initialize some vals.
if type(values) != list:
message = "Values must be a list of values (current: %s)" % (
str(values))
log(message, log_path)
log(message, error_path)
raise ValueError(message)
query_base = "INSERT INTO "
# these will need to be modified to exist within a class - exclusion columns etc...
columns = columns or exclusion_columns
table = table or exclusion_table
while len(values) < len(columns):
values.append("null")
if len(values) > len(columns):
values = values[:len(columns)]
base = prepare_insert(table, columns)
print(base)
for a in values:
try:
if values.index(a) < len(columns):
#base = base%(a+" %s")
base += "'%s', " % (a)
else:
base += "'%s'" % (a)
except Exception as edna:
log("Can't add parameter(%s): %s" % (a, edna))
continue
base = base[:-1] + " )"
base.replace(", )", " )")
#base = base%(values)
curs = connection.cursor()
try:
print(base)
curs.execute(base)
connection.commit()
log("Successfully wrote values!", log_path)
except Exception as echo:
message = "unhandled exception occurred: %s" % (echo)
log(message, log_path)
log(message, error_path)
raise Exception(str(echo))
error_path = "./error_log.log"
exclusion_table = "loaded_files"
exclusion_columns = ["file_id", "file_name", "original_path", "file_hash"]
### END CONFIGURATIONS ###
# RUN INITIALIZATION IF IT DOESN'T FIND THE RUNCOUNTER #
# I"m going to set this up to run as a method in the class rather than in the entire module.
run_init = False
if os.path.isfile(os.path.dirname(data_path) + os.sep + "runcounter"):
pass
data = ""
with open(os.path.dirname(data_path) + os.sep + "runcounter") as dt:
data = dt.read()
log(
"Updating data, checking %s, value: %s" %
(os.path.dirname(data_path) + os.sep + "runcounter", data),
log_path)
try:
data = int(data)
if data < 1:
run_init = True
log(
"Appears that the runcounter isn't what it needs to be - running init process.",
log_path)
else:
log(
"Initialization appears to have run successfully, nothing to do.",
log_path)
except ValueError as VE:
log(
"Appears that someone has been tinkering with the runcounter file. Naughty Naughty.\n%s"