def insert_into_device_detail_table(self, device_detail_array,
device_detail_array_new):
self.db = my.connect(host="localhost",
user="******",
passwd=str(self.pass_obj),
db="aawaz")
self.cursor = self.db.cursor()
logger.info(
"Total numbers of record to insert in analytics_device_detail table are "
+ str(len(device_detail_array) + len(device_detail_array_new)))
device_detail_sql = "insert into analytics_device_detail(date_time, event_id, app_id, version_code,\
version_name, device_name, user_id, board, brand, device, hardware, manufacturer, model, time, sdk_int, \
os_built_release_version, app_platform) VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);"
device_detail_sql_new = "insert into analytics_device_detail(date_time, event_id, app_id, version_code,\
version_name, device_name, user_id, board, brand, device, hardware, manufacturer, model, time, sdk_int, \
os_built_release_version) VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);"
self.insert_db(device_detail_sql, device_detail_array,
"analytics_device_detail")
self.insert_db(device_detail_sql_new, device_detail_array_new,
"analytics_device_detail_new")
device_detail_array.clear()
device_detail_array_new.clear()
self.db.close()
def move_file(self, source, destination):
logger.info("Moving files to ==> " + destination + "\n")
try:
shutil.move(source, destination)
except:
logger.info(
"Can't move this file\nSimilar name file already exist in the directory\n"
)
def __init__(self):
value = getpass.getpass('Database Password: ')
self.value = value
try:
self.db = my.connect(host="localhost",
user="******",
passwd=str(value),
db="aawaz")
self.cursor = self.db.cursor()
verify_sql = "use aawaz;"
execution = self.cursor.execute(verify_sql)
self.db.close()
except Exception as e:
logger.exception("Wrong Password \nReason: " + e)
sys.exit()
# Parser for commandline input
parser = argparse.ArgumentParser(
description="Enter Command line switch",
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
groupSource = parser.add_mutually_exclusive_group(required=True)
groupSource.add_argument("-s",
"--source",
action="store",
default=None,
help="Source directory absolute path")
groupMoveOrDelete = parser.add_mutually_exclusive_group(required=True)
groupMoveOrDelete.add_argument(
"-d",
"--delete",
action="store",
default=None,
help="Delete source files after success")
groupMoveOrDelete.add_argument("-m",
"--move",
action="store",
default=None,
help="Move source files after success")
args = parser.parse_args()
logger.info("args: " + str(args))
if (args.delete):
logger.info("delete: " + str(args.delete))
elif (args.move):
logger.info("move" + str(args.move))
else:
logger.info("move or delete required")
if (args.source):
logger.info("source" + str(args.source))
def verify(self):
if ((sys.argv[1] == "-s" or sys.argv[1] == "--source")
and os.path.exists(sys.argv[2]) == True):
logger.info("Verified Source path\n")
pass
if (sys.argv[3] == "-m" or sys.argv[3] == "--move"):
pass
logger.info("Verified Move path\n")
if (os.path.exists(sys.argv[4]) == False):
os.mkdir(sys.argv[4])
logger.info("New Directory made ==> " + sys.argv[4] + "\n")
elif (sys.argv[3] == "-d" or sys.argv[3] == "--delete"):
logger.info("Verified Delete path\n")
pass
else:
logger.info("Directory does not exist\n")
sys.exit()
def insert_into_next_table(self, next_array, next_array_new):
self.db = my.connect(host="localhost",
user="******",
passwd=str(self.pass_obj),
db="aawaz")
self.cursor = self.db.cursor()
logger.info(
"Total numbers of record to insert in analytics_next table are " +
str(len(next_array) + len(next_array_new)))
next_sql = "insert into analytics_next(date_time, event_id, user_id, show_id, episode_id, start_time, app_platform)\
VALUES(%s, %s, %s, %s, %s, %s, %s);"
next_sql_new = "insert into analytics_next(date_time, event_id, user_id, show_id, episode_id, start_time)\
VALUES(%s, %s, %s, %s, %s, %s);"
self.insert_db(next_sql, next_array, "analytics_next")
self.insert_db(next_sql_new, next_array_new, "analytics_next_new")
next_array.clear()
next_array_new.clear()
self.db.close()
def insert_into_search_table(self, search_array, search_array_new):
self.db = my.connect(host="localhost",
user="******",
passwd=str(self.pass_obj),
db="aawaz")
self.cursor = self.db.cursor()
logger.info(
"Total numbers of record to insert in analytics_search table are "
+ str(len(search_array) + len(search_array_new)))
search_sql = "insert into analytics_search(date_time, event_id, user_id, query, no_of_show_count, show_ids,\
no_of_article_count, article_ids, app_platform)VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s);"
search_sql_new = "insert into analytics_search(date_time, event_id, user_id, query, no_of_show_count, show_ids,\
no_of_article_count, article_ids)VALUES(%s, %s, %s, %s, %s, %s, %s, %s);"
self.insert_db(search_sql, search_array, "analytics_search")
self.insert_db(search_sql_new, search_array_new,
"analytics_search_new")
search_array.clear()
search_array_new.clear()
self.db.close()
def insert_into_app_open_table(self, app_open_array, app_open_array_new):
self.db = my.connect(host="localhost",
user="******",
passwd=str(self.pass_obj),
db="aawaz")
self.cursor = self.db.cursor()
logger.info(
"Total numbers of record to insert in analytics_app_open table are "
+ str(len(app_open_array) + len(app_open_array_new)))
app_open_sql = "insert into analytics_app_open(date_time, event_id, user_id, app_open, app_platform)\
VALUES(%s, %s, %s, %s, %s);"
app_open_sql_new = "insert into analytics_app_open(date_time, event_id, user_id, app_open)\
VALUES(%s, %s, %s, %s);"
self.insert_db(app_open_sql, app_open_array, "analytics_app_open")
self.insert_db(app_open_sql_new, app_open_array_new,
"analytics_app_open_new")
app_open_array.clear()
app_open_array_new.clear()
self.db.close()
def insert_into_previous_table(self, previous_array, previous_array_new):
self.db = my.connect(host="localhost",
user="******",
passwd=str(self.pass_obj),
db="aawaz")
self.cursor = self.db.cursor()
logger.info(
"Total numbers of record to insert in analytics_previous table are "
+ str(len(previous_array) + len(previous_array_new)))
previous_sql = "insert into analytics_previous(date_time, event_id, user_id, show_id, episode_id, app_platform)\
VALUES(%s, %s, %s, %s, %s, %s);"
previous_sql_new = "insert into analytics_previous(date_time, event_id, user_id, show_id, episode_id)\
VALUES(%s, %s, %s, %s, %s);"
self.insert_db(previous_sql, previous_array, "analytics_previous")
self.insert_db(previous_sql_new, previous_array_new,
"analytics_previous")
previous_array.clear()
previous_array_new.clear()
self.db.close()
def move_or_delete(self):
#Generating extract path and unzip csv files
for root, dirs, files in os.walk(sys.argv[2]):
for zip in files:
if zip.endswith(".zip"):
zippath = os.path.join(root, zip)
logger.info("\nExtracting file " + zippath + "\n")
self.unzip(zippath, os.path.dirname(zippath))
if (sys.argv[3] == "-m" or sys.argv[3] == "-move"):
validate.Validate.move_file(self, zippath, sys.argv[4])
elif (sys.argv[3] == "-d" or sys.argv[3] == "-delete"):
validate.Validate.delete_file(self, zippath)
#List csv files and append its data into database
for root, dirs, files in os.walk(sys.argv[2]):
for csv in files:
if csv.endswith(".csv"):
csvpath = os.path.join(root, csv)
self.csvpath = csvpath
logger.info(
"----------------------------------------------------------------------------------------\n"
)
logger.info("CSV path to insert: " + csvpath + "\n")
self.log_obj.array_append(csvpath)
if (sys.argv[3] == "-m" or sys.argv[3] == "-move"):
validate.Validate.move_file(self, csvpath, sys.argv[4])
elif (sys.argv[3] == "-d" or sys.argv[3] == "-delete"):
validate.Validate.delete_file(self, csvpath)
def delete_file(self, source):
logger.info("Deleting compressed file ==> " + source + "\n")
os.chmod(source, stat.S_IRWXU)
os.remove(source)
def array_append(self, path):
# Value of each event
PLATFORM = "android"
PLAY_ID = "18"
PAUSE_ID = "20"
NEXT_ID = "21"
PREVIOUS_ID = "22"
FAST_FWD_ID = "23"
REWIND_ID = "24"
STOP_ID = "25"
APP_OPEN_ID = "44"
DEVICE_DETAIL_ID = "45"
SEARCH_ID = "40"
# CSV file read and append in array
with open(path, 'r') as DataCaptured:
DataCap = csv.reader(DataCaptured,
delimiter='^',
skipinitialspace=True)
for i, line in enumerate(DataCap):
if (line[1] == NEXT_ID):
if (line[-1] == "play_debug" or line[-1] == "play_release"
or line[-1] == "indus_debug"
or line[-1] == "indus_release"):
self.next_array.append(line)
elif (line[-1] == "dev_debug"
or line[-1] == "dev_release"):
pass
elif (line[-1] != "play_debug"
or line[-1] != "play_release"
or line[-1] != "indus_debug"
or line[-1] != "indus_release"):
self.next_array_new.append(line)
elif (line[1] == PLAY_ID):
if (line[-1] == "play_debug" or line[-1] == "play_release"
or line[-1] == "indus_debug"
or line[-1] == "indus_release"):
self.play_array.append(line)
elif (line[-1] == "dev_debug"
or line[-1] == "dev_release"):
pass
elif (line[-1] != "play_debug"
or line[-1] != "play_release"
or line[-1] != "indus_debug"
or line[-1] != "indus_release"):
self.play_array_new.append(line)
elif (line[1] == PAUSE_ID):
if (line[-1] == "play_debug" or line[-1] == "play_release"
or line[-1] == "indus_debug"
or line[-1] == "indus_release"):
self.pause_array.append(line)
elif (line[-1] == "dev_debug"
or line[-1] == "dev_release"):
pass
elif (line[-1] != "play_debug"
or line[-1] != "play_release"
or line[-1] != "indus_debug"
or line[-1] != "indus_release"):
self.pause_array_new.append(line)
elif (line[1] == PREVIOUS_ID):
if (line[-1] == "play_debug" or line[-1] == "play_release"
or line[-1] == "indus_debug"
or line[-1] == "indus_release"):
self.previous_array.append(line)
elif (line[-1] == "dev_debug"
or line[-1] == "dev_release"):
pass
elif (line[-1] != "play_debug"
or line[-1] != "play_release"
or line[-1] != "indus_debug"
or line[-1] != "indus_release"):
self.previous_array_new.append(line)
elif (line[1] == FAST_FWD_ID):
if (line[-1] == "play_debug" or line[-1] == "play_release"
or line[-1] == "indus_debug"
or line[-1] == "indus_release"):
self.fast_fwd_array.append(line)
elif (line[-1] == "dev_debug"
or line[-1] == "dev_release"):
pass
elif (line[-1] != "play_debug"
or line[-1] != "play_release"
or line[-1] != "indus_debug"
or line[-1] != "indus_release"):
self.fast_fwd_array_new.append(line)
elif (line[1] == REWIND_ID):
if (line[-1] == "play_debug" or line[-1] == "play_release"
or line[-1] == "indus_debug"
or line[-1] == "indus_release"):
self.rewind_array.append(line)
elif (line[-1] == "dev_debug"
or line[-1] == "dev_release"):
pass
elif (line[-1] != "play_debug"
or line[-1] != "play_release"
or line[-1] != "indus_debug"
or line[-1] != "indus_release"):
self.rewind_array_new.append(line)
elif (line[1] == STOP_ID):
if (line[-1] == "play_debug" or line[-1] == "play_release"
or line[-1] == "indus_debug"
or line[-1] == "indus_release"):
self.stop_array.append(line)
elif (line[-1] == "dev_debug"
or line[-1] == "dev_release"):
pass
elif (line[-1] != "play_debug"
or line[-1] != "play_release"
or line[-1] != "indus_debug"
or line[-1] != "indus_release"):
self.stop_array_new.append(line)
elif (line[1] == APP_OPEN_ID):
if (line[-1] == "play_debug" or line[-1] == "play_release"
or line[-1] == "indus_debug"
or line[-1] == "indus_release"):
self.app_open_array.append(line)
elif (line[-1] == "dev_debug"
or line[-1] == "dev_release"):
pass
elif (line[-1] != "play_debug"
or line[-1] != "play_release"
or line[-1] != "indus_debug"
or line[-1] != "indus_release"):
self.app_open_array_new.append(line)
elif (line[1] == DEVICE_DETAIL_ID):
if (line[-1] == "play_debug" or line[-1] == "play_release"
or line[-1] == "indus_debug"
or line[-1] == "indus_release"):
self.device_detail_array.append(line)
elif (line[-1] == "dev_debug"
or line[-1] == "dev_release"):
pass
elif (line[-1] != "play_debug"
or line[-1] != "play_release"
or line[-1] != "indus_debug"
or line[-1] != "indus_release"):
self.device_detail_array_new.append(line)
elif (line[1] == SEARCH_ID):
if (line[-1] == "play_debug" or line[-1] == "play_release"
or line[-1] == "indus_debug"
or line[-1] == "indus_release"):
self.search_array.append(line)
elif (line[-1] == "dev_debug"
or line[-1] == "dev_release"):
pass
elif (line[-1] != "play_debug"
or line[-1] != "play_release"
or line[-1] != "indus_debug"
or line[-1] != "indus_release"):
self.search_array_new.append(line)
else:
logger.info("Invalid Input")
total_inserted_data = len(self.play_array) + len(
self.pause_array
) + len(self.next_array) + len(self.previous_array) + len(
self.fast_fwd_array) + len(self.rewind_array) + len(
self.stop_array) + len(self.app_open_array) + len(
self.device_detail_array) + len(self.search_array) + len(
self.play_array_new) + len(self.pause_array_new) + len(
self.next_array_new) + len(
self.previous_array_new) + len(
self.fast_fwd_array_new) + len(
self.rewind_array_new) + len(
self.stop_array_new) + len(
self.app_open_array_new
) + len(
self.device_detail_array_new
) + len(self.search_array_new)
self.insert_stats_array.append(str(datetime.datetime.now()))
self.insert_stats_array.append(path)
self.insert_stats_array.append(total_inserted_data)
logger.info(str(self.insert_stats_array) + "\n")
# Calling table's function
self.insert_into_play_table(self.play_array, self.play_array_new)
self.insert_into_pause_table(self.pause_array, self.pause_array_new)
self.insert_into_next_table(self.next_array, self.next_array_new)
self.insert_into_previous_table(self.previous_array,
self.previous_array_new)
self.insert_into_ff_table(self.fast_fwd_array, self.fast_fwd_array_new)
self.insert_into_rewind_table(self.rewind_array, self.rewind_array_new)
self.insert_into_stop_table(self.stop_array, self.stop_array_new)
self.insert_into_app_open_table(self.app_open_array,
self.app_open_array_new)
self.insert_into_device_detail_table(self.device_detail_array,
self.device_detail_array_new)
self.insert_into_search_table(self.search_array, self.search_array_new)
self.insert_into_insert_stats_table(self.insert_stats_array)