def grab_zips(sftp): # dirlist on remote host print "%s: Reading directory..." % timestamp() dirlist = sftp.listdir('.') dirs_count=len(dirlist) completed_count=0 skipped_count=0 grabbed_count=0 print "%s: Directory contains %d items" % (timestamp(),dirs_count) sys.stdout.write("\rGrabbed: %d Skipped: %d Processed: %d (%d%%)" % (grabbed_count,skipped_count,completed_count,0)) sys.stdout.flush() for remotefile in dirlist: # processing completed_count += 1 try: if is_for_grab(remotefile): grabbed_count += 1 sftp.get(remotefile, remotefile) unzip(remotefile) else: skipped_count += 1 #sftp.remove(remotefile) except: print "Error..." # statistics percent=(completed_count*100)/dirs_count sys.stdout.write("\rGrabbed: %d Skipped: %d Processed: %d (%d%%)" % (grabbed_count,skipped_count,completed_count,percent)) sys.stdout.flush() print "\n%s: Completed!" % timestamp()
def parse_configurations_csv(csv_lines): del csv_lines[0] #header row insertions = [] track_ids = {} for line in csv_lines: parts = line.split(',') if len(parts) >= 22: if parts[0] not in track_ids: track_ids[parts[0]]=0; insertions.append((parts[0], parts[1], parts[21], timestamp())) return insertions
def parse_resources_csv(csv_lines): track_num_id=0 start_time_id=1 items_limit=2 if csv_lines[0].split(',')[0]=='fileVersion' and csv_lines[1].split(',')[0]=='14': track_num_id=1 start_time_id=2 items_limit=3 #header row del csv_lines[0] insertions = [] time_map = {} for line in csv_lines: parts = line.split(',') if len(parts) >= items_limit: time_map[parts[track_num_id]]=parts[start_time_id]; for (tracking_num, start_time) in time_map.iteritems(): insertions.append((tracking_num, start_time, timestamp())) return insertions
def insert_site_detailes(site,domain,installationId,startTime,desktopBuild,userId,userFirstName,userLastName): try: params=(site,domain,installationId,startTime,desktopBuild,userId,userFirstName,userLastName,timestamp(),installationId) db_connection.cursor().execute(insert_site_detail_if_not_exists % params) db_connection.commit() except mdb.Error, e: print "\nError %d: %s" % (e.args[0], e.args[1])
cursor.execute(insert_into_report % (table_name,siteName,domainName,table_name,siteName,domainName)) db_connection.commit() print_progress("%s table: %s build update" % (table_name,build_num_label)) cursor.execute(update_report_sql % (table_name,build_num_label,build_num_label,siteName,domainName)) db_connection.commit() except mdb.Error, e: print "\nError %d: %s" % (e.args[0], e.args[1]) # Main try: launched=False quit_if_launched('reports.index') launched=True print "Start time: %s" % timestamp() # MySQL connection if debug: db_connection = mdb.connect('127.0.0.1', 'root', 'root', 'amazon'); else: db_connection = mdb.connect('did-db2.cx50jcbnfwsq.us-east-1.rds.amazonaws.com', 'did_db2_master', 'D75QHSq4iz2WTuF', 'did_db2'); query_result=None if prepare_report_tables() is True: cursor = db_connection.cursor() cursor.execute(select_join_tracking_number % 90) print "Result Select has finished" query_result = cursor.fetchall() if query_result:
def parse_sessions_csv(csv_lines): del csv_lines[0] #header row insertions = [] track_ids = {} for line in csv_lines: parts = line.split(',') if len(parts) >= 6: if parts[0] not in track_ids: track_ids[parts[0]]=0; insert_site(parts[1], parts[2]) insertions.append((parts[0], parts[1], parts[2], parts[3], parts[4], parts[5], timestamp())) return insertions
def insert_site_detailes(site,domain,installationId,startTime,desktopBuild,userId,userFirstName,userLastName): table_name = "%s_%s" % (site, domain) table_name=table_name.replace('.', '_', 5) db_connection.cursor().execute(create_site_table % table_name) db_connection.commit() params=(table_name,installationId,startTime,desktopBuild,userId,userFirstName,userLastName,timestamp(),table_name,installationId,desktopBuild,userId) db_connection.cursor().execute(insert_site_detail_if_not_exists % params) db_connection.commit()
# main(): try: launched=False launched=quit_if_launched('csv_to_db.index') launched=True # MySQL connection if debug: db_connection = mdb.connect('127.0.0.1', 'root', 'root', 'amazon'); else: db_connection = mdb.connect('did-db2.cx50jcbnfwsq.us-east-1.rds.amazonaws.com', 'did_db2_master', 'D75QHSq4iz2WTuF', 'did_db2'); # process data read_bucket() cursor = db_connection.cursor() cursor.execute(select_site_detailes) fetch = None #cursor.fetchall() if fetch: for record in fetch: insert_site_detailes(record[0], record[1], record[2], record[3], record[4], record[5], record[6], record[7]) except mdb.Error, e: print "Error %d: %s" % (e.args[0], e.args[1]) sys.exit(1) finally: if db_connection: db_connection.close() if launched: os.remove('csv_to_db.index') print "\nEnd time: %s" % timestamp()
def insert_site_detailes(site,domain,installationId,startTime,desktopBuild,userId,userFirstName,userLastName): params=(site,domain,installationId,startTime,desktopBuild,userId,userFirstName,userLastName,timestamp(),installationId,desktopBuild,userId) db_connection.cursor().execute(insert_site_detail_if_not_exists % params) db_connection.commit()