def doJob(): ''' Do the scrape every interval time ''' #get all csv and db paths conn = sqlite3.connect('data/setting.db') c = conn.cursor() csv_db_file_list = c.execute( 'SELECT CSV_FILE_PATH, DB_FILE_PATH FROM CSV_DB').fetchall() c.close() conn.close() threads = [] fblogin() for item in csv_db_file_list: csv_path = item[0] db_path = item[1] s = Scraper() thread = ScrapeThread(s, csv_path, db_path) #thread.start() thread.run() # run threads one by one threads.append(thread) # Wait for all threads to complete #for t in threads: # t.join() log.info('all scraper threads finished in doJob()') return
def doJob(): ''' Do the scrape every interval time ''' #get all csv and db paths conn = sqlite3.connect('data/setting.db') c = conn.cursor() csv_db_file_list = c.execute('SELECT CSV_FILE_PATH, DB_FILE_PATH FROM CSV_DB').fetchall() c.close() conn.close() threads = [] fblogin() for item in csv_db_file_list: csv_path = item[0] db_path = item[1] s = Scraper() thread = ScrapeThread(s,csv_path, db_path) #thread.start() thread.run() # run threads one by one threads.append(thread) # Wait for all threads to complete #for t in threads: # t.join() log.info('all scraper threads finished in doJob()') return
company.micro_metrics['tw_percent'], company.micro_metrics['yt_percent'], company.micro_metrics['fb_abs'], company.micro_metrics['tw_abs'], company.micro_metrics['yt_abs'], company.time_taken )) count += 1 except Exception as e: log.error(e) pass conn.commit() c.close() conn.close() return count if __name__ == '__main__': log.info('begin') args = sys.argv if len(args) >= 2: file = open(args[1], 'r') fblogin() s = Scraper() count = s.write_db(s.get_social_media(s.read_csv(file), 'data/data.db'), 'data/data.db') print '\n' print '%d records has been saved to database %s' % (count, 'data/data.db') else: print 'Please input the file name as the first parameter.' log.info('end')