def doJob():
'''
Do the scrape every interval time
'''
#get all csv and db paths
conn = sqlite3.connect('data/setting.db')
c = conn.cursor()
csv_db_file_list = c.execute(
'SELECT CSV_FILE_PATH, DB_FILE_PATH FROM CSV_DB').fetchall()
c.close()
conn.close()
threads = []
fblogin()
for item in csv_db_file_list:
csv_path = item[0]
db_path = item[1]
s = Scraper()
thread = ScrapeThread(s, csv_path, db_path)
#thread.start()
thread.run() # run threads one by one
threads.append(thread)
# Wait for all threads to complete
#for t in threads:
# t.join()
log.info('all scraper threads finished in doJob()')
return
def doJob():
'''
Do the scrape every interval time
'''
#get all csv and db paths
conn = sqlite3.connect('data/setting.db')
c = conn.cursor()
csv_db_file_list = c.execute('SELECT CSV_FILE_PATH, DB_FILE_PATH FROM CSV_DB').fetchall()
c.close()
conn.close()
threads = []
fblogin()
for item in csv_db_file_list:
csv_path = item[0]
db_path = item[1]
s = Scraper()
thread = ScrapeThread(s,csv_path, db_path)
#thread.start()
thread.run() # run threads one by one
threads.append(thread)
# Wait for all threads to complete
#for t in threads:
# t.join()
log.info('all scraper threads finished in doJob()')
return
company.micro_metrics['tw_percent'],
company.micro_metrics['yt_percent'],
company.micro_metrics['fb_abs'],
company.micro_metrics['tw_abs'],
company.micro_metrics['yt_abs'],
company.time_taken
))
count += 1
except Exception as e:
log.error(e)
pass
conn.commit()
c.close()
conn.close()
return count
if __name__ == '__main__':
log.info('begin')
args = sys.argv
if len(args) >= 2:
file = open(args[1], 'r')
fblogin()
s = Scraper()
count = s.write_db(s.get_social_media(s.read_csv(file), 'data/data.db'), 'data/data.db')
print '\n'
print '%d records has been saved to database %s' % (count, 'data/data.db')
else:
print 'Please input the file name as the first parameter.'
log.info('end')
