timestamp = get_timestamp(day_delta=1)
else:
timestamp = sys.argv[1]
table_name = "raw_data_%s" % timestamp
db = DBUtils()
keywords = []
c = db.select(table='brand')
for one in c.fetchall():
id,name,buss = one
keywords.append(name)
site_types = {}
c = db.select(table='sites')
for one in c.fetchall():
id,name,url,site_type = one
site_types[url] = site_type
print site_types.items()
cursor = db.select(table=table_name,columns=["count(*)"])
(total,) = cursor.fetchone()
DELTA = 10000
for start in range(0,total,DELTA):
thread.start_new_thread(filter_data,(db,table_name,start,DELTA,timestamp,keywords,site_types))
db.close()
print start_time," ",datetime.now()
