curs.execute(
"select * FROM flights WHERE direction=%s and dst=%s and date=%s and dep_time=%s and company=%s",
(i['direction'], DST, str(i['year']) + "-" + str(i['month']) + "-" +
str(i['day']), '00:00', str(company_id)))
if (len(curs.fetchall()) > 0):
curs.execute(
"DELETE FROM flights WHERE direction=%s and dst=%s and date=%s and dep_time=%s and company=%s",
(i['direction'], DST, str(i['year']) + "-" + str(i['month']) +
"-" + str(i['day']), '00:00', str(company_id)))
curs.execute(
"INSERT INTO flights (company, scrape_time, direction, dst, price, dep_time, arr_time, date, dur_time) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)",
(str(company_id), str(scrape_time), i['direction'], DST, int(
i['price']), '00:00', '00:00', str(i['year']) + "-" +
str(i['month']) + "-" + str(i['day']), get_flight_time(i, DST)))
curs.execute(
"INSERT INTO archive_flights (company, scrape_time, direction, dst, price, dep_time, arr_time, date, dur_time) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)",
(str(company_id), str(scrape_time), i['direction'], DST, int(
i['price']), '00:00', '00:00', str(i['year']) + "-" +
str(i['month']) + "-" + str(i['day']), get_flight_time(i, DST)))
if cleandone == 1:
curs.execute(
"delete from flights where company=%s and dst=%s and date>=%s and date<%s and scrape_time<%s",
(str(company_id), DST, str(Start_orig.strftime("%Y-%m-%d")),
str(Stop.strftime("%Y-%m-%d")), str(scrape_time)))
print "Done!"
db.commit()
print datetime.datetime.now()
print ""
if debug_flag:
print "Debug: efore clean_dup: Out, Inc: "
print flightsList
fl2=up.clean_dup_max(flightsList, 1)
fl3=up.clean_dup_max(flightsList, 2)
fl2.extend(fl3)
if debug_flag:
print "Debug: After clean_dup: Out, Inc: "
for i in fl2:
print i
#db= psycopg2.connect( host="manegerdb.cjjasb6ckbh1.us-east-1.rds.amazonaws.com", database="GabiScrape", user="******", password="******")
curs = db.cursor()
curs.execute("select id from companies where name='elal'")
company_id=curs.fetchone()[0]
for i in fl2:
curs.execute("select * FROM flights WHERE direction=%s and dst=%s and date=%s and dep_time=%s and company=%s", (i['direction'],DST,str(i['year'])+"-"+str(i['month'])+"-"+str(i['day']), str(i['dep_time']), str(company_id)))
if (len(curs.fetchall()) > 0):
curs.execute("DELETE FROM flights WHERE direction=%s and dst=%s and date=%s and dep_time=%s and company=%s", (i['direction'],DST,str(i['year'])+"-"+str(i['month'])+"-"+str(i['day']), str(i['dep_time']), str(company_id)))
curs.execute("INSERT INTO flights (company, scrape_time, direction, dst, price, dep_time, arr_time, date, dur_time) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)", (str(company_id), str(scrape_time), i['direction'], DST, int(i['price']), str(i['dep_time']), str(i['arr_time']), str(i['year'])+"-"+str(i['month'])+"-"+str(i['day']), get_flight_time(i, DST)))
curs.execute("INSERT INTO archive_flights (company, scrape_time, direction, dst, price, dep_time, arr_time, date, dur_time) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)", (str(company_id), str(scrape_time), i['direction'], DST, int(i['price']), str(i['dep_time']), str(i['arr_time']), str(i['year'])+"-"+str(i['month'])+"-"+str(i['day']), get_flight_time(i, DST)))
if cleandone==1:
curs.execute("delete from flights where company=%s and dst=%s and date>=%s and date<%s and scrape_time<%s", (str(company_id), DST, str(Start_orig.strftime("%Y-%m-%d")), str(Stop.strftime("%Y-%m-%d")), str(scrape_time)))
print "Done!"
db.commit()
print datetime.datetime.now()