company_id = curs.fetchone()[0]
for i in fl2:
curs.execute(
"select * FROM flights WHERE direction=%s and dst=%s and date=%s and dep_time=%s and company=%s",
(i['direction'], DST, str(i['year']) + "-" + str(i['month']) + "-" +
str(i['day']), '00:00', str(company_id)))
if (len(curs.fetchall()) > 0):
curs.execute(
"DELETE FROM flights WHERE direction=%s and dst=%s and date=%s and dep_time=%s and company=%s",
(i['direction'], DST, str(i['year']) + "-" + str(i['month']) +
"-" + str(i['day']), '00:00', str(company_id)))
curs.execute(
"INSERT INTO flights (company, scrape_time, direction, dst, price, dep_time, arr_time, date, dur_time) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)",
(str(company_id), str(scrape_time), i['direction'], DST, int(
i['price']), '00:00', '00:00', str(i['year']) + "-" +
str(i['month']) + "-" + str(i['day']), get_flight_time(i, DST)))
curs.execute(
"INSERT INTO archive_flights (company, scrape_time, direction, dst, price, dep_time, arr_time, date, dur_time) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)",
(str(company_id), str(scrape_time), i['direction'], DST, int(
i['price']), '00:00', '00:00', str(i['year']) + "-" +
str(i['month']) + "-" + str(i['day']), get_flight_time(i, DST)))
if cleandone == 1:
curs.execute(
"delete from flights where company=%s and dst=%s and date>=%s and date<%s and scrape_time<%s",
(str(company_id), DST, str(Start_orig.strftime("%Y-%m-%d")),
str(Stop.strftime("%Y-%m-%d")), str(scrape_time)))
print "Done!"
db.commit()
print ""
if debug_flag:
print "Debug: efore clean_dup: Out, Inc: "
print flightsList
fl2=up.clean_dup_max(flightsList, 1)
fl3=up.clean_dup_max(flightsList, 2)
fl2.extend(fl3)
if debug_flag:
print "Debug: After clean_dup: Out, Inc: "
for i in fl2:
print i
#db= psycopg2.connect( host="manegerdb.cjjasb6ckbh1.us-east-1.rds.amazonaws.com", database="GabiScrape", user="******", password="******")
curs = db.cursor()
curs.execute("select id from companies where name='elal'")
company_id=curs.fetchone()[0]
for i in fl2:
curs.execute("select * FROM flights WHERE direction=%s and dst=%s and date=%s and dep_time=%s and company=%s", (i['direction'],DST,str(i['year'])+"-"+str(i['month'])+"-"+str(i['day']), str(i['dep_time']), str(company_id)))
if (len(curs.fetchall()) > 0):
curs.execute("DELETE FROM flights WHERE direction=%s and dst=%s and date=%s and dep_time=%s and company=%s", (i['direction'],DST,str(i['year'])+"-"+str(i['month'])+"-"+str(i['day']), str(i['dep_time']), str(company_id)))
curs.execute("INSERT INTO flights (company, scrape_time, direction, dst, price, dep_time, arr_time, date, dur_time) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)", (str(company_id), str(scrape_time), i['direction'], DST, int(i['price']), str(i['dep_time']), str(i['arr_time']), str(i['year'])+"-"+str(i['month'])+"-"+str(i['day']), get_flight_time(i, DST)))
curs.execute("INSERT INTO archive_flights (company, scrape_time, direction, dst, price, dep_time, arr_time, date, dur_time) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)", (str(company_id), str(scrape_time), i['direction'], DST, int(i['price']), str(i['dep_time']), str(i['arr_time']), str(i['year'])+"-"+str(i['month'])+"-"+str(i['day']), get_flight_time(i, DST)))
if cleandone==1:
curs.execute("delete from flights where company=%s and dst=%s and date>=%s and date<%s and scrape_time<%s", (str(company_id), DST, str(Start_orig.strftime("%Y-%m-%d")), str(Stop.strftime("%Y-%m-%d")), str(scrape_time)))
print "Done!"
db.commit()
print datetime.datetime.now()
for i in fl2:
curs.execute(
"select * FROM flights WHERE direction=%s and dst=%s and date=%s and dep_time=%s and company=%s",
(i['direction'], DST, str(i['year']) + "-" + str(i['month']) + "-" +
str(i['day']), str(i['dep_time']), str(company_id)))
if (len(curs.fetchall()) > 0):
curs.execute(
"DELETE FROM flights WHERE direction=%s and dst=%s and date=%s and dep_time=%s and company=%s",
(i['direction'], DST, str(i['year']) + "-" + str(i['month']) +
"-" + str(i['day']), str(i['dep_time']), str(company_id)))
curs.execute(
"INSERT INTO flights (company, scrape_time, direction, dst, price, dep_time, arr_time, date, dur_time) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)",
(str(company_id), str(scrape_time), i['direction'], DST, int(
i['price']), str(i['dep_time']), str(i['arr_time']),
str(i['year']) + "-" + str(i['month']) + "-" + str(i['day']),
get_flight_time(i, DST)))
curs.execute(
"INSERT INTO archive_flights (company, scrape_time, direction, dst, price, dep_time, arr_time, date, dur_time) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)",
(str(company_id), str(scrape_time), i['direction'], DST, int(
i['price']), str(i['dep_time']), str(i['arr_time']),
str(i['year']) + "-" + str(i['month']) + "-" + str(i['day']),
get_flight_time(i, DST)))
if cleandone == 1:
curs.execute(
"delete from flights where company=%s and dst=%s and date>=%s and date<%s and scrape_time<%s",
(str(company_id), DST, str(Start_orig.strftime("%Y-%m-%d")),
str(Stop.strftime("%Y-%m-%d")), str(scrape_time)))
print "Done!"
db.commit()
print ""
if debug_flag:
print "Debug: efore clean_dup: Out, Inc: "
print flightsList
fl2=up.clean_dup_max(flightsList, 1)
fl3=up.clean_dup_max(flightsList, 2)
fl2.extend(fl3)
if debug_flag:
print "Debug: After clean_dup: Out, Inc: "
for i in fl2:
print i
#db= psycopg2.connect( host="manegerdb.cjjasb6ckbh1.us-east-1.rds.amazonaws.com", database="GabiScrape", user="******", password="******")
curs = db.cursor()
curs.execute("select id from companies where name='sundor'")
company_id=curs.fetchone()[0]
for i in fl2:
curs.execute("select * FROM flights WHERE direction=%s and dst=%s and date=%s and dep_time=%s and company=%s", (i['direction'],DST,str(i['year'])+"-"+str(i['month'])+"-"+str(i['day']),'00:00',str(company_id)))
if (len(curs.fetchall()) > 0):
curs.execute("DELETE FROM flights WHERE direction=%s and dst=%s and date=%s and dep_time=%s and company=%s", (i['direction'],DST,str(i['year'])+"-"+str(i['month'])+"-"+str(i['day']),'00:00',str(company_id)))
curs.execute("INSERT INTO flights (company, scrape_time, direction, dst, price, dep_time, arr_time, date, dur_time) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)", (str(company_id), str(scrape_time), i['direction'], DST, int(i['price']), '00:00', '00:00', str(i['year'])+"-"+str(i['month'])+"-"+str(i['day']), get_flight_time(i, DST)))
curs.execute("INSERT INTO archive_flights (company, scrape_time, direction, dst, price, dep_time, arr_time, date, dur_time) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)", (str(company_id), str(scrape_time), i['direction'], DST, int(i['price']), '00:00', '00:00', str(i['year'])+"-"+str(i['month'])+"-"+str(i['day']), get_flight_time(i, DST)))
if cleandone==1:
curs.execute("delete from flights where company=%s and dst=%s and date>=%s and date<%s and scrape_time<%s", (str(company_id), DST, str(Start_orig.strftime("%Y-%m-%d")), str(Stop.strftime("%Y-%m-%d")), str(scrape_time)))
print "Done!"
db.commit()
print datetime.datetime.now()
== 0) else datetime.datetime.strftime(
datetime.datetime.strptime(i['dep_time'], "%H:%M") -
datetime.timedelta(minutes=60), "%H:%M")
curs.execute(
"select * FROM flights WHERE direction=%s and dst=%s and date=%s and dep_time>%s and dep_time<%s and company=%s",
(i['direction'], DST, str(i['date']), depm1, depp1, str(company_id)))
if (len(curs.fetchall()) > 0):
curs.execute(
"DELETE FROM flights WHERE direction=%s and dst=%s and date=%s and dep_time>%s and dep_time<%s and company=%s",
(i['direction'], DST, str(
i['date']), depm1, depp1, str(company_id)))
curs.execute(
"INSERT INTO flights (company, scrape_time, direction, dst, price, dep_time, arr_time, date, dur_time) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)",
(str(company_id), str(scrape_time), i['direction'], DST, int(
i['price']), i['dep_time'], i['arr_time'], str(
i['date']), get_flight_time(i, DST)))
curs.execute(
"INSERT INTO archive_flights (company, scrape_time, direction, dst, price, dep_time, arr_time, date, dur_time) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)",
(str(company_id), str(scrape_time), i['direction'], DST, int(
i['price']), i['dep_time'], i['arr_time'], str(
i['date']), get_flight_time(i, DST)))
if cleandone == 1:
curs.execute(
"delete from flights where company=%s and dst=%s and date>=%s and date<%s and scrape_time<%s",
(str(company_id), DST, str(Start_orig.strftime("%Y-%m-%d")),
str(Stop.strftime("%Y-%m-%d")), str(scrape_time)))
print "Done!"
db.commit()
