def calculate_days_on_market(self, first_timestamp):
timestamp = generate_timestamp()
time_dif = first_timestamp - datetime.strptime(timestamp,
"%Y.%m.%d:%H:%M:%S")
time_dif = time_dif.days
return time_dif
def set_sold_status(self, listing_id, days_for_selling):
timestamp = generate_timestamp()
with self.open_session() as session:
smtp = update(Car).where(Car.listing_id == listing_id). \
values(status="Removed",
removed_timestamp=timestamp,
days_posted=abs(days_for_selling))
session.execute(smtp)
session.commit()
def update_listing(self, listing_id, price, days_on_market):
with self.open_session() as session:
try:
timestamp = generate_timestamp()
smtp = update(Car).where(Car.listing_id == listing_id). \
values(current_price=price,
update_timestamp=timestamp,
updated=True,
days_on_market=days_on_market)
session.execute(smtp)
session.commit()
except:
session.rollback()
def update_data(self, url_data):
timestamp = generate_timestamp()
url_id = url_data['id']
listing_id = url_data['listing_id']
print listing_id
url = url_data['url']
first_timestamp = url_data['timestamp']
time_dif = first_timestamp - datetime.strptime(timestamp,
"%Y.%m.%d:%H:%M:%S")
time_dif = time_dif.days
response = self.request_manager.take_get_request(url)
source_code = response['source_code']
parsed_code = self.source_code_manager.parse_code(source_code)
expired = parsed_code.find('div', {'id': 'expired-ad-message'})
if expired is not None:
self.db.set_sold_status(listing_id=listing_id,
days_for_selling=time_dif)
#self.db.remove_listing(listing_id)
self.db.set_url_inactive(url_id)
print "updated"
return
elif response['status_code'] == 404:
print 404, listing_id
self.db.set_sold_status(listing_id=listing_id,
days_for_selling=time_dif)
#self.db.remove_listing(listing_id)
self.db.set_url_inactive(url_id)
print "updated"
return
try:
price = parsed_code.find('span', {
'id': 'actualprice'
}).text.replace(',', '').replace('.', '')
except:
price = 0
# days = self.__calc_days_on_market(listing_id)
self.db.update_listing(listing_id=listing_id,
price=int(price),
days_on_market=time_dif)
self.db.set_updated(listing_id=listing_id)
print "updated"
def update_days_on_market(self):
timestamp = generate_timestamp()
list_of_listings = self.get_all_urls()
with self.open_session() as session:
for listing in list_of_listings:
listing_id = listing['listing_id']
print listing_id
first_timestamp = listing['timestamp']
time_dif = first_timestamp - datetime.strptime(timestamp,
"%Y.%m.%d:%H:%M:%S")
time_dif = time_dif.days
smtp = update(Car).where(Car.listing_id==listing_id).values(days_on_market=abs(time_dif))
session.execute(smtp)
session.commit()
def insert_data(self, data, listing_id, url, source):
with self.open_session() as session:
make = data['make']
model = data['model']
trim = data['trim']
year = data['year']
kilometres = data['kilometres']
color = data['color']
specs = data['specs']
first_price = data['price']
phone = data['phone']
listing_id = listing_id
timestamp = generate_timestamp()
new_row = Car(make=make,
model=model,
trim=trim,
year=year,
km=kilometres,
color=color,
specs=specs,
first_price=first_price,
pc=first_price,
phone=phone,
listing_id=listing_id,
url=url,
timestamp=timestamp,
updated=False,
status="Active",
country=source,
days_on_market=1)
try:
session.add(new_row)
session.commit()
except Exception as exc:
print exc
session.rollback()
return False
def insert_urls(self, urls_list, source):
""":param urls_list - {'url': '', 'hash': '', 'proxie': ''}"""
amount_of_existing = 0
with self.open_session() as session:
for url in urls_list:
url_ = url['url']
listing_id = url['listing_id']
timestamp = generate_timestamp()
new_url = Url(url=url_,
listing_id=listing_id,
processed=False,
active=True,
timestamp=timestamp,
source=source,
updated=0)
try:
session.add(new_url)
session.flush()
session.commit()
except Exception as exc:
print exc
amount_of_existing += 1
session.rollback()
return {'amount_of_existing': amount_of_existing}