def __init__(self):
print self.DOMAIN
self.logger = Logger(name='dubizzle_data_log')
self.err_logger = Logger(name='err_dubizzle_data_log')
self.request_manager = RequestManager()
self.source_code_manager = SourceCodeManager()
self.generator = Generator()
self.db = DatabaseManager()
def chart(request):
list_data = DatabaseManager().get_similar_cars_for_graph(make='BMW',
model='3-Series',
trim='320i',
year=2007)
list_make = DatabaseManager().get_all_make()
cars_data = list_data['data']
cars_urls = list_data['urls']
print('Length:', len(cars_data))
return render(request, 'car_chart/scatter_chart.html', {
'values': cars_data,
'urls': cars_urls,
'makes': list_make
})
def __init__(self):
# self.pool = Pool(5)
self.db = DatabaseManager()
# self.rmq_extract = RabbitMQ.RabbitMQManager.RabbitMQManager('uae_extract_urls')
# self.rmq_update = RabbitMQ.RabbitMQManager.RabbitMQManager('uae_update_urls')
# ======== Dubizzle =======
#self.dubizzle_data_extractor = DubizzleDataExtractor()
# self.dubizzle_links_extractor = DubizzleLinksExtractor()
# ======== Dubicars =======
#self.dubicars_data_extractor = DubicarsDataExtractor()
# self.dubicars_links_extractor = DubicarsLinksExtractor()
self.deal_quality = DealQualityManager()
def handle_ImageMessage(event):
image = line_bot_api.get_message_content(event.message.id)
image_url = ImageUtil().upload(image.content)
record.image = image_url
line_bot_api.push_message(event.source.user_id,
TextSendMessage(text="You can send 'compensation' to choose the type of compensation."))
DatabaseManager().save_record(record)
def activate(self, member):
try:
with DatabaseManager() as db:
sql = "UPDATE `users` SET `active`=%s WHERE `user_id`=%s"
db.execute(sql, (1, member.id))
print("Activated account for user %s" % (member.id))
except Exception as e:
print("Error activating user %s.\n%s" % (member.id, e))
def create(self, member, details):
try:
with DatabaseManager() as db:
sql = "INSERT INTO `profiles` (`user_id`, `battlenet_tag`, `name`, `race`, `primary_profession`, `secondary_profession`) VALUES (%s, %s, %s, %s, %s, %s)"
db.execute(sql, (member.id, details['battlenet'], details['name'], details['race'], details['primary_profession'], details['secondary_profession']))
db.commit()
print("Added profile for user %s" % member.id)
except Exception as e:
print("Error creating profile %s.\n%s" % (member.id, e))
def get_models_for_make(request):
print('get_models_for_make WAS RUNNED')
if request.GET:
make = request.GET['make']
print(make)
models = json.dumps(DatabaseManager().get_all_models(make))
print(models)
if len(models) > 0:
return HttpResponse(models, content_type="text/html")
else:
pass
def read(self, member):
try:
with DatabaseManager() as db:
sql = "SELECT * from `users` WHERE `user_id`=%s AND `active`=%s"
result = db.query(sql, (member, 1))
if not result:
print("User does not exist: %s" % member)
else:
return result[0]
except Exception as e:
print("Error fetching user id %s.\n%s" % (member, e))
def fetch(self):
try:
with DatabaseManager() as db:
sql = "SELECT * from `users` where `active`=%s ORDER BY `credits` DESC LIMIT 5"
result = db.query(sql, (1))
if not result:
print("Error fetching leaderboard")
else:
return result
except Exception as e:
print("Error fetching leaderboard.\n%s" % (e))
def read(self, member):
# Fetch user object to retrieve ID
try:
with DatabaseManager() as db:
sql = "SELECT * FROM `profiles` WHERE `user_id`=%s"
result = db.query(sql, member.id)
if not result:
print("User does not exist: %s" % member.id)
return None
else:
return result[0]
except Exception as e:
print("Error fetching profile for user %s.\n%s" % (member.id, e))
def get_trim_for_model(request):
print('get_trim_for_model WAS RUNNED')
if request.GET:
make = request.GET['make']
model = request.GET['model']
print(make)
print(model)
models = json.dumps(DatabaseManager().get_all_trim(make=make,
model=model))
print(models)
if len(models) > 0:
return HttpResponse(models, content_type="text/html")
else:
pass
def create(self, member):
if member.bot == 1 or self.read(member, 1):
return
if self.read(member, 0):
self.activate(member)
return
try:
with DatabaseManager() as db:
sql = "INSERT INTO `users` (`user_id`, `name`, `guild`, `joined_at`, `active`, `received_credits`) VALUES (%s, %s, %s, %s, %s, %s)"
db.execute(sql, (member.id, member.name, member.guild.id,
member.joined_at, 1, member.joined_at))
db.commit()
except Exception as e:
print("Error fetching user %s.\n%s" % (member.id, e))
def read(self, member, active=1):
try:
with DatabaseManager() as db:
if active == 1:
sql = "SELECT * from `users` WHERE `user_id`=%s AND `active`=1"
else:
sql = "SELECT * from `users` WHERE `user_id`=%s AND `active`=0"
result = db.query(sql, (member.id))
if not result:
print("User does not exist or is inactive: %s" % member.id)
return None
else:
return result[0]
except Exception as e:
print("Error fetching user id %s.\n%s" % (member.id, e))
def get_year_for_trim(request):
print('get_year_for_trim WAS RUNNED')
if request.GET:
make = request.GET['make']
model = request.GET['model']
trim = request.GET['trim']
# print(make)
# print(model)
# print(trim)
models = json.dumps(DatabaseManager().get_all_year(make=make,
model=model,
trim=trim))
print(models)
if len(models) > 0:
return HttpResponse(models, content_type="text/html")
else:
pass
def update(self, member, amount, operation="add"):
user = self.read(member)
if operation == "minus":
new_credits = user['credits'] - amount
else:
new_credits = user['credits'] + amount
date = datetime.now()
try:
with DatabaseManager() as db:
sql = "UPDATE `users` SET `credits`=%s, `received_credits`=%s WHERE `user_id`=%s"
db.execute(sql, (new_credits, date, member))
print("Updated %s's credits from %s to %s" %
(user['name'], user['credits'], new_credits))
except Exception as e:
print("Error when updating credits for %s.\n%s" % (member, e))
def get_graph(request):
print('get_graph WAS RUNNED')
if request.GET:
make = request.GET['make']
model = request.GET['model']
trim = request.GET['trim']
year = int(request.GET['year'])
list_data = DatabaseManager().get_similar_cars_for_graph(make=make,
model=model,
trim=trim,
year=year)
# print( 'Length:',len(list_data['data']))
# print(list_data['data'])
# print(list_data['urls'])
if len(list_data) > 0:
return HttpResponse(json.dumps(list_data),
content_type="text/html")
class ProxieManager():
def __init__(self):
self.db = DatabaseManager()
def check_existing_proxie(self):
proxie_list = self.db.get_proxies()
for proxie_data in proxie_list:
proxie = self.__generate_proxie(proxie_data)
def __check_proxie(self, proxie):
pass
@staticmethod
def __generate_proxie(proxie_data):
protocol = proxie_data['protocol']
host = proxie_data['host']
port = proxie_data['port']
proxie = protocol + "://" + host + ":" + port
return proxie
price = price.replace('AED', "").replace(" ", ""). \
replace(",", ""). \
replace(".", ""). \
replace("-", "")
return int(price)
except:
return 0
def __find_tag_by_text(self, code, text):
tag_with_text = code.find(text=text)
needed_tag = tag_with_text.parent.find_next_sibling()
return needed_tag.text
if __name__ == '__main__':
extractor = DataExtractor()
db = DatabaseManager()
# print extractor.extract_data(db.get_all_urls(extractor.DOMAIN))
# list = db.get_all_urls(extractor.DOMAIN)[200:250]
# for url in list:
# extractor.extract_data(url)
extractor.extract_data({
'url':
'https://www.dubicars.com/2006-toyota-land-cruiser-vxr-v8-152617.html',
'listing_id': '152617',
'id': 221717L
})
class DataExtractor:
DOMAIN = 'dubicars.com'
PROJECT_ID = 13
PATH = 'phones/'
def __init__(self):
print self.DOMAIN
self.logger = Logger(name='dubicars_data_log')
self.err_logger = Logger(name='err_dubicars_data_log')
self.request_manager = RequestManager()
self.source_code_manager = SourceCodeManager()
self.generator = Generator()
self.db = DatabaseManager()
self.trim_list = self.db.get_trim_list()
def extract_data(self, url_data):
print url_data
url_id = url_data['id']
url = url_data['url']
listing_id = url_data['listing_id']
data = {}
response = self.request_manager.take_get_request(url)
source_code = response['source_code']
parsed_code = self.source_code_manager.parse_code(source_code)
expired = parsed_code.find('img', {'class': 'sold'})
if expired is not None:
self.db.set_url_inactive(url_id)
self.err_logger.error("EXPIRED " + str(url_data))
return
elif response['status_code'] == 404:
self.db.set_url_inactive(url_id)
self.err_logger.error("404 " + str(url_data))
return
try:
marka = self.__find_make(parsed_code)
year = self.__find_year(parsed_code)
kilometres = self.__find_km(parsed_code)
color = self.__find_color(parsed_code)
specs = self.__find_specs(parsed_code)
price = self.__find_price(parsed_code)
model = self.__find_model(parsed_code, make=marka)
trim = self.__find_trim(parsed_code, marka=marka, model=model)
if trim == 'Other':
self.db.set_url_processed(url_id)
self.db.set_url_inactive(url_id)
return
phone = self.__find_phone(parsed_code)
except Exception as exc:
self.err_logger.error(str(exc) + str(url_data))
self.db.set_url_processed(url_id)
return
try:
data['year'] = int(year)
data['price'] = int(price)
data['kilometres'] = int(kilometres)
data['color'] = color
data['specs'] = specs
data['trim'] = trim
data['model'] = model
data['make'] = marka
data['phone'] = phone
print data
except Exception as exc:
self.err_logger.error(str(exc) + url_data)
self.db.set_url_processed(url_id)
self.db.set_url_inactive(url_id)
return
self.db.insert_data(data=data,
listing_id=listing_id,
url=url,
source=self.DOMAIN)
self.db.set_url_processed(url_id)
def update_data(self, url_data):
timestamp = generate_timestamp()
url_id = url_data['id']
listing_id = url_data['listing_id']
print listing_id
url = url_data['url']
first_timestamp = url_data['timestamp']
time_dif = first_timestamp - datetime.strptime(timestamp,
"%Y.%m.%d:%H:%M:%S")
time_dif = time_dif.days
response = self.request_manager.take_get_request(url)
source_code = response['source_code']
parsed_code = self.source_code_manager.parse_code(source_code)
expired = parsed_code.find('img', {'class': 'sold'})
if expired is not None:
self.db.set_sold_status(listing_id=listing_id,
days_for_selling=time_dif)
#self.db.remove_listing(listing_id)
self.db.set_url_inactive(url_id)
return
elif response['status_code'] == 404:
print 404, listing_id
self.db.set_sold_status(listing_id=listing_id,
days_for_selling=time_dif)
#self.db.remove_listing(listing_id)
self.db.set_url_inactive(url_id)
return
try:
price = self.__find_price(parsed_code)
except:
price = 0
# days = self.__calc_days_on_market(listing_id)
self.db.update_listing(listing_id=listing_id,
price=int(price),
days_on_market=time_dif)
self.db.set_updated(listing_id=listing_id)
def __find_make(self, code):
try:
make = self.__find_tag_by_text(code, text='Make:')
return make
except:
return ''
def __find_year(self, code):
try:
year = self.__find_tag_by_text(code, text='Year:')
year_list = year.split()
for year in year_list:
try:
year = int(year)
return year
except:
continue
except:
return ''
def __find_km(self, code):
try:
km = self.__find_tag_by_text(code, text='Kilometers:')
km = km.replace(",", "").replace(".", "").replace(" ", "")
return int(km)
except:
return 0
def __find_color(self, code):
try:
color = self.__find_tag_by_text(code, text='Color:')
return color.strip()
except:
return ''
def __find_specs(self, code):
try:
specs = self.__find_tag_by_text(code, text='Specs:')
return specs.strip()
except:
return ''
# ============= TRIM ===============
# =====
def __generateEditedTrims(self, marka, trim):
for example_trim in self.trim_list:
try:
if len(example_trim['trim']) <= 3:
continue
except:
continue
if '-' in example_trim['trim']:
if example_trim['make'] == marka:
edited_example_trim = example_trim['trim'].replace(
'-', ' ')
if edited_example_trim in trim:
print example_trim['trim']
return example_trim['trim']
edited_example_trim = example_trim['trim'].replace(
'-', ' ').title()
if edited_example_trim in trim:
print example_trim['trim']
return example_trim['trim']
return ''
def __find_trim(self, code, marka, model):
try:
to_return_trim = ''
not_edited_trim = self.__find_tag_by_text(code,
text='Model:').strip()
trim = not_edited_trim.replace(model, '').strip()
if len(trim.split()) == 0:
print not_edited_trim, 'there is no Trim!!!!'
return not_edited_trim.strip()
for example_trim in self.trim_list:
if example_trim['make'] == marka:
if example_trim['trim'] in trim:
if len(example_trim['trim']) <= 2:
if ' ' + example_trim[
'trim'] + ' ' in ' ' + trim + ' ':
if len(example_trim['trim']) > len(
to_return_trim):
print example_trim['trim']
to_return_trim = example_trim['trim']
continue
if len(example_trim['trim']) > len(to_return_trim):
print example_trim['trim']
to_return_trim = example_trim['trim']
edited_trim = self.__generateEditedTrims(marka=marka, trim=trim)
if len(edited_trim) > len(to_return_trim):
return edited_trim
elif to_return_trim == '':
if len(trim.split()) <= 2 and len(trim.split()) > 0:
return trim
else:
return to_return_trim
except:
return ''
# =====
# ============= TRIM ===============
def __find_model(self, code, make):
try:
breadcrumbs = code.findAll('span', {'typeof': 'v:Breadcrumb'})
name = breadcrumbs[-1].text
len_make = len(make.split())
trim = name.split()[len_make:]
trim = ' '.join(trim)
return trim.strip()
except Exception as exc:
print exc
return ''
def __find_phone(self, code):
try:
phone = code.find('p', {
'id': 'contact-buttons'
}).find('a')['data-reveal']
phone = phone.replace('"',
"").replace(" ",
"").replace("[",
"").replace("]", "")
return phone.strip()
except Exception as exc:
print exc
return ''
def __find_price(self, code):
try:
price = code.find('strong', {'class': 'money'}).text
price = price.replace('AED', "").replace(" ", "").\
replace(",", "").\
replace(".", "").\
replace("-", "")
return int(price)
except:
try:
price = code.find('strong', {'class': 'money reduced'}).text
price = price.replace('AED', "").replace(" ", ""). \
replace(",", ""). \
replace(".", ""). \
replace("-", "")
return int(price)
except:
return 0
def __find_tag_by_text(self, code, text):
tag_with_text = code.find(text=text)
needed_tag = tag_with_text.parent.find_next_sibling()
return needed_tag.text
class DealQualityManager:
def __init__(self):
self.db = DatabaseManager()
def group_cars(self, object_list):
grouped_list = [[]]
temp_object = object_list[0]
index = 0
for object in object_list:
if object[1:5] == temp_object[1:5]:
grouped_list[index].append(object)
else:
index += 1
temp_object = object
grouped_list.append([])
grouped_list[index].append(object)
return grouped_list
# def dealRating(self, list_cars):
# result_km = 0
# result_price = 0
# summ_km = 0
# summ_price = 0
#
# for car in list_cars:
# summ_km += car[5]
# summ_price += car[6]
#
# average_km = (summ_km / len(list_cars))
# average_price = (summ_price / len(list_cars))
# print('Average km: ', average_km)
# print('Average price: ', average_price)
#
# average_km_degree = average_km / 17
# average_price_degree = average_price / 20
#
# for car in list_cars:
# id = car[0]
# different_km = average_km - car[5]
# different_price = average_price - car[6]
# try:
# result_km = int(different_km / average_km_degree)
# except ZeroDivisionError:
# result_km = 0
# try:
# result_price = int(different_price / average_price_degree)
# except ZeroDivisionError:
# result_price = 0
# status = str(result_km + result_price)
# self.db.set_deal_quality(status=status, listing_id=id, price_difference=different_price)
def dealRating(self, list_cars):
result_km = 0
result_price = 0
summ_km = 0
summ_price = 0
for car in list_cars:
summ_km += car['kilometres']
summ_price += car['price']
average_km = (summ_km / len(list_cars))
average_price = (summ_price / len(list_cars))
print('Average km: ', average_km)
print('Average price: ', average_price)
average_km_degree = average_km / 17
average_price_degree = average_price / 20
for car in list_cars:
id = car['id']
different_km = average_km - car['kilometres']
different_price = average_price - car['price']
try:
result_km = int(different_km / average_km_degree)
except ZeroDivisionError:
result_km = 0
try:
result_price = int(different_price / average_price_degree)
except ZeroDivisionError:
result_price = 0
status = str(result_km + result_price)
self.db.set_deal_quality(status=status,
listing_id=id,
price_difference=different_price)
def main(self, car_list):
car_list = self.group_cars(car_list)
for grouped_cars in car_list:
self.dealRating(list(grouped_cars))
def __init__(self):
self.db = DatabaseManager()
class Runner():
TEMPLATE_DUBIZZLE = 'https://uae.dubizzle.com/motors/used-cars/?page={}&seller_type=OW&is_search=1&is_basic_search_widget=0&places__id__in=--&ot=desc&o=2'
TEMPLATE_DUBICARS = 'https://www.dubicars.com/search?ajax=true&view=&o=&l=&ma=&mo=0&c=new-and-used&pf=&pt=&yf=&yt=&kf=10000&kt=&b=&co=&ci=&s=&gi=&page={}'
DUBIZZLE_DOMAIN = 'dubai.dubizzle.com'
DUBICARS_DOMAIN = 'dubicars.com'
def __init__(self):
# self.pool = Pool(5)
self.db = DatabaseManager()
# self.rmq_extract = RabbitMQ.RabbitMQManager.RabbitMQManager('uae_extract_urls')
# self.rmq_update = RabbitMQ.RabbitMQManager.RabbitMQManager('uae_update_urls')
# ======== Dubizzle =======
#self.dubizzle_data_extractor = DubizzleDataExtractor()
# self.dubizzle_links_extractor = DubizzleLinksExtractor()
# ======== Dubicars =======
#self.dubicars_data_extractor = DubicarsDataExtractor()
# self.dubicars_links_extractor = DubicarsLinksExtractor()
self.deal_quality = DealQualityManager()
def main(self, update_mode=False, deal_update=False, extract_data=False):
# if update_mode:
# self.update_db()
# self.deal_quality.main(self.db.get_grouped_listings())
# print "Updated!"
# self.db.reset_updates()
# return
# if deal_update:
# self.deal_quality.main(self.db.get_grouped_listings())
# return
# if extract_data: # Not work
# new_urls = self.db.get_urls(source=None)
# print len(new_urls)
# self.pool.map(self.dubizzle_data_extractor.extract_data, new_urls)
# return
self.deal_quality.main(self.db.get_grouped_listings())
def extract_data_dubizzle(self):
# ======== Dubizzle =======
# self.dubizzle_links_extractor.extract_urls()
# print self.DUBIZZLE_DOMAIN
new_urls = self.db.get_urls(source=self.DUBIZZLE_DOMAIN) # Dubizzle
print len(new_urls)
self.rmq_extract.load_urls(
self.generate_urls_to_queue(activity='extract',
source='dubizzle',
urls=new_urls))
def extract_data_dubicars(self):
# ======== Dubicars =======
self.dubicars_links_extractor.main(self.TEMPLATE_DUBICARS)
new_urls = self.db.get_urls(source=self.DUBICARS_DOMAIN) # Dubcars
print len(new_urls)
self.rmq_extract.load_urls(
self.generate_urls_to_queue(activity='extract',
source='dubicars',
urls=new_urls))
def update_db(self):
# ======== Dubizzle =======
dubizzle_urls_data = self.db.get_all_urls(source=self.DUBIZZLE_DOMAIN)
urls_to_queue = self.generate_urls_to_queue(source='dubizzle',
activity='update',
urls=dubizzle_urls_data)
self.rmq_update.load_urls(urls=urls_to_queue)
# ======== Dubicars =======
dubicars_urls_data = self.db.get_all_urls(source=self.DUBICARS_DOMAIN)
urls_to_queue = self.generate_urls_to_queue(source='dubicars',
activity='update',
urls=dubicars_urls_data)
self.rmq_update.load_urls(urls=urls_to_queue)
def generate_urls_to_queue(self, source, activity, urls):
for url in urls:
url['activity'] = activity
url['source'] = source
return urls
def deal_quality_func(self):
self.deal_quality.main(self.db.get_grouped_listings())
for car in list_cars:
id = car['id']
different_km = average_km - car['kilometres']
different_price = average_price - car['price']
try:
result_km = int(different_km / average_km_degree)
except ZeroDivisionError:
result_km = 0
try:
result_price = int(different_price / average_price_degree)
except ZeroDivisionError:
result_price = 0
status = str(result_km + result_price)
self.db.set_deal_quality(status=status,
listing_id=id,
price_difference=different_price)
def main(self, car_list):
car_list = self.group_cars(car_list)
for grouped_cars in car_list:
self.dealRating(list(grouped_cars))
if __name__ == '__main__':
db = DatabaseManager()
DQM = DealQualityManager()
object_list = db.get_grouped_listings()
DQM.main(object_list)
def __init__(self):
self.dbmanager = DatabaseManager('api', 'password')
class databaseinterface:
def __init__(self):
self.dbmanager = DatabaseManager('api', 'password')
def getChannels(self):
"""
:return: List of all channels
"""
return self.dbmanager.lookupChannels()
def getUsers(self):
"""
:return: List of all users
"""
return self.dbmanager.lookupUsers()
def newUser(self, user):
"""
Adds a user to the database
:param user: User object
:return: None
"""
self.dbmanager.createUser((user.id, user.alias, user.password))
def changeUser(self, userid, user):
"""
Updates the user in the database at user ID to be user
Basically removes then re adds
:param userid: Id of user being updated
:param user: User object
:return: None
"""
self.dbmanager.updateUser(userid, user.name, user.password)
def getUser(self, userid):
"""
Returns the information to create a user object from what was stored in the database
:param userid: Id of user being looked up
:return: User object
"""
(id, alias, password) = self.dbmanager.lookupUser(userid)
return (alias, id, None, None)
def getUserAlias(self, alias):
"""
returns user with alias==alias, none if not found
:param alias: Alias of user being looked up
:return: User object
"""
x = self.dbmanager.lookupUser(alias, "alias")
if not x:
return None
(id, alias, password) = x
return (alias, id, None, None)
def newChannel(self, channel):
"""
Adds a channel object to the database
:param channel: Channel object
:return:
"""
self.dbmanager.createChannel(
(channel.id, channel.name, channel.permisions))
def getChannel(self, channel):
"""
Retrieve information from the database and create a channel object
:param channel:
:return:
"""
(id, name, permissions,
blocked_users) = self.dbmanager.lookupChannel(channel)
return (name, permissions, id, blocked_users)
def change_default_permisions(self, channel, permissions):
regex = re.match(r'([01]{3})', permissions)
if regex:
self.dbmanager.updateChannelPermissions(channel.id, permissions)
# change permision in channel with given id
def blockUser(self, channel, user):
"""
Blocks a given user from a given channel
:param channel: Channel object
:param user: User object
:return: None
"""
self.dbmanager.appendBlockedChannel(channel.id, user.id)
def UnblockUser(self, channel, user):
"""
Unblocks a given user from a given channel
:param channel: Channel object
:param user: User object
:return: None
"""
self.dbmanager.removeBlockedUser(channel.id, user.id)
class DataExtractor:
DOMAIN = 'dubai.dubizzle.com'
PROJECT_ID = 13
PATH = 'phones/'
def __init__(self):
print self.DOMAIN
self.logger = Logger(name='dubizzle_data_log')
self.err_logger = Logger(name='err_dubizzle_data_log')
self.request_manager = RequestManager()
self.source_code_manager = SourceCodeManager()
self.generator = Generator()
self.db = DatabaseManager()
def extract_data(self, url_data):
print url_data
url_id = url_data['id']
url = url_data['url']
listing_id = url_data['listing_id']
data = {}
response = self.request_manager.take_get_request(url)
source_code = response['source_code']
parsed_code = self.source_code_manager.parse_code(source_code)
expired = parsed_code.find('div', {'id': 'expired-ad-message'})
if expired is not None:
#self.db.remove_listing(listing_id)
self.db.set_url_inactive(url_id)
self.err_logger.error("EXPIRED " + str(url_data))
return
elif response['status_code'] == 404:
#self.db.remove_listing(listing_id)
self.err_logger.error("404 " + str(url_data))
self.db.set_url_inactive(url_id)
return
bread = parsed_code.find('span', {'id': 'browse_in_breadcrumb'})
items = bread.findAll('div')
try:
year = parsed_code.find('img', attrs={
'alt': 'Year'
}).parent.text.replace('Year', '').strip()
kilometres = parsed_code.find('img', attrs={
'alt': 'Kilometers'
}).parent.text.replace('Kilometers',
'').strip().replace(',',
'').replace('.', '')
color = parsed_code.find('img', attrs={
'alt': 'Color'
}).parent.text.replace('Color', '').strip()
specs = parsed_code.find('img', attrs={
'alt': 'Specs'
}).parent.text.replace('Specs', '').strip()
trim = parsed_code.find('img', attrs={
'alt': 'Trim'
}).parent.parent.text.replace('Trim', '').strip()
if trim == 'Other':
self.db.set_url_processed(url_id)
return
price = parsed_code.find('span', {
'id': 'actualprice'
}).text.replace(',', '').replace('.', '')
model = items[-1].find('a').text.strip()
marka = items[-2].find('a').text.strip()
phone = self.extract_phone(parsed_code, id=url_id)
except Exception as exc:
self.err_logger.error(str(exc) + str(url_data))
self.db.set_url_processed(url_id)
return
data['year'] = int(year)
data['price'] = int(price)
data['kilometres'] = int(kilometres)
data['color'] = color
data['specs'] = specs
data['trim'] = trim
data['model'] = model
data['make'] = marka
data['phone'] = phone
self.db.insert_data(data=data,
listing_id=listing_id,
url=url,
source=self.DOMAIN)
self.db.set_url_processed(url_id)
def update_data(self, url_data):
timestamp = generate_timestamp()
url_id = url_data['id']
listing_id = url_data['listing_id']
print listing_id
url = url_data['url']
first_timestamp = url_data['timestamp']
time_dif = first_timestamp - datetime.strptime(timestamp,
"%Y.%m.%d:%H:%M:%S")
time_dif = time_dif.days
response = self.request_manager.take_get_request(url)
source_code = response['source_code']
parsed_code = self.source_code_manager.parse_code(source_code)
expired = parsed_code.find('div', {'id': 'expired-ad-message'})
if expired is not None:
self.db.set_sold_status(listing_id=listing_id,
days_for_selling=time_dif)
#self.db.remove_listing(listing_id)
self.db.set_url_inactive(url_id)
print "updated"
return
elif response['status_code'] == 404:
print 404, listing_id
self.db.set_sold_status(listing_id=listing_id,
days_for_selling=time_dif)
#self.db.remove_listing(listing_id)
self.db.set_url_inactive(url_id)
print "updated"
return
try:
price = parsed_code.find('span', {
'id': 'actualprice'
}).text.replace(',', '').replace('.', '')
except:
price = 0
# days = self.__calc_days_on_market(listing_id)
self.db.update_listing(listing_id=listing_id,
price=int(price),
days_on_market=time_dif)
self.db.set_updated(listing_id=listing_id)
print "updated"
# def __calc_days_on_market(self, listing_id):
# days_on_market = self.db.get_car_data(listing_id).days_on_market
# if days_on_market is None:
# return 0
# days_on_market += 1
# return days_on_market
def extract_phone(self, code, id):
img = code.find('img', {'class': 'phone-num-img'})['src']
ext = img.partition('data:image/')[2].split(';')[0]
with open(self.PATH + str(id) + '.' + ext, 'wb') as f:
f.write(ba.a2b_base64(img.partition('base64,')[2]))
text = textract.process(self.PATH + str(id) + '.' + ext).replace(
' ', '')
if '+971' in text:
pass
else:
text = '+971' + text
os.remove(self.PATH + str(id) + '.' + ext)
return text.strip()
def __init__(self):
self.request_manager = RequestManager()
self.source_code_manager = SourceCodeManager()
self.generator = Generator()
self.db = DatabaseManager()
class DataExtractor:
DOMAIN = 'carswitch.com'
PROJECT_ID = 15
def __init__(self):
self.request_manager = RequestManager()
self.source_code_manager = SourceCodeManager()
self.generator = Generator()
self.db = DatabaseManager()
def update_data(self, db_list_listings):
list_cars = self.take_js_request()
if len(list_cars) == 0:
print 'Error Carswitch, len of list cars is 0'
return
list_expired_listing_id = []
list_active_cars = []
for listing in db_list_listings:
checker = False
for car in list_cars:
active_car = car['inspectionID']
if str(listing) == str(active_car):
list_active_cars.append({
'listing_id':active_car,
'price': car['salePrice']
})
checker = True
break
if checker is False:
list_expired_listing_id.append(listing)
if len(list_expired_listing_id) > 0:
for expired_id in list_expired_listing_id:
self.db.set_sold_status(listing_id=expired_id, days_for_selling=0)
print expired_id, 'Not Active'
if len(list_active_cars) > 0:
for active_car in list_active_cars:
timestamp = self.get_info_about_car(active_car['listing_id']).timestamp
days_on_market = self.calculate_days_on_market(first_timestamp=timestamp)
self.db.update_listing(listing_id=active_car['listing_id'], price=active_car['price'], days_on_market=days_on_market)
print active_car, 'Active'
def take_js_request(self):
data = '{"requests": [{"indexName": "All_Carswitch_Cars","params": "query=&' \
'numericFilters=%2CinspectionStatus!%3D9%2Cpromoted!%3D1%2C(new!%3D1)&facetFilters=&page=&hitsPerPage=1200"}]}'
url = 'http://ih3kc909gb-dsn.algolia.net/1/indexes/*/queries?x-algolia-agent=Algolia%20for%20AngularJS%203.15.1&x-algolia-application-id=IH3KC909GB&x-algolia-api-key=0a4fcd3b57535f88c86172d5646d6787'
try:
response = urllib2.urlopen(
url,
data=data)
data = json.loads(response.read())
return data['results'][0]['hits']
except Exception as e:
print('Error: ' + str(e))
return []
def indicate_specs(self, specs_index):
if specs_index == 0:
return 'American'
elif specs_index == 1:
return 'GCC'
elif specs_index == 2:
return 'European'
elif specs_index == 3:
return 'Japanese'
elif specs_index == 4:
return 'Canadian'
def extract_data(self, db_list_listings):
list_cars = self.take_js_request()
if len(list_cars) == 0:
print 'Error Carswitch, len of list cars is 0'
return
print len(list_cars)
for car in list_cars:
data = {}
listing_id = car['inspectionID']
inspectionID = car['carID']
if str(listing_id) in db_list_listings:
print listing_id, 'Exist!'
continue
data['make'] = car['make']
data['model'] = car['model']
data['trim'] = car['displayTrim']
data['year'] = car['year']
data['kilometres'] = car['mileage']
data['color'] = car['_highlightResult']['colorPaint']['value']
data['specs'] = self.indicate_specs(car['GCCspecs'])
data['price'] = car['salePrice']
url = 'http://carswitch.com/uae/used-car/{0}/{1}/{2}/{3}-{4}'\
.format(data['make'],data['model'],data['year'],listing_id,inspectionID)
data['phone'] = ''
print data
self.db.insert_data(data=data, listing_id=listing_id, url=url, source=self.DOMAIN)
def get_info_about_car(self, listing_id):
car_obj = self.db.get_car_data(listing_id=listing_id)
return car_obj
def calculate_days_on_market(self, first_timestamp):
timestamp = generate_timestamp()
time_dif = first_timestamp - datetime.strptime(timestamp,
"%Y.%m.%d:%H:%M:%S")
time_dif = time_dif.days
return time_dif
def main(self):
list_listings = self.db.get_all_cars_listings(self.DOMAIN)
self.extract_data(list_listings)
self.update_data(list_listings)
class DealQualityManager:
def __init__(self):
self.db = DatabaseManager()
def group_cars(self, object_list):
grouped_list = [[]]
temp_object = object_list[0]
index = 0
for object in object_list:
if object[1:5] == temp_object[1:5]:
grouped_list[index].append(object)
else:
index += 1
temp_object = object
grouped_list.append([])
grouped_list[index].append(object)
return grouped_list
def calculate_middle_price(self, car_list):
summ = 0
errors = 0
for car in car_list:
try:
price = int(car[5])
summ += price
except:
errors += 1
try:
middle_price = summ/(len(car_list)-errors)
except ZeroDivisionError:
middle_price = 0
return middle_price
def dealRating(self, list_cars):
result_km = 0
result_price = 0
summ_km = 0
summ_price = 0
for car in list_cars:
summ_km += car[5]
summ_price += car[6]
average_km = (summ_km / len(list_cars))
average_price = (summ_price / len(list_cars))
print 'Average km: ', average_km
print 'Average price: ', average_price
average_km_degree = average_km / 20
average_price_degree = average_price / 20
for car in list_cars:
id = car[0]
different_km = average_km - car[5]
different_price = average_price - car[6]
try:
result_km = int(different_km / average_km_degree)
except ZeroDivisionError:
result_km = 0
try:
result_price = int(different_price / average_price_degree)
except ZeroDivisionError:
result_price = 0
status = str(result_km + result_price)
self.db.set_deal_quality(status=status, listing_id=id, price_difference=different_price)
def calculate_deal_quality(self, car_list):
middle_price = self.calculate_middle_price(car_list)
for car in car_list:
difference = 0
id = car[0]
price = int(car[5])
difference = middle_price - price
print car, middle_price, difference
if difference > 3000:
self.db.set_deal_quality(status="Excellent", listing_id=id, price_difference=difference)
elif difference > 1500:
self.db.set_deal_quality(status="Good", listing_id=id, price_difference=difference)
elif difference < -1500:
self.db.set_deal_quality(status="Poor", listing_id=id, price_difference=difference)
else:
self.db.set_deal_quality(status="Fair", listing_id=id, price_difference=difference)
def main(self, car_list):
car_list = self.group_cars(car_list)
for grouped_cars in car_list:
self.dealRating(list(grouped_cars))
from sqlalchemy import Table
from sqlalchemy.orm import sessionmaker
from database.DatabaseManager import DatabaseManager
databaseManagerII = DatabaseManager('database/inverted-index.db',
'database/inverted-index.sql')
BaseII = databaseManagerII.get_base()
metadataII = databaseManagerII.get_metadata()
engineII = databaseManagerII.get_engine()
databaseManagerD = DatabaseManager('database/documents.db',
'database/documents.sql')
BaseD = databaseManagerD.get_base()
metadataD = databaseManagerD.get_metadata()
engineD = databaseManagerD.get_engine()
# Reflect each database table using metadata
class IndexWord(BaseII):
__table__ = Table('IndexWord', metadataII, autoload=True)
class Posting(BaseII):
__table__ = Table('Posting', metadataII, autoload=True)
class Document(BaseD):
__table__ = Table('Document', metadataD, autoload=True)
# Create a sessionmaker function for each Database to use the tables
