def main_link(self):
year_from = Settings.get_solo().date_from or '1990'
if isinstance(year_from, date):
year_from = year_from.year
year_to = Settings.get_solo().date_to or '2017'
if isinstance(year_to, date):
year_to = year_to.year
return f'{self.link}/search' \
f'?api_key={self.api_key}' \
'&category_id=1' \
f'&s_yers={year_from}' \
f'&po_yers={year_to}' \
'&price_ot=100' \
'&price_do=10000' \
'¤cy=1' \
'&abroad=2' \
'&custom=1' \
'&damage=2' \
'&fuelRatesType=city' \
'&marka_id=0' \
'&model_id[0]=0' \
'&engineVolumeFrom=' \
'&engineVolumeTo=' \
'&power_name=1' \
'&countpage=100'
def filter_parse(title, description, price, currency, city=''):
for stop in StopWordList.objects.values_list('word', flat=True):
if stop in title or stop in description or stop in city:
return False
if price and isinstance(price, str):
price = int(price)
if isinstance(price, int):
if currency == '$':
price_usd_from = Settings.get_solo().price_usd_from
price_usd_to = Settings.get_solo().price_usd_to
if price_usd_from and price < price_usd_from:
return False
if price_usd_to and price > price_usd_to:
return False
else:
price_hrn_from = Settings.get_solo().price_hrn_from
price_hrn_to = Settings.get_solo().price_hrn_to
if price_hrn_from and price < price_hrn_from:
return False
if price_hrn_to and price > price_hrn_to:
return False
return True
def main_link(self):
year_from = Settings.get_solo().date_from or '1990'
if isinstance(year_from, datetime.date):
year_from = year_from.year
return f'https://www.olx.ua/transport/legkovye-avtomobili/' \
'q-%D0%B4%D1%82%D0%BF/?' \
'search[filter_float_price%3Afrom]=2500&' \
'search[filter_float_price%3Ato]=525000&' \
f'search[filter_float_motor_year%3Afrom]={year_from}'
def __init__(self, link=None, stream=False):
if not link:
link = self.main_link
year_from = Settings.get_solo().date_from or '1990'
if isinstance(year_from, datetime.date):
year_from = year_from.year
year_to = Settings.get_solo().date_to or ''
if isinstance(year_to, datetime.date):
year_to = year_to.year
self.parameters['y1'] = year_from
self.parameters['y2'] = year_to
self.response = requests.get(link,
params=self.parameters,
headers=self.headers,
stream=stream)
self.parsed_body = html.fromstring(self.response.text)
def send(self, phone):
enable_disable_sms = Settings.get_solo().enable_disable_sms
if enable_disable_sms:
if self.auth == 'Вы успешно авторизировались':
if float(self.balance):
return self._result(phone)
else:
self.error_message = f'Ваш баланс {self.balance}'
elif self.auth == 'Неверный логин или пароль':
self.error_message = 'Неверный логин или пароль'
else:
self.never_send = True
def __init__(self, page=1, link=None):
self.parameter['start'] = page
if not link:
self.link = self.main_link
else:
self.link = link
self.parameter = {
'year[]': [1991, 2018],
}
year_from = Settings.get_solo().date_from or '1990'
if isinstance(year_from, datetime.date):
year_from = year_from.year
self.parameter['year[]'][0] = year_from
year_to = Settings.get_solo().date_to or '2017'
if isinstance(year_to, datetime.date):
year_to = year_to.year
self.parameter['year[]'][1] = year_to
self.response = requests.get(self.link, params=self.parameter, headers=self.headers)
# print(self.response.content)
self.parsed_body = html.fromstring(self.response.text)
class SmsSender:
# text = 'Готовы выкупить ваше авто, 90% рыночной стоимости, ' \
# 'деньги в день обращения. 0672323060, 0932714661'
text = Settings.get_solo().message_text
def __init__(self):
self.error_message = ''
self.never_send = False
self.msg_status = ''
self.msg_id = ''
self.client = Client('http://turbosms.in.ua/api/wsdl.html')
self.auth = self.client.service.Auth(login='******',
password='******')
self.balance = self.get_balance()
def send(self, phone):
enable_disable_sms = Settings.get_solo().enable_disable_sms
if enable_disable_sms:
if self.auth == 'Вы успешно авторизировались':
if float(self.balance):
return self._result(phone)
else:
self.error_message = f'Ваш баланс {self.balance}'
elif self.auth == 'Неверный логин или пароль':
self.error_message = 'Неверный логин или пароль'
else:
self.never_send = True
def _result(self, phone):
result = self.client.service.SendSMS(sender='Top Vykup',
destination=phone,
text=self.text)
if result[0] == 'Сообщения успешно отправлены':
self.msg_status = self.get_msg_status(result[1])
self.msg_id = result[1]
return True
else:
self.error_message = result[0]
def get_msg_status(self, sms_id):
self.msg_status = self.client.service.GetMessageStatus(
MessageId=sms_id)
return self.msg_status
def get_balance(self):
return self.client.service.GetCreditBalance()
def start(self):
print('Start Parse RST')
sms = SmsSender()
sms_enable = Settings.get_solo().enable_disable_sms
email_enable = Settings.get_solo().enable_disable_email
for i in range(1, 8):
list_link_articles = Requester(i).get_link('//a[@class="rst-ocb-i-a"]/@href')
for article_link in list_link_articles:
print('Parse', article_link)
page_article = Requester(link=article_link)
id_article = self._get_id_article(page_article)
if Collections.objects.filter(
id_donor__exact=id_article).exists():
continue
date_article = self._get_date_article(page_article)
if not date_article:
continue
title = self._get_title(page_article)
price, currency = self._get_price(page_article)
name, phones = self._get_phone(page_article)
dict_phones = {key: value for key, value in enumerate(phones)}
description = self._get_description(page_article)
city = self._get_location(page_article)
if 'отсутствуют' in name:
continue
if not filter_parse(title, description, price, currency, city):
continue
collection = Collections.objects.create(
create_at=date_article,
donor=Donor.RST,
id_donor=id_article,
city=city,
title=title,
description=description,
link=article_link,
price=price,
currency=currency,
phones=dict_phones,
name=name,
never_send=False
)
if collection.sms_is_send:
continue
if not email_enable:
self.send_email_to_admin(collection)
if sms_enable:
collection.never_send = False
collection.save()
sms_status = sms.send(validate_sms_phone(collection))
if sms_status:
collection.sms_is_send = True
collection.save()
if email_enable:
self.send_email_to_admin(collection)
def start(self):
print('Start Parse ICAR')
sms = SmsSender()
sms_enable = Settings.get_solo().enable_disable_sms
email_enable = Settings.get_solo().enable_disable_email
list_link = Requester().get_link(
'//div[@class="car"]/div[@class="h"]/a/@href')
for link_article in list_link:
page = Requester(link=link_article)
id_article = self._get_id_article(page)
if not id_article:
continue
if Collections.objects.filter(id_donor__exact=id_article).exists():
continue
date_article = self._get_date_article(page)
if not date_article:
continue
title = self._get_title(page)
price, currency = self._get_price(page)
name, phones = self._get_phone(page)
dict_phones = {key: value for key, value in enumerate(phones)}
description = self._get_description(page)
city = self._get_location(page)
if not filter_parse(title, description, price, currency, city):
continue
collection = Collections.objects.create(create_at=date_article,
donor=Donor.ICAR,
id_donor=id_article,
city=city,
title=title,
description=description,
link=link_article,
price=price,
currency=currency,
phones=dict_phones,
name=name,
never_send=False)
if collection.sms_is_send:
continue
if not email_enable:
self.send_email_to_admin(collection)
if sms_enable:
collection.never_send = False
collection.save()
sms_status = sms.send(validate_sms_phone(collection))
if sms_status:
collection.sms_is_send = True
collection.save()
if email_enable:
self.send_email_to_admin(collection)
def parse_pages(self):
sms = SmsSender()
sms_enable = Settings.get_solo().enable_disable_sms
email_enable = Settings.get_solo().enable_disable_email
for article_url in self._get_list_article():
page_article = Requester(article_url)
stop_date, date_article = self._stop_date_article(page_article)
if stop_date:
break
id_in_site = self._get_id_article(page_article)
if Collections.objects.filter(id_donor__exact=id_in_site).exists():
self.stop_parse += 1
continue
id_article = article_url.split('.')[-2].split('-')[-1]
price, currency = self._get_price(page_article)
phones = self._get_phone(page_article, id_article)
dict_phones = {key: value for key, value in enumerate(phones)}
name = self._get_name(page_article)
description = self._get_description(page_article)
title = self._get_title(page_article)
city = self._get_location(page_article)
if not filter_parse(title, description, price, currency, city):
continue
try:
collection = Collections.objects.create(
create_at=date_article,
donor=Donor.OLX,
id_donor=id_in_site,
city=city,
title=title,
description=description,
link=article_url,
price=price,
currency=currency,
phones=dict_phones,
name=name,
never_send=False
)
except ValueError as e:
print(str(e))
else:
if collection.sms_is_send:
continue
if not email_enable:
self.send_email_to_admin(collection)
if sms_enable:
collection.never_send = False
collection.save()
sms_status = sms.send(validate_sms_phone(collection))
if sms_status:
collection.sms_is_send = True
collection.save()
if email_enable:
self.send_email_to_admin(collection)
def start(self):
print('Start Parse Ria')
sms = SmsSender()
sms_enable = Settings.get_solo().enable_disable_sms
email_enable = Settings.get_solo().enable_disable_email
ids = Requester(self.main_link).get_ids_article()
collections = Collections.objects.filter(
donor=Donor.AUTORIA).values_list('id_donor', flat=True)
if ids:
for id_article in ids:
if id_article in collections:
continue
data_article = Requester(
self.article_link(id_article)).get_json()
all_author_articles = Requester(
self.author_link(data_article.get('userId')))
if all_author_articles.response.status_code == 200:
if len(all_author_articles.get_json().get('result').get(
'search_result').get('ids')) > 1:
continue
phones = [self._normailize_phone(data_article)]
dict_phones = {key: value for key, value in enumerate(phones)}
date_article = self._get_date_article(data_article)
if isinstance(date_article, bool):
continue
city = data_article.get('locationCityName', '')
description = data_article.get('autoData', {}) \
.get('description', '')
title = data_article.get('title', '')
link = ''.join(
[self.link_site,
data_article.get('linkToView', '')])
name = self._get_name(link)
price = data_article.get('USD', '')
if not filter_parse(title, description, price, '$', city):
continue
collection = Collections.objects.create(
create_at=date_article,
donor=Donor.AUTORIA,
id_donor=id_article,
city=city,
title=title,
description=description,
link=link,
price=price,
currency='$',
phones=dict_phones,
name=name,
never_send=False)
if collection.sms_is_send:
continue
if not email_enable:
self.send_email_to_admin(collection)
if sms_enable:
collection.never_send = False
collection.save()
sms_status = sms.send(validate_sms_phone(collection))
if sms_status:
collection.sms_is_send = True
collection.save()
if email_enable:
self.send_email_to_admin(collection)
def __init__(self):
super().__init__()
self.collections = Collections.objects.filter(
sms_is_send=False, never_send=False).order_by('create_at')
self.email_enable = Settings.get_solo().enable_disable_email