def start(self):
sms = SmsSender()
self.check_email()
for collection in self.collections:
phone = validate_sms_phone(collection)
if phone:
sms_status = sms.send(phone)
if sms_status:
error = collection.phones.get('error')
if error:
collection.phones['error'] = ''
collection.sms_is_send = True
if self.email_enable:
if not collection.email_is_send:
self.send_email_to_admin(collection)
else:
collection.phones['error'] = sms.error_message
collection.save()
def start(self):
print('Start Parse RST')
sms = SmsSender()
sms_enable = Settings.get_solo().enable_disable_sms
email_enable = Settings.get_solo().enable_disable_email
for i in range(1, 8):
list_link_articles = Requester(i).get_link('//a[@class="rst-ocb-i-a"]/@href')
for article_link in list_link_articles:
print('Parse', article_link)
page_article = Requester(link=article_link)
id_article = self._get_id_article(page_article)
if Collections.objects.filter(
id_donor__exact=id_article).exists():
continue
date_article = self._get_date_article(page_article)
if not date_article:
continue
title = self._get_title(page_article)
price, currency = self._get_price(page_article)
name, phones = self._get_phone(page_article)
dict_phones = {key: value for key, value in enumerate(phones)}
description = self._get_description(page_article)
city = self._get_location(page_article)
if 'отсутствуют' in name:
continue
if not filter_parse(title, description, price, currency, city):
continue
collection = Collections.objects.create(
create_at=date_article,
donor=Donor.RST,
id_donor=id_article,
city=city,
title=title,
description=description,
link=article_link,
price=price,
currency=currency,
phones=dict_phones,
name=name,
never_send=False
)
if collection.sms_is_send:
continue
if not email_enable:
self.send_email_to_admin(collection)
if sms_enable:
collection.never_send = False
collection.save()
sms_status = sms.send(validate_sms_phone(collection))
if sms_status:
collection.sms_is_send = True
collection.save()
if email_enable:
self.send_email_to_admin(collection)
def start(self):
print('Start Parse ICAR')
sms = SmsSender()
sms_enable = Settings.get_solo().enable_disable_sms
email_enable = Settings.get_solo().enable_disable_email
list_link = Requester().get_link(
'//div[@class="car"]/div[@class="h"]/a/@href')
for link_article in list_link:
page = Requester(link=link_article)
id_article = self._get_id_article(page)
if not id_article:
continue
if Collections.objects.filter(id_donor__exact=id_article).exists():
continue
date_article = self._get_date_article(page)
if not date_article:
continue
title = self._get_title(page)
price, currency = self._get_price(page)
name, phones = self._get_phone(page)
dict_phones = {key: value for key, value in enumerate(phones)}
description = self._get_description(page)
city = self._get_location(page)
if not filter_parse(title, description, price, currency, city):
continue
collection = Collections.objects.create(create_at=date_article,
donor=Donor.ICAR,
id_donor=id_article,
city=city,
title=title,
description=description,
link=link_article,
price=price,
currency=currency,
phones=dict_phones,
name=name,
never_send=False)
if collection.sms_is_send:
continue
if not email_enable:
self.send_email_to_admin(collection)
if sms_enable:
collection.never_send = False
collection.save()
sms_status = sms.send(validate_sms_phone(collection))
if sms_status:
collection.sms_is_send = True
collection.save()
if email_enable:
self.send_email_to_admin(collection)
def parse_pages(self):
sms = SmsSender()
sms_enable = Settings.get_solo().enable_disable_sms
email_enable = Settings.get_solo().enable_disable_email
for article_url in self._get_list_article():
page_article = Requester(article_url)
stop_date, date_article = self._stop_date_article(page_article)
if stop_date:
break
id_in_site = self._get_id_article(page_article)
if Collections.objects.filter(id_donor__exact=id_in_site).exists():
self.stop_parse += 1
continue
id_article = article_url.split('.')[-2].split('-')[-1]
price, currency = self._get_price(page_article)
phones = self._get_phone(page_article, id_article)
dict_phones = {key: value for key, value in enumerate(phones)}
name = self._get_name(page_article)
description = self._get_description(page_article)
title = self._get_title(page_article)
city = self._get_location(page_article)
if not filter_parse(title, description, price, currency, city):
continue
try:
collection = Collections.objects.create(
create_at=date_article,
donor=Donor.OLX,
id_donor=id_in_site,
city=city,
title=title,
description=description,
link=article_url,
price=price,
currency=currency,
phones=dict_phones,
name=name,
never_send=False
)
except ValueError as e:
print(str(e))
else:
if collection.sms_is_send:
continue
if not email_enable:
self.send_email_to_admin(collection)
if sms_enable:
collection.never_send = False
collection.save()
sms_status = sms.send(validate_sms_phone(collection))
if sms_status:
collection.sms_is_send = True
collection.save()
if email_enable:
self.send_email_to_admin(collection)
def start(self):
print('Start Parse Ria')
sms = SmsSender()
sms_enable = Settings.get_solo().enable_disable_sms
email_enable = Settings.get_solo().enable_disable_email
ids = Requester(self.main_link).get_ids_article()
collections = Collections.objects.filter(
donor=Donor.AUTORIA).values_list('id_donor', flat=True)
if ids:
for id_article in ids:
if id_article in collections:
continue
data_article = Requester(
self.article_link(id_article)).get_json()
all_author_articles = Requester(
self.author_link(data_article.get('userId')))
if all_author_articles.response.status_code == 200:
if len(all_author_articles.get_json().get('result').get(
'search_result').get('ids')) > 1:
continue
phones = [self._normailize_phone(data_article)]
dict_phones = {key: value for key, value in enumerate(phones)}
date_article = self._get_date_article(data_article)
if isinstance(date_article, bool):
continue
city = data_article.get('locationCityName', '')
description = data_article.get('autoData', {}) \
.get('description', '')
title = data_article.get('title', '')
link = ''.join(
[self.link_site,
data_article.get('linkToView', '')])
name = self._get_name(link)
price = data_article.get('USD', '')
if not filter_parse(title, description, price, '$', city):
continue
collection = Collections.objects.create(
create_at=date_article,
donor=Donor.AUTORIA,
id_donor=id_article,
city=city,
title=title,
description=description,
link=link,
price=price,
currency='$',
phones=dict_phones,
name=name,
never_send=False)
if collection.sms_is_send:
continue
if not email_enable:
self.send_email_to_admin(collection)
if sms_enable:
collection.never_send = False
collection.save()
sms_status = sms.send(validate_sms_phone(collection))
if sms_status:
collection.sms_is_send = True
collection.save()
if email_enable:
self.send_email_to_admin(collection)