def mock_ads():
all_cat = Category.query.all()
for i in range(100):
posn = GPSPosition(randint(-360000000,360000000), randint(-360000000,360000000))
rand_cat = all_cat[randint(0, len(all_cat)-1)]
location = Location(
longitude=posn.longitude,
latitude=posn.latitude
)
Ad.create(location, "*****@*****.**", "Random Mock App" + random_string(), str(randint(0,1000)), "sample_upload_pic.jpg", rand_cat," ".join([random_string() for i in range(10)]))
db.session.commit()
def get_ads_and_write_to_db(self, link, response):
try:
category = Category.get(name=link)
except DoesNotExist:
category = Category.create(
name=link
)
ad_blocks = response.html.xpath('//div[@class="offer-wrapper"]')
for ad in ad_blocks:
try:
ad_title = ad.xpath('//h3')[0].text
ad_date = ad.xpath('//p[@class="lheight16"]/small[2]')[0].text
ad_price = ad.xpath('//p[@class="price"]')[0].text
ad_photo = ad.xpath('//img/@src')[0]
ad_link = ad.xpath('//h3/a/@href')[0]
ad_city = ad.xpath('//p[@class="lheight16"]/small[1]')[0].text
ad_data = {
'title': ad_title,
'date': ad_date,
'price': ad_price,
'photo': ad_photo,
'link': ad_link,
'city': ad_city,
'category_id': category
}
except Exception as e:
print('Parse error\n', e, type(e))
try:
Ad.create(**ad_data)
except Exception as e:
print('DB error\n', e, type(e))
def create():
"""
Creates an ad
Requires the following param:
* long, lat
* category(id) , email, title, price, image, description
"""
# validation
required_fields = ("long", "lat", "email", "title", "price", "image")
for field in required_fields:
if not field in request.form:
return jsonify({"res": False, "error": "There is an error creating your ad due to missing field(s)."})
# check for price integer
try:
float(request.form.get("price"))
except ValueError:
return jsonify({"res": False, "error": "Price needs to be a numerical value."})
# email format validation
validate_me = StringValidator(request.form.get("email"))
if not validate_me.isEmail():
return jsonify({"res": False, "error": "Not a valid email address."})
from db import Location, Category, Ad
location = Location(longitude=request.form.get("long"), latitude=request.form.get("lat"))
category = Category.get(request.form.get("category"))
id = Ad.create(
location,
request.form.get("email"),
request.form.get("title"),
request.form.get("price"),
save_file(request.form.get("image")),
category,
request.form.get("description", ""),
)
return jsonify({"res": id})
def olx_ad_parser(self, url):
is_ad = True if '/obyavlenie/' in url else False
self.browser.get(url)
sleep(5)
if is_ad:
phone_button = self.browser.find_element_by_xpath(
'//div[contains(@class, "contact-button")]')
phone_button.click()
sleep(2)
html_code = self.browser.page_source
# browser.quit()
dom_tree = html.fromstring(html_code)
dom_tree.make_links_absolute(base_url=url)
if is_ad:
try:
ad_name = dom_tree.xpath('//h1')[0].text.strip()
except Exception as e:
ad_name = 'not found'
try:
ad_price = dom_tree.xpath(
'//div[@class="price-label"]')[0].text_content().strip()
except Exception as e:
ad_price = 'not found'
try:
ad_image = dom_tree.xpath(
'//div[@id="photo-gallery-opener"]/img/@src')[0]
except Exception as e:
ad_image = 'not found'
author = dom_tree.xpath(
'//div[@class="offer-user__details "]/h4/a')
phone = dom_tree.xpath(
'//div[contains(@class, "contact-button")]/strong')[0].text
author = {
'name': author[0].text.strip(),
'profile_link': author[0].attrib['href'],
'phone': phone
}
db_author, created = Author.get_or_create(**author)
categories = dom_tree.xpath('//td[@class="middle"]/ul/li/a')
db_cats = []
for n, cat in enumerate(categories):
if n == 0:
parent = 0
else:
parent = db_cats[n - 1].id
cat = {
'name': cat.text_content().strip(),
'link': cat.attrib['href'],
'parent': parent
}
cat, created = Category.get_or_create(**cat)
db_cats.append(cat)
print(db_cats)
ad = {
'author': db_author,
'url': url,
'name': ad_name.strip(),
'price': ad_price,
'image': ad_image,
'date': datetime.now()
}
db_ad = Ad.create(**ad)
for cat in db_cats:
db_ad.categories.add(cat)
print(ad)
links = dom_tree.xpath('//a/@href')
links = {x for x in links}
return links