def download_ads(api_url: str, current_party: str) -> None:
"""
Request ads from the Facebook Ad Library API.
Parse and write all found ads.
If the request returns a paging url, a recursive call is made to follow that url.
:param api_url: The url to request.
:param current_party: The party we are requesting ads for.
"""
response = requests.get(api_url)
response_data = response.json()
if "error" in response_data:
logging.error(f"Error from API: '{response_data['error']}'")
return
if len(response_data["data"]) > 0:
logging.info(f"Got {len(response_data['data'])} ads ({current_party})")
Ad.insert_many(
json_to_ad_dict(ad, current_party)
for ad in response_data["data"]).on_conflict_replace().execute()
if "paging" in response_data:
download_ads(response_data["paging"]["next"], current_party)
def test_parse_ad(self):
response = '''
{"response":[
{"id":"607256","campaign_id":"123","name":"Ad1","status":0,"approved":0,"all_limit":0,"cost_type":0,"cpm":118},
{"id":"664868","campaign_id":"123","name":"Ad2","status":1,"approved":1,"all_limit":100,"cost_type":1,"cpc":488}
]}
'''
account = AccountFactory(remote_id=1)
campaign = CampaignFactory(account=account, remote_id=1, fetched=datetime.now())
instance = Ad(campaign=campaign, fetched=datetime.now())
instance.parse(json.loads(response)['response'][0])
instance.save(commit_remote=False)
self.assertTrue(isinstance(instance.campaign, Campaign))
self.assertEqual(instance.campaign.remote_id, 1)
self.assertEqual(instance.remote_id, 607256)
self.assertEqual(instance.name, "Ad1")
self.assertEqual(instance.status, False)
self.assertEqual(instance.approved, False)
self.assertEqual(instance.all_limit, 0)
self.assertEqual(instance.cpm, 118)
instance = Ad(campaign=campaign, fetched=datetime.now())
instance.parse(json.loads(response)['response'][1])
instance.save(commit_remote=False)
self.assertEqual(instance.remote_id, 664868)
self.assertEqual(instance.name, "Ad2")
self.assertEqual(instance.status, True)
self.assertEqual(instance.approved, True)
self.assertEqual(instance.all_limit, 100)
self.assertEqual(instance.cpc, 488)
def place_ad(chat_id):
""" Place the ad"""
u = session.query(User).filter(User.chat_id == int(chat_id)).one()
ad = Ad(cash_dict[chat_id]['OBJ'].lower(),
cash_dict[chat_id]['DESCRIPTION'], cash_dict[chat_id]['PRICE'])
ad.user = u
session.add(ad)
session.commit()
bot.send_message(chat_id, 'Your ad was created.')
def ad_add():
idcrag = request.form['idcrag'].strip()
description = request.form['description'].strip()
title = request.form['title'].strip()
posting_time = request.form['posting_time'].strip()
scheduled_action = request.form['scheduled_action'].strip()
repost_timeout = request.form['repost_timeout'].strip()
prev_action = ""
prev_act_time = ""
prev_act_stat = ""
status = "Not posted"
idusers = request.form['idusers'].strip()
idcategory = request.form['category'].strip()
idarea = request.form['area'].strip()
replymail = None #request.form['replymail']
allowed_actions = "None,add"
contact_phone = request.form['contact_phone'].strip()
contact_name = request.form['contact_name'].strip()
postal = request.form['postal'].strip()
specific_location = request.form['specific_location'].strip()
haslicense = request.form['has_license'].strip()
license_info = request.form['license'].strip()
a = Ad(idcrag,
description,
title,
posting_time,
scheduled_action,
repost_timeout,
prev_action,
prev_act_time,
prev_act_stat,
status,
idusers,
idcategory,
idarea,
replymail,
allowed_actions,
contact_phone,
contact_name,
postal,
specific_location,
haslicense,
license_info)
db_session.add(a)
try:db_session.commit()
except Exception as e:
db_session.rollback()
logging.error("Application did not create "+a.__repr__()+
" becouse " + e.message)
raise Exception('DB commit is not OK')
return a.idads.__str__()
def is_new(url, title):
"""
Checks if a certain ad was already added to the db
:param url:
:param title:
:return:
"""
try:
Ad.get(Ad.url == url, Ad.title == title)
return False
except DoesNotExist:
return True
def add_db_entry(url, title):
"""
Adds the ad to the db
:param url:
:param title:
:return:
"""
row = Ad(url=url, title=title)
try:
row.save()
except DatabaseError as e:
logging.exception(e)
def add_ad(self, title, seller_id, car_id):
"""
Добавить объявление
:param title: Тайтл объявления
:param seller_id: айди пользователя-продавца
:param car_id: айди автомобиля в системе
:return: Type dict: новое объявление
"""
new_ad = Ad(title=title, date=time.time(), seller_id=seller_id, car_id=car_id)
self.session.add(new_ad)
self.session.commit()
return new_ad.as_dict()
def build_avito_xml(data):
list_ads = []
for item in data:
list_ads.append(Ad(item))
ads = etree.Element("Ads", formatVersion="3", target="Avito.ru")
for item in list_ads:
if item.validate():
ad = etree.SubElement(ads, "Ad")
etree_item(ad, "Id", item.id)
etree_item(ad, "Category", item.category)
etree_item(ad, "OperationType", item.operationType)
etree_item(ad, "DateBegin", item.get_date_begin_text())
etree_item(ad, "DateEnd", item.get_date_end_text())
etree_item(ad, "Address", item.address)
etree_item(ad, "Latitude", item.latitude)
etree_item(ad, "Longitude", item.longitude)
etree_item(ad, "Description", item.description)
etree_item(ad, "Price", item.price)
etree_item(ad, "CompanyName", item.companyName)
etree_item(ad, "ManagerName", item.managerName)
etree_item(ad, "AllowEmail", item.get_allow_email_text())
if item.images and len(item.images):
group = etree.SubElement(ad, "Images")
for url in item.images:
etree.SubElement(group, "Images", url=url)
etree_item(ad, "VideoURL", item.videoURL)
etree_item(ad, "AdStatus", item.adStatus)
etree_item(ad, "Rooms", item.rooms)
etree_item(ad, "Square", item.square)
etree_item(ad, "Floor", item.floor)
etree_item(ad, "Floors", item.floors)
etree_item(ad, "HouseType", item.houseType)
etree_item(ad, "LeaseType", item.leaseType)
etree_item(ad, "PropertyRights", item.propertyRights)
etree_item(ad, "LeaseCommissionSize", item.leaseCommissionSize)
etree_item(ad, "LeaseDeposit", item.leaseDeposit)
etree_item(ad, "LeaseBeds", item.leaseBeds)
etree_item(ad, "LeaseSleepingPlaces", item.leaseSleepingPlaces)
etree_grouping(ad, "LeaseMultimedia", item.leaseMultimedia)
etree_grouping(ad, "LeaseAppliances", item.leaseAppliances)
etree_grouping(ad, "LeaseComfort", item.leaseComfort)
etree_grouping(ad, "LeaseAdditionally", item.leaseAdditionally)
etree_item(ad, "MarketType", item.marketType)
etree_item(ad, "NewDevelopmentId", item.newDevelopmentId)
tree = etree.ElementTree(ads)
tree.write("output/avito.xml",
pretty_print=True,
xml_declaration=True,
encoding="utf-8")
def get_or_create(session, model, adv_dict):
instance = session.query(model).filter_by(id=adv_dict['id']).first()
if instance:
return instance, True
else:
instance = Ad()
for key, value in adv_dict.items():
if hasattr(instance, key):
setattr(instance, key, value)
session.add(instance)
return instance, False
def ad_add():
idcrag = request.form['idcrag'].strip()
description = request.form['description'].strip()
title = request.form['title'].strip()
posting_time = request.form['posting_time'].strip()
scheduled_action = request.form['scheduled_action'].strip()
repost_timeout = request.form['repost_timeout'].strip()
prev_action = ""
prev_act_time = ""
prev_act_stat = ""
status = "Not posted"
idusers = request.form['idusers'].strip()
idcategory = request.form['category'].strip()
idarea = request.form['area'].strip()
replymail = None #request.form['replymail']
allowed_actions = "None,add"
contact_phone = request.form['contact_phone'].strip()
contact_name = request.form['contact_name'].strip()
postal = request.form['postal'].strip()
specific_location = request.form['specific_location'].strip()
haslicense = request.form['has_license'].strip()
license_info = request.form['license'].strip()
a = Ad(idcrag, description, title, posting_time, scheduled_action,
repost_timeout, prev_action, prev_act_time, prev_act_stat, status,
idusers, idcategory, idarea, replymail, allowed_actions,
contact_phone, contact_name, postal, specific_location, haslicense,
license_info)
db_session.add(a)
try:
db_session.commit()
except Exception as e:
db_session.rollback()
logging.error("Application did not create " + a.__repr__() +
" becouse " + e.message)
raise Exception('DB commit is not OK')
return a.idads.__str__()
def success(request):
website = None
tmpname = request.POST['website_name']
tmpurl = request.POST['website_url']
try:
website = Website.objects.get(url=tmpurl)
except Website.DoesNotExist:
website = Website(name=tmpname, url=tmpurl)
website.save()
data = json.loads(request.POST['ads'])
for ad in data:
a = Ad(name=ad['fields']['name'],
age=ad['fields']['age'],
ethnicity=ad['fields']['ethnicity'],
phone_number=ad['fields']['phone_number'],
location=ad['fields']['location'],
ad=ad['fields']['ad'],
date=ad['fields']['date'],
website=website)
a.save()
return HttpResponse("Success! <a href=\"/upload/\">Add More</a> <a href=\"/\">Home</a>")
def create_ad():
if request.method == 'POST':
title = request.form['title']
body = request.form['body']
try:
ad = Ad(title=title, body=body)
db.session.add(ad)
db.session.commit()
except:
print("Unsuccessfully")
return redirect(url_for('ads.index'))
form = AdForm()
return render_template('ads/create_ad.html', form=form)
def handle_uploaded_file(f, w):
print("in huf")
skipped = False
results = []
for row in csv.reader(f, delimiter=','):
if not skipped:
skipped = True
else:
#print(row)
a = Ad(name=row[0],
age=row[1],
ethnicity=row[2],
phone_number=row[3],
location=row[4],
ad=row[5],
date=row[6],
website=w)
results.append(a)
return results
def targeting_stats(request, ad_id=None):
'''
Return targeting audiency and recommended bid
'''
kwargs = dict([(k, v) for k, v in request.POST.items()
if k[:6] in ['layout', 'target']])
if 'account_id' in request.POST:
try:
account = Account.objects.get(id=request.POST['account_id'])
except:
pass
elif ad_id:
try:
account = Ad.objects.get(id=ad_id).account
except:
pass
stat = TargetingStats.remote.get(ad=Ad(account=account, **kwargs))
response = stat.__dict__
del response['_state']
return JsonResponse(response)
def test_crud_ad(self):
account = AccountFactory(remote_id=ACCOUNT_ID)
client = ClientFactory(remote_id=CLIENT_ID, account=account)
campaign = Campaign.remote.create(account=account, client=client, name='Test_campaign1', day_limit=1000, all_limit=2000)
image = Image(**IMAGE_INITIAL_FIELDS)
self.objects_to_delete += [campaign]
self.assertTrue(campaign.remote_id > 0)
# create
ad = Ad(campaign=campaign, name='Test_ad1', status=False, cost_type=0, image=image, cpc=100,
layout__title='111', layout__link_url='http://ya.ru', layout__description='q'*50)
ad.save()
self.objects_to_delete += [ad]
# create another
image.id = None
ad = Ad.remote.create(campaign=campaign, name='Test_ad2', status=False, cost_type=0, image=image, cpc=100,
layout__title='111', layout__link_url='http://ya.ru', layout__description='q'*50)
self.objects_to_delete += [ad]
self.assertTrue(ad.remote_id > 0)
self.assertEqual(ad.name, 'Test_ad2')
self.assertEqual(ad.status, False)
self.assertEqual(ad.cost_type, 0)
self.assertEqual(ad.cpc, 100)
# update
ad.name = 'Test_ad3'
ad.save()
campaign.fetch_ads(ids=[ad.remote_id])
ad1 = Ad.objects.get(remote_id=ad.remote_id)
self.assertTrue(ad1.name == ad.name == 'Test_ad3')
self.assertEqual(ad1.cpc, 100)
# delete
ad1.delete()
self.assertEqual(Ad.objects.filter(remote_id=ad.remote_id)[0].archived, True)
campaign.fetch_ads(ids=[ad.remote_id])
self.assertEqual(Ad.objects.filter(remote_id=ad.remote_id)[0].archived, True)
campaign.delete()
self.objects_to_delete = []
from models import Ad
try:
db_postfix = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
db_backup = shutil.copyfile(DB_PATH, f'{DB_PATH}.backup_{db_postfix}')
print(f'DB backup saved here: {db_backup}')
except Exception as e:
print(f'Backup error: {e}')
try:
ads = db.session.query(Ad).filter(Ad.active == True)
ads_ids = set([ad.id for ad in ads])
with open('db/ads.json') as file:
new_ads = json.loads(file.read())
new_ads_ids = set((new_ad['id'] for new_ad in new_ads))
ads_ids_to_deactivate = ads_ids.difference(new_ads_ids)
ads = db.session.query(Ad).filter(Ad.id.in_(ads_ids_to_deactivate)).update(
{'active': False}, synchronize_session=False)
db.session.commit()
print(f'{len(ads_ids_to_deactivate)} ads were deactivated.')
for new_ad in new_ads:
db.session.merge(Ad(**new_ad))
db.session.commit()
print('{} new ads were upserted with status "active". \n'
'There are {} entries totally in the database.'.format(
len(new_ads), len(Ad.query.all())))
except Exception as e:
print(
f'Import error: {e}. Please restore you backup DB file and check your json file'
)
from models import Ad
from utils import recursive_round, render_template, time_range_len
logging.basicConfig(
format="[%(asctime)s] %(levelname)s: %(message)s",
level=logging.INFO,
datefmt="%Y-%m-%d %H:%M:%S",
)
SEPT_1 = date(year=2020, month=9, day=1)
NUMBER_OF_DATES = time_range_len(SEPT_1, date.today())
if __name__ == "__main__":
ads_per_party = {
p: Ad.ads_in_time_range(first_date=SEPT_1).where(Ad.party == p)
for p in PARTIES
}
for party in PARTIES:
logging.info(
f"Processing {len(ads_per_party[party])} ads for {party}.")
party_data = {
"total-ads":
len(ads_per_party[party]),
"spending-total-lower":
sum(ad.spending_lower for ad in ads_per_party[party]),
"spending-total-upper":
sum(ad.spending_upper for ad in ads_per_party[party]),
}
from constants import DATA_TYPES, PARTIES
from models import Ad
from utils import recursive_round, render_template, time_range_len
logging.basicConfig(
format="[%(asctime)s] %(levelname)s: %(message)s",
level=logging.INFO,
datefmt="%Y-%m-%d %H:%M:%S",
)
SEPT_1 = date(year=2020, month=9, day=1)
NUMBER_OF_DATES = time_range_len(SEPT_1, date.today())
if __name__ == "__main__":
ads = list(Ad.ads_in_time_range(first_date=SEPT_1))
ads_per_party = {p: [ad for ad in ads if ad.party == p] for p in PARTIES}
most_expensive_ad = max(ads, key=lambda e: e.average_spending_per_day)
logging.info("Creating general data.")
general_data = {
"number-of-ads-total":
len(ads),
"number-of-ads-party": [len(ads_per_party[p]) for p in PARTIES],
"spending-total-lower":
sum(ad.spending_lower for ad in ads),
"spending-total-upper":
sum(ad.spending_upper for ad in ads),
"spending-party":
def delete(self, ad_id):
ad = Ad.by_id(ad_id)
user = ad.creator_id
ad.delete()
return jsonify({'message': f'Ad was deleted'})
def post(self):
ad = Ad(**request.json)
ad.add()
return jsonify(ad.to_dict())
def get(self, ad_id):
ad = Ad.by_id(ad_id)
return jsonify(ad.to_dict())
def json_to_ad_dict(ad_json_data: dict, party: str) -> dict:
"""
Transform a json object into an dictionary that corresponds with the Ad model.
:param ad_json_data: Json object representing an ad from the Facebook API.
:param party: Current party to parse.
:return: A dict corresponds with the Ad model.
"""
spending_lower, spending_upper = _parse_estimated_value(
ad_json_data, "spend")
spending_lower *= CURRENCY_EXCHANGE_RATE_MAP[ad_json_data["currency"]]
spending_upper *= CURRENCY_EXCHANGE_RATE_MAP[ad_json_data["currency"]]
impressions_lower, impressions_upper = _parse_estimated_value(
ad_json_data, "impressions")
audience_size_lower, audience_size_upper = _parse_estimated_value(
ad_json_data, "estimated_audience_size")
ad_dict = {
"ad_id":
ad_json_data["id"],
"page_id":
ad_json_data["page_id"],
"party":
party,
"creation_date":
_parse_date(ad_json_data, "ad_creation_time"),
"start_date":
_parse_date(ad_json_data, "ad_delivery_start_time"),
"end_date":
_parse_date(ad_json_data, "ad_delivery_stop_time"),
"creative_bodies":
_parse_content(ad_json_data, "ad_creative_bodies"),
"creative_link_captions":
_parse_content(ad_json_data, "ad_creative_link_captions"),
"creative_link_descriptions":
_parse_content(ad_json_data, "ad_creative_link_descriptions"),
"creative_link_titles":
_parse_content(ad_json_data, "ad_creative_link_titles"),
"spending_lower":
spending_lower,
"spending_upper":
spending_upper,
"impressions_lower":
impressions_lower,
"impressions_upper":
impressions_upper,
"audience_size_lower":
audience_size_lower,
"audience_size_upper":
audience_size_upper,
}
if "languages" in ad_json_data and ad_json_data["languages"] != ["nl"]:
logging.warning(
f"Non-dutch language detected "
f"({ad_dict['ad_id']}): {','.join(ad_json_data['languages'])}")
if "delivery_by_region" in ad_json_data:
for distribution in ad_json_data["delivery_by_region"]:
region = distribution["region"]
percentage = Decimal(distribution["percentage"])
if region == "North Brabant":
region = "Noord-Brabant"
if region in REGIONS:
field_name = Ad.demographic_to_field_name(region)
ad_dict[field_name] = percentage
else:
if region not in REGION_IGNORE_LIST:
logging.warning(
f"Unknown region: {region} ({ad_dict['ad_id']})")
if "demographic_distribution" in ad_json_data:
for distribution in ad_json_data["demographic_distribution"]:
percentage = Decimal(distribution["percentage"])
for demographic in (distribution["gender"], distribution["age"]):
if demographic in GENDERS or demographic in AGE_RANGES:
field_name = Ad.demographic_to_field_name(demographic)
if field_name not in ad_dict:
ad_dict[field_name] = percentage
else:
ad_dict[field_name] += percentage
else:
if demographic not in GENDER_IGNORE_LIST:
logging.warning(f"Unknown gender/age group: "
f"{demographic} ({ad_dict['ad_id']})")
ad_dict["themes"] = _parse_themes(" ".join([
ad_dict["creative_bodies"],
ad_dict["creative_link_descriptions"],
ad_dict["creative_link_titles"],
]))
return ad_dict
def new_ad(request):
if request.method == 'POST':
form = AdForm(request.POST, request.FILES)
if form.is_valid():
title_url = title_url1 = slugify(form.cleaned_data['title'])
tmp = 0
while Ad.objects.filter(title_url=title_url1).count() > 0:
tmp += 1
title_url1 = title_url + '-' + str(tmp)
title_url = title_url1
ad = Ad(user_id=request.user.id, title = form.cleaned_data['title'])
ad.city = form.cleaned_data['city']
ad.address = form.cleaned_data['address']
ad.psc = form.cleaned_data['psc']
ad.category = form.cleaned_data['category']
ad.text_of_ad = form.cleaned_data['text_of_ad']
ad.type_of_ad = form.cleaned_data['type_of_ad']
ad.is_new = form.cleaned_data['is_new']
ad.is_antique = form.cleaned_data['is_antique']
ad.is_inventory = form.cleaned_data['is_inventory']
ad.can_be_reservated = form.cleaned_data['can_be_reservated']
ad.time_of_publication = datetime.datetime.now()
ad.title_url = title_url
ad.latitude, ad.longitude = getGPS(ad.address, ad.city.city, ad.psc)
ad.save()
if form.cleaned_data['image1'] != None:
image1 = AdImage(ad_id=ad.id, image=form.cleaned_data['image1'])
image1.save()
if form.cleaned_data['image2'] != None:
image2 = AdImage(ad_id=ad.id, image=form.cleaned_data['image2'])
image2.save()
if form.cleaned_data['image3'] != None:
image3 = AdImage(ad_id=ad.id, image=form.cleaned_data['image3'])
image3.save()
if form.cleaned_data['image4'] != None:
image4 = AdImage(ad_id=ad.id, image=form.cleaned_data['image4'])
image4.save()
messages.info(request, 'Vaše inzerát bol pridaný')
return HttpResponseRedirect('/')
else:
form = AdForm(initial={'type_of_ad': True})
page_info = {}
page_info['title'] = 'Informácie o Vás - Váš profil'
page_info['page'] = 0
page_info['form_name'] = 'new_ad'
page_info['form_action'] = '/pridat-inzerat/'
return render_to_response('pridaj_inzerat.html', {'form': form, 'countInfo': countInfo, 'recentNews': recentNews,
'page_info': page_info},
context_instance=RequestContext(request))
def parse_home(self, response):
debug_html_content(response,"parse_home",1)
sel = scrapy.Selector(text=response.body, type='html')
for row in sel.xpath('//tr')[2:]: # crrop table head
status = row.xpath('./td[contains(@class,"status")]/small/text()')\
.extract_first()
allowed_categories = [cat.fullname for cat in Category.query.all()]
rawcatname = row.xpath('./td[contains(@class,"areacat")]/text()')\
.extract()
cat_name = filter(lambda x: x.strip()!='', rawcatname)[0].strip()
if cat_name in allowed_categories:
#+ default None action
allowed_actions = row.xpath('./td[contains(@class,"buttons")]'+
'/form/input[@class="managebtn"]'+
'/@value').extract() + ["None"]
url = row.xpath('./td[contains(@class,"title")]/a/@href')\
.extract_first()
idcrag = row.xpath('./td[contains(@class,"postingID")]/text()')\
.extract_first().strip()
title = row.xpath('./td[contains(@class,"title")]/text()')\
.extract_first().strip()
area_code = row.xpath('./td[contains(@class,"areacat")]/b/text()')\
.extract_first().strip()
area = Area.query.filter(Area.clcode == area_code).first()
category = Category.query.filter(Category.fullname == cat_name)\
.first()
ad = Ad.query.filter(Ad.idcrag == idcrag).first()
if not ad:
ad = Ad(
idcrag = idcrag,
description='',
title=title,
posting_time='',
scheduled_action="",
repost_timeout="",
prev_action ="",
prev_act_time ="",
prev_act_stat ="",
status=status,
idusers=self.user.idusers,
idcategory=category.idcategory,
idarea=area.idarea,
replymail='',
allowed_actions = ','.join(allowed_actions),
contact_phone='',
contact_name='',
postal='',
specific_location='',
haslicense='',
license_info='')
db_session.add(ad)
try:
db_session.commit()
except:
db_session.rollback()
raise Exception("DB commit is not OK")
elif ad:
ad.status = status
ad.allowed_actions = ','.join(allowed_actions)
db_session.add(ad)
try:
db_session.commit()
except Exception as e:
db_session.rollback()
raise Exception("DB commit is not OK\n"+e.message)
else: logging.error('')
if url:
print "RUNNING OUT AD", ad.idads
yield scrapy.Request(
url=url,
meta={'idads':ad.idads},
callback=self.parse_ad)
