def find_steals(self):
shop_urls = [self._mens_shoes, self._mens_cloths]
found_steals = []
for products_url in shop_urls:
products_json = json.loads(requests.get(products_url).content)
page = 1
while products_json['foundProductResults']:
for property in products_json['sections']:
for item in property['items']:
if item['inWallContentCard']:
continue
if item['numberOfColors'] > 1:
for color in item['colorways']:
if color['overriddenLocalPrice']:
title = str(item['title'])
title = title.replace("'", "")
title = title.replace(" ", "_")
old_price = int(
str(color['localPrice']).split()[0])
new_price = int(
str(color['overriddenLocalPrice']).
split()[0])
url = str(color['pdpUrl'])
url_image = str(color['imageUrl'])
item_color = str(color['colorDescription'])
offer = Offer(title, url, url_image,
new_price, old_price,
item_color)
found_steals.append(offer)
# offer.print_offer()
elif color['overriddenLocalPrice']:
title = str(item['title'])
title = title.replace("'", "")
title = title.replace(" ", "_")
old_price = int(
str(color['localPrice']).split()[0])
new_price = int(
str(color['overriddenLocalPrice']).split()[0])
url = str(item['pdpUrl'])
url_image = str(item['spriteSheet'])
offer = Offer(title, url, url_image, new_price,
old_price, item_color)
found_steals.append(offer)
# offer.print_offer()
products_json = json.loads(
requests.get(products_url + "=" + str(page)).content)
page += 1
# for i in found_steals:
# print(i.print_offer())
return found_steals
def getOffers(url):
offers = []
response = simple_get(url)
if response.statusCode != 200:
return response.onError()
html = BeautifulSoup(response.content, 'html.parser')
items = html.select('.ad-listitem')
for item in items:
details = item.select('.aditem-details')
main = item.select('.aditem-main')
if details.__len__() == 0:
continue
price_label = details[0].select('strong')
if len(price_label) == 0 or len(price_label[0].contents) == 0:
continue
price = price_label[0].contents[0]
vb = True if "VB" in price else False
price = u' '.join(price).encode('utf-8').replace(' ', '').replace(
'€', '').replace('VB', '').replace('.', '')
title = main[0].select('h2')[0].select('a')[0].contents[0]
url = main[0].select('h2')[0].find('a', href=True)
url = url['href']
description = main[0].select('p')[0].contents[0]
#keywords = ['tausche', 'Tausch']
#if any(re.findall('|'.join(keywords), description)) or any(re.findall('|'.join(keywords), title)):
if price:
offers.append(Offer(title, price, description, vb, url))
return offers
def find_steals(self):
shop_urls = [self._mens_shoes, self._mens_cloths]
found_steals = []
for products_url in shop_urls:
page = 0
while page<1000:
print(products_url)
raw_html = requests.get(products_url).content
# raw_html = requests.get(products_url+str(page)).content
# soup_html = BeautifulSoup(raw_html, "html.parser")
print(raw_html)
for product_tag in soup_html.find_all('div', {"class": "gl-product-card glass-product-card___1PGiI"}):
print(soup_html)
if product_tag.find('span', {"class": "gl-price__value gl-price__value--crossed"}):
title = product_tag.find('div', {"class": "gl-product-card__name gl-label gl-label--m"}).text
old_price = product_tag.find('div', {"class": "gl-price__value gl-price__value--crossed"}).text
new_price = product_tag.find('div', {"class": "gl-price__value gl-price__value--sale"}).text
url = product_tag.find('a', href=True).attrs['href']
url_image = product_tag.find('img', src=True).attrs['src']
offer = Offer(title, url, url_image, new_price, old_price)
offer.print_offer()
found_steals.append(offer)
page += 48
return found_steals
def place_order(self, user, iid, mid, side, price, quantity):
result = []
ctype = self.contract_type.lookup(iid, mid)
messages = Offer(user, ctype, side, price, quantity).place()
for message in messages:
if message.account == user.id:
result.append(message)
return result
def scrap_linux_milan():
# retrieve links to all recent offers in Milan + 30km related to 'linux'
r = requests.get(
"https://it.jobrapido.com/?&w=linux&l=milano&r=45&sortby=publish_date&shm=all"
)
soup = BeautifulSoup(r.content, 'html.parser')
for a in soup.findAll('a', attrs={'class': 'advert-link'}):
link = a['href']
Offer('Shitty JobRapido Title', link).save_to_file()
def new_offer(id):
if request.method == 'GET':
return render_template('new_offer.html', User=User.get_user(id))
elif request.method == "POST":
values = (None, request.form['title'], request.form['description'],
request.form['price'], id)
Offer(*values).create()
return redirect('/offers/{}/'.format(id))
def scrap_linux_bologna():
# retrieve links to all recent offers in Bologna + 30km related to 'linux'
r = requests.get(
"https://www.helplavoro.it/ricerca/linux/in-provincia-di-bologna.html")
soup = BeautifulSoup(r.content, 'html.parser')
for a in soup.findAll('a', attrs={'class': 'posizione link-offerta'}):
description = a['title'][18::].title()
link = 'https://www.helplavoro.it' + a['href']
Offer(description, link).save_to_file()
def scrap_linux_bologna():
# retrieve links to all recent offers in Bologna + 30km related to 'linux'
r = requests.get(
"https://it.indeed.com/offerte-lavoro?q=linux&l=Bologna%2C+Emilia-Romagna&sort=date"
)
soup = BeautifulSoup(r.content, 'html.parser')
for a in soup.select('a[onmousedown*="rclk(this,jobmap"]'):
description = str(a.contents)[4:-2:]
link = "https://www.indeed.com" + a['href']
Offer(description, link).save_to_file()
def _parse_offers(self, raw=False):
r = requests.get(self.url)
soup = BeautifulSoup(r.content, "html.parser")
tables = soup.find_all("table", class_="bs_kurse")
raw_offers_list = []
for table in tables:
raw_offers_list.extend(table.find("tbody").find_all("tr"))
offers_list = [Offer(offer) for offer in raw_offers_list]
return offers_list
def scrap_linux_bologna():
# retrieve links to all recent offers in Bologna + 30km related to 'linux'
r = requests.get(
"https://www.monster.it/lavoro/cerca/?q=linux&where=Bologna__2C-Emilia-Romagna&client=classic&cy=it&rad=30&intcid=swoop_Hero_Search"
)
soup = BeautifulSoup(r.content, 'html.parser')
for a in soup.findAll('div', attrs={'class': 'summary'}):
name = (a.find('a', href=True))
description = str(name.contents)[2:-6:].title()
link = name['href']
Offer(description, link).save_to_file()
def make_offer(self, player_maker, resources_from, resources_to):
#offers are created as {RESOURCE: number_of_cards, .....}
offer_id = str(uuid.uuid4())
for r, n in resources_from.items():
if player_maker.resources[r] < n:
return False, "You do not have enough resources to create this trade!"
self.offers[offer_id] = Offer(player_maker, resources_from,
resources_to)
return True, ''
def new_offer():
if request.method == 'GET':
return render_template('new_offer.html')
elif request.method == 'POST':
# user = User.find(request.form['email'])
values = (None, user, request.form['title'],
request.form['description'],
request.form['price'],
request.form['date'])
Offer(*values).create()
return redirect(url_for('logged'))
def submitLoanOffer(self, investor, interest):
print('submitLoanOffer ... in progress ...')
validation = self.__isValidOffer(investor, interest)
if validation['isValid']:
offer = Offer(investor, interest, self)
investor.offers.append(offer)
self.offers.append(offer)
return print('Thanks your offer is successfully submitted! \n')
else:
return print(validation['validationErr'])
def upload():
if not current_user.is_authenticated:
return redirect(url_for('index'))
form = UploadForm()
if form.validate_on_submit():
f = form.ffield.data
filename = secure_filename(f.filename)
offerManager.removeFile(filename)
f.save(os.path.join(app.instance_path, '_user_files', filename))
ending = filename[-3:]
print("ending: %s" % (ending))
if ending == "jpg" or ending == "png":
filetype = "image"
elif ending == "mp4" or ending == "flv":
filetype = "video"
elif ending == "txt" or ending == "doc" or ending == "pdf":
filetype = "text"
else:
filetype = "music"
offer = Offer(form.price.data, filename, filetype, current_user.id,
True)
offerManager.offers.append(offer)
offerManager.writeToFile()
readable_hash = ""
with open(os.path.join(app.instance_path, '_user_files', filename),
"rb") as f:
bytes = f.read() # read entire file as bytes
readable_hash = hashlib.sha256(bytes).hexdigest()
if url != "undefined":
requests.post("%s/file/registerFile" % (url),
data={
"filename": filename,
"owner": current_user.email,
"type": "filetype",
"price": form.price.data,
"available": "1",
"hash": readable_hash
})
return redirect(url_for('success'))
else:
return "nope, no good data"
def create_offer(offer):
id = offer['offerId']
link = 'https://www.travelplanet.pl' + offer['offerUrl']
country = offer['country']
region = offer['region']
hotel_name = offer['hotelName']
trip_departure_name = offer['tripDepartureName']
trip_duration = offer['tripDurationText']
trip_dates = offer['tripDatesText']
tour_operator = offer['touroperatorName']
price_one_person = offer['priceOnePerson']
return Offer(id, link, country, region, hotel_name, trip_departure_name, trip_duration, trip_dates, tour_operator,
price_one_person)
def find_steals(self):
self._db_connection_cursor.execute("SELECT * FROM offers")
raw_db_steals = self._db_connection_cursor.fetchall()
db_steals = []
for db_steal in raw_db_steals:
db_steal = (db_steal[:3] + db_steal[3 + 1:])
db_steal = Offer(db_steal[0], db_steal[1], db_steal[2],
db_steal[3], db_steal[4], db_steal[5])
# print(db_steal.print_offer())
db_steals.append(db_steal)
nikeScrapper = NikeScrapper()
# adidasScrapper = AdidasScrapper()
shop_steals = nikeScrapper.find_steals()
# shop_steals = adidasScrapper.find_steals()
exists = 0
self._new_steals = []
for new_steal in shop_steals:
for db_steal in db_steals:
if new_steal._title == db_steal._title and new_steal._color == db_steal._color:
exists = 1
if not exists:
self._new_steals.append(new_steal)
self._db_connection_cursor.execute(
new_steal.get_insert_offer_query())
db_steals.append(new_steal)
exists = 0
self._db_connection.commit()
exists = 0
self._removed_steals = []
for db_steal in db_steals:
for new_steal in shop_steals:
if new_steal._title == db_steal._title and new_steal._color == db_steal._color:
exists = 1
if not exists:
self._removed_steals.append(db_steal)
self._db_connection_cursor.execute(
"DELETE FROM offers WHERE title=? AND color=?",
(db_steal._title, db_steal._color))
exists = 0
self._db_connection.commit()
def __init__(self, documentPath, buysPath):
self.documentPath = documentPath
self.buysPath = buysPath
self.offers = []
self.buys = {}
if not os.path.isfile(self.documentPath):
with open(self.documentPath, "w+") as fo:
fo.write('{"offers":[]}')
if not os.path.isfile(self.buysPath):
f = open(self.buysPath, "w+")
f.write('{}')
f.close()
dataFile = open(self.documentPath, "r")
data = dataFile.read()
dataFile.close()
offers = json.loads(data)
for offerObj in offers["offers"]:
if 'price' in offerObj.keys() and 'filename' in offerObj.keys(
) and 'type' in offerObj.keys() and 'offererId' in offerObj.keys(
) and 'available' in offerObj.keys():
offer = Offer(offerObj['price'], offerObj['filename'],
offerObj['type'], offerObj['offererId'],
offerObj['available'])
self.offers.append(offer)
buysFile = open(self.buysPath, "r")
data = buysFile.read()
buysFile.close()
buysData = json.loads(data)
for buyerId in buysData:
bId = int(buyerId)
self.buys[bId] = []
buysList = buysData[buyerId]
for singleBuy in buysList:
self.buys[bId].append(singleBuy)
print("registered offersmanager at %s with %s stored offers" %
(self.documentPath, str(len(self.offers))))
def __init__(self):
self.server_port = 0
self.server_ip = NETWORK_ADDR
self.server_socket = None
self.selector = selectors.DefaultSelector()
self.udp_broadcast_socket = socket.socket(socket.AF_INET,
socket.SOCK_DGRAM)
self.udp_broadcast_socket.setsockopt(socket.SOL_SOCKET,
socket.SO_BROADCAST, 1)
self.broadcast_thread = None
self.run_broadcast = True
self.broadcast_iterations = 0
self.init_server_socket()
self.offer = Offer(self.server_port)
self.on_game = False
self.init_groups_data()
self.game_data = GameData()
def parse_offers(self):
offer_tree = self.yml_tree.find('.//offers')
for offer_xml in offer_tree.findall('offer'):
offer = Offer()
offer.url = get_text(offer_xml.find('url'))
offer.price = float(get_text(offer_xml.find('price'), 0.0))
offer.oldprice = float(get_text(offer_xml.find('oldprice'), 0.0))
offer.currencyId = get_text(offer_xml.find('currencyId'))
offer.categoryId = get_text(offer_xml.find('categoryId'))
for picture in offer_xml.findall('picture'):
offer.pictures_append(get_text(picture))
offer.store = get_text(offer_xml.find('store'), 'false')
offer.pickup = get_text(offer_xml.find('pickup'), 'false')
offer.delivery = get_text(offer_xml.find('delivery'), 'false')
offer.local_delivery_cost = get_text(
offer_xml.find('local_delivery_cost'))
offer.name = get_text(offer_xml.find('name'))
offer.vendor = get_text(offer_xml.find('vendor'))
offer.vendorCode = get_text(offer_xml.find('vendorCode'))
offer.description = get_text(offer_xml.find('description'))
offer.sales_notes = get_text(offer_xml.find('sales_notes'))
offer.manufacturer_warranty = get_text(
offer_xml.find('manufacturer_warranty'))
offer.country_of_origin = get_text(
offer_xml.find('country_of_origin'))
offer.age = get_text(offer_xml.find('age'))
if not offer.age is None:
offer.age_unit = offer_xml.find('age').attrib.get('unit', None)
offer.barcode = get_text(offer_xml.find('barcode'))
offer.cpa = get_text(offer_xml.find('cpa'))
for param_xml in offer_xml.findall('param'):
try:
param_value = get_text(param_xml)
param_name = param_xml.attrib.get('name', None)
if not param_value is None and not param_name is None:
offer.params_append(param_name=param_name,
param_value=param_value)
except AttributeError:
pass
self.yml.offers_append(offer, validate=False)
def executeTradeStrategy(self, currentManager):
teams_ranking_last_season = sorted(
self.model.managers,
key=lambda manager: manager.game_history[-34:].count(1),
reverse=True)
own_ranking_place = teams_ranking_last_season.index(currentManager)
n_traitors_recruiting = 0
lower_boundary_wins = 0.4
middle_boundary_wins = 0.7
proportion_won_last_season = currentManager.game_history[-34:].count(
1) / 34
if proportion_won_last_season < lower_boundary_wins:
n_traitors_recruiting = 5
if lower_boundary_wins <= proportion_won_last_season <= middle_boundary_wins:
n_traitors_recruiting = 3
budget_for_replacing_player = currentManager.assets / n_traitors_recruiting if n_traitors_recruiting != 0 else 0
candidates_recruitment = []
for better_team in teams_ranking_last_season[:own_ranking_place - 1]:
for pos, player in better_team.team.items():
if player != None:
if player.stats[
'Release Clause'] <= budget_for_replacing_player:
candidates_recruitment.append(player)
narrowed_down_candidate_list = []
if len(candidates_recruitment) >= n_traitors_recruiting:
for i in range(n_traitors_recruiting):
candidate = random.choice(candidates_recruitment)
narrowed_down_candidate_list.append(candidate)
candidates_recruitment.remove(candidate)
elif len(candidates_recruitment) < n_traitors_recruiting:
if len(candidates_recruitment) != 0:
for i in range(len(candidates_recruitment)):
candidate = random.choice(candidates_recruitment)
narrowed_down_candidate_list.append(candidate)
candidates_recruitment.remove(candidate)
for final_candidate in narrowed_down_candidate_list:
Offer(currentManager, final_candidate, final_candidate.position)
def executeTradeStrategy(self, currentManager):
'''
Every season swap your worst player for the best possible player you can buy
'''
# Get worst player
worst_player = None
overall = 100
for pos, player in currentManager.team.items():
if player:
if player.stats['Overall'] < overall:
worst_player = player
overall = worst_player.stats['Overall']
position = worst_player.position
# Buy or send offer to best player you can buy
possible_players = self.pick_player(position, currentManager.assets)
if len(possible_players > 0):
chosen_player = possible_players.iloc[0]
if (chosen_player['Overall'] > worst_player.stats['Overall']):
# Might have to catch a key error if the player isn't in the dictionary here
player_agent = self.model.player_lookup[chosen_player['Name']]
# Send offer to the player
Offer(currentManager, player_agent, position)
def parse(url, offers_container, hours_offset):
url = 'https://www.hltv.org' + url
page = requests.get(url)
sleep(1)
html = Bs(page.content, 'html.parser')
unix_time = html.find('div', class_='time')['data-unix']
match_date_time = datetime.datetime.fromtimestamp(
int(unix_time) / 1000) + datetime.timedelta(hours=hours_offset)
tournament = html.find('div', class_='event text-ellipsis').find('a').text
coef_container = html.find('table', class_='table')
if coef_container:
teams = coef_container.find_all('td', class_='team-cell')
if not teams:
return
team1 = teams[0].text
team2 = teams[2].text
offers_list = []
for offer in coef_container.find_all('tr', class_='provider'):
if not (offer.find('td', class_='noOdds')
or offer['class'][0] == 'hidden'):
offer_container_anchors = offer.find_all('a')
try:
offers_list.append(
Offer(link1=offer_container_anchors[1]['href'],
link2=offer_container_anchors[1]['href'],
coef1=float(offer_container_anchors[1].text),
coef2=float(offer_container_anchors[3].text)))
except ValueError:
pass
add_offer_info_to_container(team1, team2, tournament, match_date_time,
offers_list, url, offers_container)
def test_add_offer(input):
offer = Offer()
offer.add(input, save=False)
assert offer.df.tail(1).to_dict('r')[0] == input
def get_offer_info(url):
html = download_page(url)
# find application/Json script
appJson = html.find(id='server-app-state') if html else None
if appJson:
initialProps = json.loads(appJson.text).get('initialProps') or {}
meta_data = initialProps.get('meta', {}).get('target') or {}
advert_data = initialProps.get('data', {}).get('advert') or {}
photos = []
for photo in advert_data.get('photos', []):
photos.append(
photo.get('large') or photo.get('medium') or photo.get('small')
or photo.get('thumbnail'))
# offers title
name = initialProps.get('meta', {}).get('seo', {}).get('title')
location = advert_data.get('location', {})
address = location.get('address')
coordinates = [
location.get('coordinates', {}).get(item)
for item in ('latitude', 'longitude')
]
# region, city, distinct, sub-region
geo_level = {
item.get('type'): item.get('label')
for item in location.get('geoLevel', [])
}
# rent, rooms number, heating type and others
characteristics = {
item.get('key'): item.get('value')
for item in advert_data.get('characteristics', [])
}
price = {
i: advert_data.get('price', {}).get(i)
for i in ('value', 'unit', 'suffix')
}
# dishwasher, fridge, oven and others
features = advert_data.get('features', [])
features_en = meta_data.get('Equipment_types', []) + meta_data.get(
'Security_types', [])
features_en += meta_data.get('Media_types', []) + meta_data.get(
'Extras_types', [])
if features:
if len(features) > len(features_en):
if features_en:
features_en += features[:-len(features_en)]
else:
features_en = features
# advertiser type: private, business...; agency data; advert type
extra = {
item: advert_data.get(item)
for item in ('advertiser_type', 'advert_type', 'agency')
}
return Offer(
url=url,
name=name,
description=advert_data.get('description'),
photos=photos,
address=address,
coordinates=coordinates,
geo_level=geo_level,
characteristics=characteristics,
price=price,
features=features_en,
owner=advert_data.get('advertOwner'),
extra=extra,
date_created=advert_data.get('dateCreated'),
date_modified=advert_data.get('dateModified')) if appJson else Offer(
url=url)
faces = json.load(f)
f.close()
for t in templates:
if t['name'] == args.format:
template = t
homedir = os.environ['HOME']
try:
working_dir = '{}/Downloads/renders/{}px'.format(
homedir, template['name'])
os.makedirs(working_dir, exist_ok=True)
except FileExistsError:
pass
ad = Offer(drawBot, template, args.spotlight)
i = 0
for c in COLORS:
frame_path = template['name'] + c + '@2x'
face = faces[randint(0,
len(faces) - 1)] # pick a random portrait to use
i += 1
print('Rendering {} of {}'.format(i, len(COLORS)))
ad.render(frame_path, face, args.copy, args.cta, args.offer)
fp = "{}/Downloads/renders/{}px/{}-{}.png".format(
homedir, template['name'], template['name'], 'offer' + c)
ad.save(fp)
ad.end()
# resize the image
