for headline in news_headlines:
print(headline['text'] + '\n' + headline['time_stamp'])
headline_exist = NewsHeadline.objects.filter(text=headline['text'])
if(len(headline_exist) == 0):
headline_entry = NewsHeadline()
headline_entry.text = headline['text']
headline_entry.time_stamp = headline['time_stamp']
headline_entry.save()
d.csvNewsFileName = '.\\crawler\\news\\news_' + str(datetime.datetime.now().day) +'_' + str(datetime.datetime.now().month) +'_' + str(datetime.datetime.now().year) +'_' + str(datetime.datetime.now().hour) +'_' + str(datetime.datetime.now().minute) + '_' + str(datetime.datetime.now().second) +'.csv'
d.DumpNewsCSV(news_headlines)
prices = d.ParsePricesURL(priceStartDate)
for price in prices:
print(price['value'] + '\n' + price['time_stamp'])
price_exist = Price.objects.filter(time_stamp=price['time_stamp'])
if(len(price_exist) == 0):
price_entry = Price()
price_entry.value = price['value']
price_entry.time_stamp = price['time_stamp']
price_entry.save()
d.csvPricesFileName = '.\\crawler\\prices\\prices_'+ str(datetime.datetime.now().day) +'_' + str(datetime.datetime.now().month) +'_' + str(datetime.datetime.now().year) +'_' + str(datetime.datetime.now().hour) +'_' + str(datetime.datetime.now().minute) + '_' + str(datetime.datetime.now().second) +'.csv'
d.DumpPricesCSV(prices)
# Crawl every hour
time.sleep(3600)
def update_market(game):
"""Get market prices"""
supremacy = Supremacy(game.game_id, game.game_host)
result = supremacy.market()
orders = result["asks"][1] + result["bids"][1]
market = Market()
market.game_id = game.id
market.datetime = datetime.now()
db.session.add(market)
prices = {}
for resource in orders:
if resource[1]:
lowest_order = resource[1][0]
price = Price()
price.value = lowest_order["limit"]
price.buy = lowest_order["buy"]
price.resource_id = lowest_order["resourceType"]
market.prices.append(price)
prices[price.resource_id] = price
for order_json in resource[1]:
player = game.players.filter(
Player.player_id == order_json["playerID"]).first()
order = Order()
order.order_id = order_json["orderID"]
order.amount = order_json["amount"]
order.buy = order_json["buy"]
order.limit = order_json["limit"]
order.resource_id = order_json["resourceType"]
market.orders.append(order)
if player is not None:
player.orders.append(order)
db.session.add(order)
db.session.commit()
prev_market = market.previous
if prev_market:
prev_prices = prev_market.price_list
if prev_prices:
for resource, price in prices.items():
if prev_prices[resource]:
price.previous_id = prev_prices[resource].id
for resource, price in prices.items():
prev_price = price.previous
if prev_price:
prev_prev_price = prev_price.previous
if prev_prev_price:
if prev_prev_price.value == prev_price.value and \
prev_price.value == price.value:
price.previous_id = prev_prev_price.id
db.session.commit()
db.session.delete(prev_price)
db.session.commit()
