for headline in news_headlines: print(headline['text'] + '\n' + headline['time_stamp']) headline_exist = NewsHeadline.objects.filter(text=headline['text']) if(len(headline_exist) == 0): headline_entry = NewsHeadline() headline_entry.text = headline['text'] headline_entry.time_stamp = headline['time_stamp'] headline_entry.save() d.csvNewsFileName = '.\\crawler\\news\\news_' + str(datetime.datetime.now().day) +'_' + str(datetime.datetime.now().month) +'_' + str(datetime.datetime.now().year) +'_' + str(datetime.datetime.now().hour) +'_' + str(datetime.datetime.now().minute) + '_' + str(datetime.datetime.now().second) +'.csv' d.DumpNewsCSV(news_headlines) prices = d.ParsePricesURL(priceStartDate) for price in prices: print(price['value'] + '\n' + price['time_stamp']) price_exist = Price.objects.filter(time_stamp=price['time_stamp']) if(len(price_exist) == 0): price_entry = Price() price_entry.value = price['value'] price_entry.time_stamp = price['time_stamp'] price_entry.save() d.csvPricesFileName = '.\\crawler\\prices\\prices_'+ str(datetime.datetime.now().day) +'_' + str(datetime.datetime.now().month) +'_' + str(datetime.datetime.now().year) +'_' + str(datetime.datetime.now().hour) +'_' + str(datetime.datetime.now().minute) + '_' + str(datetime.datetime.now().second) +'.csv' d.DumpPricesCSV(prices) # Crawl every hour time.sleep(3600)
def update_market(game): """Get market prices""" supremacy = Supremacy(game.game_id, game.game_host) result = supremacy.market() orders = result["asks"][1] + result["bids"][1] market = Market() market.game_id = game.id market.datetime = datetime.now() db.session.add(market) prices = {} for resource in orders: if resource[1]: lowest_order = resource[1][0] price = Price() price.value = lowest_order["limit"] price.buy = lowest_order["buy"] price.resource_id = lowest_order["resourceType"] market.prices.append(price) prices[price.resource_id] = price for order_json in resource[1]: player = game.players.filter( Player.player_id == order_json["playerID"]).first() order = Order() order.order_id = order_json["orderID"] order.amount = order_json["amount"] order.buy = order_json["buy"] order.limit = order_json["limit"] order.resource_id = order_json["resourceType"] market.orders.append(order) if player is not None: player.orders.append(order) db.session.add(order) db.session.commit() prev_market = market.previous if prev_market: prev_prices = prev_market.price_list if prev_prices: for resource, price in prices.items(): if prev_prices[resource]: price.previous_id = prev_prices[resource].id for resource, price in prices.items(): prev_price = price.previous if prev_price: prev_prev_price = prev_price.previous if prev_prev_price: if prev_prev_price.value == prev_price.value and \ prev_price.value == price.value: price.previous_id = prev_prev_price.id db.session.commit() db.session.delete(prev_price) db.session.commit()