def main(self):
if self.isRegionValid():
#items = AuctionEu.objects.distinct('item')
cursor = connection.cursor()
cursor.execute("SELECT DISTINCT `item` FROM %s" % Auction._meta.db_table)
if cursor:
auctionItems = cursor.fetchall()
item_service = ItemService()
itemsIds = item_service.getItemList()
created = 0
log.debug('Inserting...')
for auctionItem in auctionItems:
if not auctionItem[0] in itemsIds:
# Insert
url = self.getItemApiLink(str(auctionItem[0]))
newItem = None
try:
newItem = self.getApiLinkData(url)
except urllib.error.URLError as e:
log.error(label['@DBU7'] % url)
log.error(e)
if newItem != None:
if item_service.insert(newItem):
created += 1
else:
# error
pass
else:
pass
# Item exist in database
log.info('Inserted %s items' % str(created))
else:
log.error(label['@DBU6'])
def item_list(request):
item_service = ItemService()
items = item_service.getAllItems()
if items:
serializer = ItemDetailSerializer(items, many=True)
return Response(serializer.data)
else:
return Response(status=status.HTTP_404_NOT_FOUND)
def item_detail(request, itemId):
item_service = ItemService()
item = item_service.getItemByItemId(itemId)
if item:
serializer = ItemDetailSerializer(item)
return Response(serializer.data)
else:
return Response(status=status.HTTP_404_NOT_FOUND)
def realm_item(request, itemId, realm_slug):
item_service = ItemService()
realm_service = RealmService()
tsd_service = TSDHourlyService()
realm = realm_service.getRealmBySlug(realm_slug)
item = item_service.getItemByItemId(itemId)
if realm and item:
tsd = tsd_service.getRealmItemLastData(item, realm.connected_realm)
serializer = SingleTSDHourlySerializer(tsd)
return Response(serializer.data)
else:
return Response(status=status.HTTP_404_NOT_FOUND)
def tsd_daily_details_realm_time_chart(request, itemId, realm_slug):
item_service = ItemService()
realm_service = RealmService()
tsd_service = TSDDailyService()
realm = realm_service.getRealmBySlug(realm_slug)
item = item_service.getItemByItemId(itemId)
if realm and item:
tsd_list = tsd_service.getRealmItemChartData(item, realm.connected_realm)
serializer = TSDDailyDetailsChartSerializer(tsd_list, many=True)
return Response(serializer.data)
else:
return Response(status=status.HTTP_404_NOT_FOUND)
def hourlyMain(self):
log.debug(label['@DBU1'])
log.debug(label['@DBU24'] % 'TSD HOURLY')
realm_service = RealmService()
connected_realm_service = ConnectedRealmService()
item_service = ItemService()
auction_service = AuctionService()
tsd_hourly_service = TSDHourlyService()
connected_realms_tab = realm_service.getActiveRealmsConnectedRealmId()
# Get itemId price list map
item_list = item_service.getAllItems()
# IMPORTANT! BELOW SET MAX ALLOW AMOUT OF PROCCESS (depending on CPU cores)
max_proc = 3
item_proc_list = [[] for _ in range(max_proc)]
for i, item in enumerate(item_list):
item_proc_list[i % max_proc].append(item)
total = 0
total_bulk = 0
for connected_realms_id in connected_realms_tab:
out_q = multiprocessing.Queue()
realms = realm_service.getRealmNamesByConnectedRealmId(
connected_realms_id)
auctions = []
for realm in realms:
auctions += auction_service.getAllRealmActiveAuctionsList(
realm)
if len(auctions) > 0:
calculation = Calculation(auctions)
log.debug(label['@DBU39'] % str(connected_realms_id))
log.debug(label['@DBU41'] % str(len(auctions)))
connected_realm = connected_realm_service.getConnectedRealm(
connected_realms_id)
calculations = [
multiprocessing.Process(target=self.hourlyProcCalculation,
args=(item_sublist, calculation,
out_q))
for item_sublist in item_proc_list
]
for p in calculations:
p.start()
result = []
for i in range(max_proc):
result += out_q.get()
for p in calculations:
p.join()
result_bulk = int(len(result) / 500)
result_bulk_list = [[] for _ in range(result_bulk)]
for i, each in enumerate(result):
result_bulk_list[i % result_bulk].append(each)
total_bulk = 0
for result_bulk_sublist in result_bulk_list:
total_bulk += len(result_bulk_sublist)
tsd_hourly_service.batchInsert(result_bulk_sublist,
connected_realm.id)
if total_bulk != len(item_list):
log.warning("Missing %d items in bulk insert" %
(len(item_list) - total_bulk))
else:
log.warning(label['@DBU41'] % str(len(auctions)))
log.debug(label['@DBU40'] % str(connected_realms_id))
total += total_bulk
log.debug(label['@DBU38'] % ('TSD HOURLY', str(total)))
log.debug(label['@DBU2'])