def parse(self, response): system_id = response.url.split("/")[-1] if (dt.now().minute in [0, 5, 10, 15, 25, 30, 35, 40, 45, 50, 55]): #log parsed html every 30 minutes file_name = str(dt.now()) f = open(f"{BASE_DIR}/parsed/{system_id}-{file_name}-ALL.html", "w+") f.write(response.text) f.close() noData = response.selector.xpath( '//div[@id="openTrades"]/div/span/text()').extract_first() if (noData == 'No data to display.'): self.logger.info('No open trades found for %s', response.url) return else: self.logger.info('Open trades found for %s', response.url) operations = response.selector.xpath( '//div[@id="openTrades"]/table/tr') if not len(operations): operations = response.selector.xpath( '//div[@id="openTrades"]/table/tbody/tr') self.logger.info('Operations: ' + str(len(operations))) if len(operations) < 3: #header, [...], footer self.logger.info('Empty operations set') else: persistentstore = PersistentStore() persistentstore.set_inited( ) #are items availabe to compare for closing column_names = operations[0] for operation in operations[1:-1]: #remove header and total rows self.logger.info('processing operation...') if (dt.now().minute in [0, 30]): #log parsed html every 30 minutes file_name = str(dt.now()) f = open( f"{BASE_DIR}/parsed/{system_id}-{file_name}-OPENED.html", "w+") f.write(response.text) f.close() try: openDateRaw = operation.xpath( ('td[not(@style="display:none")][%i]//text()' % self.get_column_index_by_name( column_names, 'Open Date'))).extract_first().strip() symbolRaw = operation.xpath( ('td[not(@style="display:none")][%i]//span//text()' % self.get_column_index_by_name( column_names, 'Symbol'))).extract_first() actionRaw = operation.xpath( ('td[not(@style="display:none")][%i]//text()' % self.get_column_index_by_name( column_names, 'Action'))).extract_first().strip() lotsRaw = 5 openPriceRaw = operation.xpath( ('td[not(@style="display:none")][%i]//text()' % self.get_column_index_by_name( column_names, 'Open Price'))).extract_first().strip() if not openPriceRaw.strip(): openPriceRaw = 0 self.logger.info( 'openDate: %s, symbolRaw: %s, actionRaw: %s, lotsRaw: %s, openPriceRaw: %s', openDateRaw, symbolRaw, actionRaw, lotsRaw, openPriceRaw) item = Operation() str_hash = str(openDateRaw) + str(symbolRaw) + str( actionRaw) + str(lotsRaw) + str( openPriceRaw) + system_id item['hashid'] = self.get_hash(str_hash) item['timeOrderDiscovered'] = time.time() item['timeTradeOpened'] = None item['timeTradeClosed'] = None item['timeTradeAttempedOpened'] = None item['symbol'] = symbolRaw item['action'] = actionRaw item['lots'] = 1 #TODO fix this item['openPrice'] = float(openPriceRaw) item['closePrice'] = None item['url'] = response.url item['profit'] = None item['orderid'] = None item['tradeid'] = None self.logger.info('Position parsing completed') yield item except: self.logger.error('Exception parsing position')