def __init__(self, config): self.config = config self.store = PersistentStore(self.config['dbfilename']) self.spotify_session = SpotifyAsyncSessionManager(useragent="SpotifyThings") self.spotify_session.player_state_changed += self.__player_state_changed self.spotify_session.current_track_changed += self.__current_track_changed self.spotify_session.playback_progress += self.__playback_progress_event self.spotify_session.playqueue_changed += self.__playqueue_changed_event self.spotify_session.login_state_changed += self.__login_state_changed_event self.spotify_session.password_blob_changed += self.__login_password_blob_changed self.wsregistry = WebSocketRegistry(); self.wsregistry.on_new_client += self.__on_new_client self.webapp = WebApplication(self) self.asyncioloop = AsyncIOLoop() self.attached_rfid_readers = dict() self.rfid = RFID() self.rfid.setOnAttachHandler(self.rfidAttached) self.rfid.setOnDetachHandler(self.rfidDetached) self.rfid.setOnErrorhandler(self.rfidError) self.rfid.setOnTagHandler(self.rfidTagGained) self.rfid.setOnTagLostHandler(self.rfidTagLost)
def _crawl(result): deferred = process.crawl('rofxnet', domain='rofxnet') persistentstore = PersistentStore() logging.info('Persistent store...') persistentstore.log(logging) logging.info('Parsing items...') try: to_be_opened = persistentstore.to_be_opened_items() to_be_closed = persistentstore.to_be_closed_items() logging.info('To be opened: ' + str(len(to_be_opened))) logging.info('To be closed: ' + str(len(to_be_closed))) for k, v in orderTradeMap.items(): persistentstore.tag_trade_id(k, v) for key, item in to_be_opened.items(): symbol = get_close_matches(item['symbol'], symbols) # EURUSD > EUR/USD symbol = next(iter(symbol), None) if symbol: is_buy = (item['action'].upper() == 'BUY') lots = item['lots'] order = con.open_trade(symbol=symbol, is_buy=is_buy, rate=0, is_in_pips=False, amount=lots, time_in_force='GTC', order_type='AtMarket') if order != 0: persistentstore.tag_opened(key, str(order.__orderId__), time.time()) logging.info('New order opened: ' + key) persistentstore.tag_opened(key, str(order.__orderId__), time.time()) if order is None: #returns None when instantly opened? grab from trades table fake_order_id = int(time.time()) persistentstore.tag_attemped_opened(key, fake_order_id) logging.info( 'Server returned None instead of Order but order was opened' ) logging.info('Attemping manual grep from trades table.') trades = con.get_open_positions() trades = trades.loc[(trades['currency'] == symbol) & ( trades['isBuy'] == is_buy)].sort_values( by=['time']) #get open trades for given parameters for index, trade in trades.iterrows(): trade_id = trade['tradeId'] existing = persistentstore.get_item_by_trade_id( trade_id) #might be opened already? if existing is None: persistentstore.tag_opened(key, str(fake_order_id), time.time()) orderTradeMap[str(fake_order_id)] = str(trade_id) persistentstore.tag_trade_id( fake_order_id, trade_id) break for hashid in to_be_closed: item = persistentstore.get_item_by_id(hashid) lots = item['lots'] con.close_trade(trade_id=item['tradeid'], amount=lots) profit = 0 #TODO persistentstore.tag_closed(hashid, profit, time.time()) logging.info('Position closed: ' + hashid) except: logging.error('Exception parsing item') persistentstore.clear_parsed_items() persistentstore.dump() deferred.addCallback( lambda results: logging.info('waiting 60 seconds before restart...')) deferred.addCallback(sleep, seconds=60) deferred.addCallback(_crawl) deferred.addErrback(crash) # <-- add errback here return deferred
def parse(self, response): system_id = response.url.split("/")[-1] if (dt.now().minute in [0, 5, 10, 15, 25, 30, 35, 40, 45, 50, 55]): #log parsed html every 30 minutes file_name = str(dt.now()) f = open(f"{BASE_DIR}/parsed/{system_id}-{file_name}-ALL.html", "w+") f.write(response.text) f.close() noData = response.selector.xpath( '//div[@id="openTrades"]/div/span/text()').extract_first() if (noData == 'No data to display.'): self.logger.info('No open trades found for %s', response.url) return else: self.logger.info('Open trades found for %s', response.url) operations = response.selector.xpath( '//div[@id="openTrades"]/table/tr') if not len(operations): operations = response.selector.xpath( '//div[@id="openTrades"]/table/tbody/tr') self.logger.info('Operations: ' + str(len(operations))) if len(operations) < 3: #header, [...], footer self.logger.info('Empty operations set') else: persistentstore = PersistentStore() persistentstore.set_inited( ) #are items availabe to compare for closing column_names = operations[0] for operation in operations[1:-1]: #remove header and total rows self.logger.info('processing operation...') if (dt.now().minute in [0, 30]): #log parsed html every 30 minutes file_name = str(dt.now()) f = open( f"{BASE_DIR}/parsed/{system_id}-{file_name}-OPENED.html", "w+") f.write(response.text) f.close() try: openDateRaw = operation.xpath( ('td[not(@style="display:none")][%i]//text()' % self.get_column_index_by_name( column_names, 'Open Date'))).extract_first().strip() symbolRaw = operation.xpath( ('td[not(@style="display:none")][%i]//span//text()' % self.get_column_index_by_name( column_names, 'Symbol'))).extract_first() actionRaw = operation.xpath( ('td[not(@style="display:none")][%i]//text()' % self.get_column_index_by_name( column_names, 'Action'))).extract_first().strip() lotsRaw = 5 openPriceRaw = operation.xpath( ('td[not(@style="display:none")][%i]//text()' % self.get_column_index_by_name( column_names, 'Open Price'))).extract_first().strip() if not openPriceRaw.strip(): openPriceRaw = 0 self.logger.info( 'openDate: %s, symbolRaw: %s, actionRaw: %s, lotsRaw: %s, openPriceRaw: %s', openDateRaw, symbolRaw, actionRaw, lotsRaw, openPriceRaw) item = Operation() str_hash = str(openDateRaw) + str(symbolRaw) + str( actionRaw) + str(lotsRaw) + str( openPriceRaw) + system_id item['hashid'] = self.get_hash(str_hash) item['timeOrderDiscovered'] = time.time() item['timeTradeOpened'] = None item['timeTradeClosed'] = None item['timeTradeAttempedOpened'] = None item['symbol'] = symbolRaw item['action'] = actionRaw item['lots'] = 1 #TODO fix this item['openPrice'] = float(openPriceRaw) item['closePrice'] = None item['url'] = response.url item['profit'] = None item['orderid'] = None item['tradeid'] = None self.logger.info('Position parsing completed') yield item except: self.logger.error('Exception parsing position')
class AppContainer(object): def __init__(self, config): self.config = config self.store = PersistentStore(self.config['dbfilename']) self.spotify_session = SpotifyAsyncSessionManager(useragent="SpotifyThings") self.spotify_session.player_state_changed += self.__player_state_changed self.spotify_session.current_track_changed += self.__current_track_changed self.spotify_session.playback_progress += self.__playback_progress_event self.spotify_session.playqueue_changed += self.__playqueue_changed_event self.spotify_session.login_state_changed += self.__login_state_changed_event self.spotify_session.password_blob_changed += self.__login_password_blob_changed self.wsregistry = WebSocketRegistry(); self.wsregistry.on_new_client += self.__on_new_client self.webapp = WebApplication(self) self.asyncioloop = AsyncIOLoop() self.attached_rfid_readers = dict() self.rfid = RFID() self.rfid.setOnAttachHandler(self.rfidAttached) self.rfid.setOnDetachHandler(self.rfidDetached) self.rfid.setOnErrorhandler(self.rfidError) self.rfid.setOnTagHandler(self.rfidTagGained) self.rfid.setOnTagLostHandler(self.rfidTagLost) # player events def __player_state_changed(self, state): self.wsregistry.broadcast('app', PlayerStateMessage(state)) def __current_track_changed(self, track): self.wsregistry.broadcast('app', PlayerCurrentTrackMessage(track)) def __playback_progress_event(self, playback_time): self.wsregistry.broadcast('app', PlayerPlaybackProgressMessage(int(playback_time))) def __playqueue_changed_event(self): self.wsregistry.broadcast('app', PlayerQueueModifiedMessage()) # app event def __login_state_changed_event(self, is_logged_in, login_error, current_user): self.wsregistry.broadcast('app', SpotifyLoginStateMessage(is_logged_in, login_error, current_user)) def __login_password_blob_changed(self, username, password_blob): self.store.set_config('spotify.username', username) self.store.set_config('spotify.password_blob', password_blob) self.wsregistry.broadcast('app', SpotifyCredentialStoredMessage(self.store.get_config('spotify.password_blob') is not None)) # websocket events def __on_new_client(self, wstype, ws): def spotify_state_handler(current_track, track_playback_time, player_state): # send player state ws.write_message(PlayerStateMessage(player_state)) # send current track ws.write_message(PlayerCurrentTrackMessage(current_track)) # send playback progress ws.write_message(PlayerPlaybackProgressMessage(int(track_playback_time))) def spotify_login_state_handler(is_logged_in, login_error, current_user): # send login state message ws.write_message(SpotifyLoginStateMessage(is_logged_in, login_error, current_user)) if wstype == "app": # request spotify state self.spotify_session.get_player_state(spotify_state_handler) # request login state self.spotify_session.get_login_state(spotify_login_state_handler) # send message with currently attched rfid readers ws.write_message(RfidReaderUpdatedMessage(self.attached_rfid_readers)) # send message with credentials stored status ws.write_message(SpotifyCredentialStoredMessage(self.store.get_config('spotify.password_blob') is not None)) # rfid events def rfidAttached(self, e): device = e.device device.setAntennaOn(True) # add to list of attached readers reader = RfidReader( device.getDeviceName(), device.getDeviceVersion(), device.getSerialNum()) self.attached_rfid_readers[reader.serial_no] = reader # notify clients self.wsregistry.broadcast('app', RfidReaderUpdatedMessage(self.attached_rfid_readers)) logger.info('RFID reader %s Attached', device.getSerialNum()) def rfidDetached(self, e): device = e.device serial_no = device.getSerialNum() # remove from attached list del self.attached_rfid_readers[serial_no] # notify clients self.wsregistry.broadcast('app', RfidReaderUpdatedMessage(self.attached_rfid_readers)) logger.info('RFID reader %s Deteched', device.getSerialNum()) def rfidError(self, e): try: source = e.device logger.error("RFID %i: Phidget Error %i: %s", source.getSerialNum(), e.eCode, e.description) except PhidgetException as e: logger.exception('Phidget Exception %i: %s', e.code, e.details) def rfidTagGained(self, e): source = e.device self.rfid.setLEDOn(1) logger.debug('RFID %i: Tag Read: %s', source.getSerialNum(), e.tag) # get existing mapping, if any tagmapping = self.store.find_by_tagid(e.tag) # broadcast to any listeners any_listeners = self.wsregistry.broadcast('rfid', RfidTagReadMessage(e.tag, tagmapping)) # no listeners and resource was mapped, play resource if not any_listeners and tagmapping: self.spotify_session.play_link(tagmapping.spotifylink, None) def rfidTagLost(self, e): source = e.device self.rfid.setLEDOn(0) logger.debug('RFID %i: Tag Lost: %s', source.getSerialNum(), e.tag) def forget_spotify_credentials(self): # delete from store self.store.delete_configs(['spotify.username', 'spotify.password_blob']) # notify clients self.wsregistry.broadcast('app', SpotifyCredentialStoredMessage(self.store.get_config('spotify.password_blob') is not None)) def start(self): logger.info('Starting appcontainer') # initialize store logger.info('Initializing store') self.store.initialize() # start spotify logger.info('Starting spotify session') self.spotify_session.start() # login using stored credentials username = self.store.get_config('spotify.username') password_blob = self.store.get_config('spotify.password_blob') if username and password_blob: logger.info('Logging in with username %s', username) self.spotify_session.login(username, None, False, password_blob) # start webapp listen_port = self.config['listen_port'] logger.info('Starting web application, listening on port %s', listen_port) self.webapp.listen(listen_port) self.asyncioloop.start() # start rfid logger.info('Initializing RFID reader') self.rfid.openPhidget() logger.info('Appcontainer started') def stop(self): logger.info('Stopping appcontainer') # stop rfid logger.info('Closing RFID reader') self.rfid.closePhidget() # stop webapp logger.info('Stopping web application') self.asyncioloop.stop() # stop spotify logger.info('Stopping Spotify session') self.spotify_session.stop() logger.info('Appcontainer stopped')
def process_item(self, item, spider): persistentstore = PersistentStore() persistentstore.add_parsed_item(item) return item