def __init__(self, config):
self.config = config
self.store = PersistentStore(self.config['dbfilename'])
self.spotify_session = SpotifyAsyncSessionManager(useragent="SpotifyThings")
self.spotify_session.player_state_changed += self.__player_state_changed
self.spotify_session.current_track_changed += self.__current_track_changed
self.spotify_session.playback_progress += self.__playback_progress_event
self.spotify_session.playqueue_changed += self.__playqueue_changed_event
self.spotify_session.login_state_changed += self.__login_state_changed_event
self.spotify_session.password_blob_changed += self.__login_password_blob_changed
self.wsregistry = WebSocketRegistry();
self.wsregistry.on_new_client += self.__on_new_client
self.webapp = WebApplication(self)
self.asyncioloop = AsyncIOLoop()
self.attached_rfid_readers = dict()
self.rfid = RFID()
self.rfid.setOnAttachHandler(self.rfidAttached)
self.rfid.setOnDetachHandler(self.rfidDetached)
self.rfid.setOnErrorhandler(self.rfidError)
self.rfid.setOnTagHandler(self.rfidTagGained)
self.rfid.setOnTagLostHandler(self.rfidTagLost)
def _crawl(result):
deferred = process.crawl('rofxnet', domain='rofxnet')
persistentstore = PersistentStore()
logging.info('Persistent store...')
persistentstore.log(logging)
logging.info('Parsing items...')
try:
to_be_opened = persistentstore.to_be_opened_items()
to_be_closed = persistentstore.to_be_closed_items()
logging.info('To be opened: ' + str(len(to_be_opened)))
logging.info('To be closed: ' + str(len(to_be_closed)))
for k, v in orderTradeMap.items():
persistentstore.tag_trade_id(k, v)
for key, item in to_be_opened.items():
symbol = get_close_matches(item['symbol'],
symbols) # EURUSD > EUR/USD
symbol = next(iter(symbol), None)
if symbol:
is_buy = (item['action'].upper() == 'BUY')
lots = item['lots']
order = con.open_trade(symbol=symbol,
is_buy=is_buy,
rate=0,
is_in_pips=False,
amount=lots,
time_in_force='GTC',
order_type='AtMarket')
if order != 0:
persistentstore.tag_opened(key, str(order.__orderId__),
time.time())
logging.info('New order opened: ' + key)
persistentstore.tag_opened(key, str(order.__orderId__),
time.time())
if order is None: #returns None when instantly opened? grab from trades table
fake_order_id = int(time.time())
persistentstore.tag_attemped_opened(key, fake_order_id)
logging.info(
'Server returned None instead of Order but order was opened'
)
logging.info('Attemping manual grep from trades table.')
trades = con.get_open_positions()
trades = trades.loc[(trades['currency'] == symbol) & (
trades['isBuy'] == is_buy)].sort_values(
by=['time']) #get open trades for given parameters
for index, trade in trades.iterrows():
trade_id = trade['tradeId']
existing = persistentstore.get_item_by_trade_id(
trade_id) #might be opened already?
if existing is None:
persistentstore.tag_opened(key, str(fake_order_id),
time.time())
orderTradeMap[str(fake_order_id)] = str(trade_id)
persistentstore.tag_trade_id(
fake_order_id, trade_id)
break
for hashid in to_be_closed:
item = persistentstore.get_item_by_id(hashid)
lots = item['lots']
con.close_trade(trade_id=item['tradeid'], amount=lots)
profit = 0 #TODO
persistentstore.tag_closed(hashid, profit, time.time())
logging.info('Position closed: ' + hashid)
except:
logging.error('Exception parsing item')
persistentstore.clear_parsed_items()
persistentstore.dump()
deferred.addCallback(
lambda results: logging.info('waiting 60 seconds before restart...'))
deferred.addCallback(sleep, seconds=60)
deferred.addCallback(_crawl)
deferred.addErrback(crash) # <-- add errback here
return deferred
def parse(self, response):
system_id = response.url.split("/")[-1]
if (dt.now().minute in [0, 5, 10, 15, 25, 30, 35, 40, 45, 50,
55]): #log parsed html every 30 minutes
file_name = str(dt.now())
f = open(f"{BASE_DIR}/parsed/{system_id}-{file_name}-ALL.html",
"w+")
f.write(response.text)
f.close()
noData = response.selector.xpath(
'//div[@id="openTrades"]/div/span/text()').extract_first()
if (noData == 'No data to display.'):
self.logger.info('No open trades found for %s', response.url)
return
else:
self.logger.info('Open trades found for %s', response.url)
operations = response.selector.xpath(
'//div[@id="openTrades"]/table/tr')
if not len(operations):
operations = response.selector.xpath(
'//div[@id="openTrades"]/table/tbody/tr')
self.logger.info('Operations: ' + str(len(operations)))
if len(operations) < 3: #header, [...], footer
self.logger.info('Empty operations set')
else:
persistentstore = PersistentStore()
persistentstore.set_inited(
) #are items availabe to compare for closing
column_names = operations[0]
for operation in operations[1:-1]: #remove header and total rows
self.logger.info('processing operation...')
if (dt.now().minute
in [0, 30]): #log parsed html every 30 minutes
file_name = str(dt.now())
f = open(
f"{BASE_DIR}/parsed/{system_id}-{file_name}-OPENED.html",
"w+")
f.write(response.text)
f.close()
try:
openDateRaw = operation.xpath(
('td[not(@style="display:none")][%i]//text()' %
self.get_column_index_by_name(
column_names,
'Open Date'))).extract_first().strip()
symbolRaw = operation.xpath(
('td[not(@style="display:none")][%i]//span//text()' %
self.get_column_index_by_name(
column_names, 'Symbol'))).extract_first()
actionRaw = operation.xpath(
('td[not(@style="display:none")][%i]//text()' %
self.get_column_index_by_name(
column_names, 'Action'))).extract_first().strip()
lotsRaw = 5
openPriceRaw = operation.xpath(
('td[not(@style="display:none")][%i]//text()' %
self.get_column_index_by_name(
column_names,
'Open Price'))).extract_first().strip()
if not openPriceRaw.strip():
openPriceRaw = 0
self.logger.info(
'openDate: %s, symbolRaw: %s, actionRaw: %s, lotsRaw: %s, openPriceRaw: %s',
openDateRaw, symbolRaw, actionRaw, lotsRaw,
openPriceRaw)
item = Operation()
str_hash = str(openDateRaw) + str(symbolRaw) + str(
actionRaw) + str(lotsRaw) + str(
openPriceRaw) + system_id
item['hashid'] = self.get_hash(str_hash)
item['timeOrderDiscovered'] = time.time()
item['timeTradeOpened'] = None
item['timeTradeClosed'] = None
item['timeTradeAttempedOpened'] = None
item['symbol'] = symbolRaw
item['action'] = actionRaw
item['lots'] = 1 #TODO fix this
item['openPrice'] = float(openPriceRaw)
item['closePrice'] = None
item['url'] = response.url
item['profit'] = None
item['orderid'] = None
item['tradeid'] = None
self.logger.info('Position parsing completed')
yield item
except:
self.logger.error('Exception parsing position')
class AppContainer(object):
def __init__(self, config):
self.config = config
self.store = PersistentStore(self.config['dbfilename'])
self.spotify_session = SpotifyAsyncSessionManager(useragent="SpotifyThings")
self.spotify_session.player_state_changed += self.__player_state_changed
self.spotify_session.current_track_changed += self.__current_track_changed
self.spotify_session.playback_progress += self.__playback_progress_event
self.spotify_session.playqueue_changed += self.__playqueue_changed_event
self.spotify_session.login_state_changed += self.__login_state_changed_event
self.spotify_session.password_blob_changed += self.__login_password_blob_changed
self.wsregistry = WebSocketRegistry();
self.wsregistry.on_new_client += self.__on_new_client
self.webapp = WebApplication(self)
self.asyncioloop = AsyncIOLoop()
self.attached_rfid_readers = dict()
self.rfid = RFID()
self.rfid.setOnAttachHandler(self.rfidAttached)
self.rfid.setOnDetachHandler(self.rfidDetached)
self.rfid.setOnErrorhandler(self.rfidError)
self.rfid.setOnTagHandler(self.rfidTagGained)
self.rfid.setOnTagLostHandler(self.rfidTagLost)
# player events
def __player_state_changed(self, state):
self.wsregistry.broadcast('app', PlayerStateMessage(state))
def __current_track_changed(self, track):
self.wsregistry.broadcast('app', PlayerCurrentTrackMessage(track))
def __playback_progress_event(self, playback_time):
self.wsregistry.broadcast('app', PlayerPlaybackProgressMessage(int(playback_time)))
def __playqueue_changed_event(self):
self.wsregistry.broadcast('app', PlayerQueueModifiedMessage())
# app event
def __login_state_changed_event(self, is_logged_in, login_error, current_user):
self.wsregistry.broadcast('app', SpotifyLoginStateMessage(is_logged_in, login_error, current_user))
def __login_password_blob_changed(self, username, password_blob):
self.store.set_config('spotify.username', username)
self.store.set_config('spotify.password_blob', password_blob)
self.wsregistry.broadcast('app', SpotifyCredentialStoredMessage(self.store.get_config('spotify.password_blob') is not None))
# websocket events
def __on_new_client(self, wstype, ws):
def spotify_state_handler(current_track, track_playback_time, player_state):
# send player state
ws.write_message(PlayerStateMessage(player_state))
# send current track
ws.write_message(PlayerCurrentTrackMessage(current_track))
# send playback progress
ws.write_message(PlayerPlaybackProgressMessage(int(track_playback_time)))
def spotify_login_state_handler(is_logged_in, login_error, current_user):
# send login state message
ws.write_message(SpotifyLoginStateMessage(is_logged_in, login_error, current_user))
if wstype == "app":
# request spotify state
self.spotify_session.get_player_state(spotify_state_handler)
# request login state
self.spotify_session.get_login_state(spotify_login_state_handler)
# send message with currently attched rfid readers
ws.write_message(RfidReaderUpdatedMessage(self.attached_rfid_readers))
# send message with credentials stored status
ws.write_message(SpotifyCredentialStoredMessage(self.store.get_config('spotify.password_blob') is not None))
# rfid events
def rfidAttached(self, e):
device = e.device
device.setAntennaOn(True)
# add to list of attached readers
reader = RfidReader(
device.getDeviceName(),
device.getDeviceVersion(),
device.getSerialNum())
self.attached_rfid_readers[reader.serial_no] = reader
# notify clients
self.wsregistry.broadcast('app', RfidReaderUpdatedMessage(self.attached_rfid_readers))
logger.info('RFID reader %s Attached', device.getSerialNum())
def rfidDetached(self, e):
device = e.device
serial_no = device.getSerialNum()
# remove from attached list
del self.attached_rfid_readers[serial_no]
# notify clients
self.wsregistry.broadcast('app', RfidReaderUpdatedMessage(self.attached_rfid_readers))
logger.info('RFID reader %s Deteched', device.getSerialNum())
def rfidError(self, e):
try:
source = e.device
logger.error("RFID %i: Phidget Error %i: %s", source.getSerialNum(), e.eCode, e.description)
except PhidgetException as e:
logger.exception('Phidget Exception %i: %s', e.code, e.details)
def rfidTagGained(self, e):
source = e.device
self.rfid.setLEDOn(1)
logger.debug('RFID %i: Tag Read: %s', source.getSerialNum(), e.tag)
# get existing mapping, if any
tagmapping = self.store.find_by_tagid(e.tag)
# broadcast to any listeners
any_listeners = self.wsregistry.broadcast('rfid', RfidTagReadMessage(e.tag, tagmapping))
# no listeners and resource was mapped, play resource
if not any_listeners and tagmapping:
self.spotify_session.play_link(tagmapping.spotifylink, None)
def rfidTagLost(self, e):
source = e.device
self.rfid.setLEDOn(0)
logger.debug('RFID %i: Tag Lost: %s', source.getSerialNum(), e.tag)
def forget_spotify_credentials(self):
# delete from store
self.store.delete_configs(['spotify.username', 'spotify.password_blob'])
# notify clients
self.wsregistry.broadcast('app', SpotifyCredentialStoredMessage(self.store.get_config('spotify.password_blob') is not None))
def start(self):
logger.info('Starting appcontainer')
# initialize store
logger.info('Initializing store')
self.store.initialize()
# start spotify
logger.info('Starting spotify session')
self.spotify_session.start()
# login using stored credentials
username = self.store.get_config('spotify.username')
password_blob = self.store.get_config('spotify.password_blob')
if username and password_blob:
logger.info('Logging in with username %s', username)
self.spotify_session.login(username, None, False, password_blob)
# start webapp
listen_port = self.config['listen_port']
logger.info('Starting web application, listening on port %s', listen_port)
self.webapp.listen(listen_port)
self.asyncioloop.start()
# start rfid
logger.info('Initializing RFID reader')
self.rfid.openPhidget()
logger.info('Appcontainer started')
def stop(self):
logger.info('Stopping appcontainer')
# stop rfid
logger.info('Closing RFID reader')
self.rfid.closePhidget()
# stop webapp
logger.info('Stopping web application')
self.asyncioloop.stop()
# stop spotify
logger.info('Stopping Spotify session')
self.spotify_session.stop()
logger.info('Appcontainer stopped')
def process_item(self, item, spider):
persistentstore = PersistentStore()
persistentstore.add_parsed_item(item)
return item