Ejemplo n.º 1
0
    def __init__(self, config):
        self.config = config
        self.store = PersistentStore(self.config['dbfilename'])


        self.spotify_session = SpotifyAsyncSessionManager(useragent="SpotifyThings")
        self.spotify_session.player_state_changed += self.__player_state_changed
        self.spotify_session.current_track_changed += self.__current_track_changed
        self.spotify_session.playback_progress += self.__playback_progress_event
        self.spotify_session.playqueue_changed += self.__playqueue_changed_event
        self.spotify_session.login_state_changed += self.__login_state_changed_event
        self.spotify_session.password_blob_changed += self.__login_password_blob_changed

        self.wsregistry = WebSocketRegistry();
        self.wsregistry.on_new_client += self.__on_new_client

        self.webapp = WebApplication(self)
        self.asyncioloop = AsyncIOLoop()

        self.attached_rfid_readers = dict()

        self.rfid = RFID()
        self.rfid.setOnAttachHandler(self.rfidAttached)
        self.rfid.setOnDetachHandler(self.rfidDetached)
        self.rfid.setOnErrorhandler(self.rfidError)
        self.rfid.setOnTagHandler(self.rfidTagGained)
        self.rfid.setOnTagLostHandler(self.rfidTagLost)
Ejemplo n.º 2
0
def _crawl(result):
    deferred = process.crawl('rofxnet', domain='rofxnet')
    persistentstore = PersistentStore()

    logging.info('Persistent store...')
    persistentstore.log(logging)

    logging.info('Parsing items...')

    try:
        to_be_opened = persistentstore.to_be_opened_items()
        to_be_closed = persistentstore.to_be_closed_items()

        logging.info('To be opened: ' + str(len(to_be_opened)))
        logging.info('To be closed: ' + str(len(to_be_closed)))

        for k, v in orderTradeMap.items():
            persistentstore.tag_trade_id(k, v)

        for key, item in to_be_opened.items():
            symbol = get_close_matches(item['symbol'],
                                       symbols)  # EURUSD > EUR/USD
            symbol = next(iter(symbol), None)
            if symbol:
                is_buy = (item['action'].upper() == 'BUY')
                lots = item['lots']
                order = con.open_trade(symbol=symbol,
                                       is_buy=is_buy,
                                       rate=0,
                                       is_in_pips=False,
                                       amount=lots,
                                       time_in_force='GTC',
                                       order_type='AtMarket')
                if order != 0:
                    persistentstore.tag_opened(key, str(order.__orderId__),
                                               time.time())
                    logging.info('New order opened: ' + key)
                    persistentstore.tag_opened(key, str(order.__orderId__),
                                               time.time())

                if order is None:  #returns None when instantly opened? grab from trades table
                    fake_order_id = int(time.time())
                    persistentstore.tag_attemped_opened(key, fake_order_id)
                    logging.info(
                        'Server returned None instead of Order but order was opened'
                    )
                    logging.info('Attemping manual grep from trades table.')
                    trades = con.get_open_positions()
                    trades = trades.loc[(trades['currency'] == symbol) & (
                        trades['isBuy'] == is_buy)].sort_values(
                            by=['time'])  #get open trades for given parameters
                    for index, trade in trades.iterrows():
                        trade_id = trade['tradeId']
                        existing = persistentstore.get_item_by_trade_id(
                            trade_id)  #might be opened already?
                        if existing is None:
                            persistentstore.tag_opened(key, str(fake_order_id),
                                                       time.time())
                            orderTradeMap[str(fake_order_id)] = str(trade_id)
                            persistentstore.tag_trade_id(
                                fake_order_id, trade_id)
                            break

        for hashid in to_be_closed:
            item = persistentstore.get_item_by_id(hashid)
            lots = item['lots']
            con.close_trade(trade_id=item['tradeid'], amount=lots)
            profit = 0  #TODO
            persistentstore.tag_closed(hashid, profit, time.time())
            logging.info('Position closed: ' + hashid)
    except:
        logging.error('Exception parsing item')

    persistentstore.clear_parsed_items()
    persistentstore.dump()

    deferred.addCallback(
        lambda results: logging.info('waiting 60 seconds before restart...'))
    deferred.addCallback(sleep, seconds=60)
    deferred.addCallback(_crawl)
    deferred.addErrback(crash)  # <-- add errback here
    return deferred
Ejemplo n.º 3
0
    def parse(self, response):

        system_id = response.url.split("/")[-1]

        if (dt.now().minute in [0, 5, 10, 15, 25, 30, 35, 40, 45, 50,
                                55]):  #log parsed html every 30 minutes
            file_name = str(dt.now())
            f = open(f"{BASE_DIR}/parsed/{system_id}-{file_name}-ALL.html",
                     "w+")
            f.write(response.text)
            f.close()

        noData = response.selector.xpath(
            '//div[@id="openTrades"]/div/span/text()').extract_first()

        if (noData == 'No data to display.'):
            self.logger.info('No open trades found for %s', response.url)
            return
        else:
            self.logger.info('Open trades found for %s', response.url)

        operations = response.selector.xpath(
            '//div[@id="openTrades"]/table/tr')

        if not len(operations):
            operations = response.selector.xpath(
                '//div[@id="openTrades"]/table/tbody/tr')

        self.logger.info('Operations: ' + str(len(operations)))

        if len(operations) < 3:  #header, [...], footer
            self.logger.info('Empty operations set')
        else:
            persistentstore = PersistentStore()
            persistentstore.set_inited(
            )  #are items availabe to compare for closing

            column_names = operations[0]

            for operation in operations[1:-1]:  #remove header and total rows
                self.logger.info('processing operation...')

                if (dt.now().minute
                        in [0, 30]):  #log parsed html every 30 minutes
                    file_name = str(dt.now())
                    f = open(
                        f"{BASE_DIR}/parsed/{system_id}-{file_name}-OPENED.html",
                        "w+")
                    f.write(response.text)
                    f.close()

                try:
                    openDateRaw = operation.xpath(
                        ('td[not(@style="display:none")][%i]//text()' %
                         self.get_column_index_by_name(
                             column_names,
                             'Open Date'))).extract_first().strip()
                    symbolRaw = operation.xpath(
                        ('td[not(@style="display:none")][%i]//span//text()' %
                         self.get_column_index_by_name(
                             column_names, 'Symbol'))).extract_first()
                    actionRaw = operation.xpath(
                        ('td[not(@style="display:none")][%i]//text()' %
                         self.get_column_index_by_name(
                             column_names, 'Action'))).extract_first().strip()
                    lotsRaw = 5
                    openPriceRaw = operation.xpath(
                        ('td[not(@style="display:none")][%i]//text()' %
                         self.get_column_index_by_name(
                             column_names,
                             'Open Price'))).extract_first().strip()

                    if not openPriceRaw.strip():
                        openPriceRaw = 0

                    self.logger.info(
                        'openDate: %s, symbolRaw: %s, actionRaw: %s, lotsRaw: %s, openPriceRaw: %s',
                        openDateRaw, symbolRaw, actionRaw, lotsRaw,
                        openPriceRaw)

                    item = Operation()
                    str_hash = str(openDateRaw) + str(symbolRaw) + str(
                        actionRaw) + str(lotsRaw) + str(
                            openPriceRaw) + system_id

                    item['hashid'] = self.get_hash(str_hash)
                    item['timeOrderDiscovered'] = time.time()
                    item['timeTradeOpened'] = None
                    item['timeTradeClosed'] = None
                    item['timeTradeAttempedOpened'] = None
                    item['symbol'] = symbolRaw
                    item['action'] = actionRaw
                    item['lots'] = 1  #TODO fix this
                    item['openPrice'] = float(openPriceRaw)
                    item['closePrice'] = None
                    item['url'] = response.url
                    item['profit'] = None
                    item['orderid'] = None
                    item['tradeid'] = None

                    self.logger.info('Position parsing completed')

                    yield item
                except:
                    self.logger.error('Exception parsing position')
Ejemplo n.º 4
0
class AppContainer(object):

    def __init__(self, config):
        self.config = config
        self.store = PersistentStore(self.config['dbfilename'])


        self.spotify_session = SpotifyAsyncSessionManager(useragent="SpotifyThings")
        self.spotify_session.player_state_changed += self.__player_state_changed
        self.spotify_session.current_track_changed += self.__current_track_changed
        self.spotify_session.playback_progress += self.__playback_progress_event
        self.spotify_session.playqueue_changed += self.__playqueue_changed_event
        self.spotify_session.login_state_changed += self.__login_state_changed_event
        self.spotify_session.password_blob_changed += self.__login_password_blob_changed

        self.wsregistry = WebSocketRegistry();
        self.wsregistry.on_new_client += self.__on_new_client

        self.webapp = WebApplication(self)
        self.asyncioloop = AsyncIOLoop()

        self.attached_rfid_readers = dict()

        self.rfid = RFID()
        self.rfid.setOnAttachHandler(self.rfidAttached)
        self.rfid.setOnDetachHandler(self.rfidDetached)
        self.rfid.setOnErrorhandler(self.rfidError)
        self.rfid.setOnTagHandler(self.rfidTagGained)
        self.rfid.setOnTagLostHandler(self.rfidTagLost)



    # player events
    def __player_state_changed(self, state):
        self.wsregistry.broadcast('app', PlayerStateMessage(state))

    def __current_track_changed(self, track):
        self.wsregistry.broadcast('app', PlayerCurrentTrackMessage(track))

    def __playback_progress_event(self, playback_time):
        self.wsregistry.broadcast('app', PlayerPlaybackProgressMessage(int(playback_time)))

    def __playqueue_changed_event(self):
        self.wsregistry.broadcast('app', PlayerQueueModifiedMessage())

    # app event
    def __login_state_changed_event(self, is_logged_in, login_error, current_user):
        self.wsregistry.broadcast('app', SpotifyLoginStateMessage(is_logged_in, login_error, current_user))

    def __login_password_blob_changed(self, username, password_blob):
        self.store.set_config('spotify.username', username)
        self.store.set_config('spotify.password_blob', password_blob)

        self.wsregistry.broadcast('app', SpotifyCredentialStoredMessage(self.store.get_config('spotify.password_blob') is not None))

    # websocket events
    def __on_new_client(self, wstype, ws):
        def spotify_state_handler(current_track, track_playback_time, player_state):
            # send player state
            ws.write_message(PlayerStateMessage(player_state))

            # send current track
            ws.write_message(PlayerCurrentTrackMessage(current_track))

            # send playback progress
            ws.write_message(PlayerPlaybackProgressMessage(int(track_playback_time)))

        def spotify_login_state_handler(is_logged_in, login_error, current_user):
            # send login state message
            ws.write_message(SpotifyLoginStateMessage(is_logged_in, login_error, current_user))

        if wstype == "app":
            # request spotify state
            self.spotify_session.get_player_state(spotify_state_handler)

            # request login state
            self.spotify_session.get_login_state(spotify_login_state_handler)

            # send message with currently attched rfid readers
            ws.write_message(RfidReaderUpdatedMessage(self.attached_rfid_readers))

            # send message with credentials stored status
            ws.write_message(SpotifyCredentialStoredMessage(self.store.get_config('spotify.password_blob') is not None))


    # rfid events
    def rfidAttached(self, e):
        device = e.device
        device.setAntennaOn(True)

        # add to list of attached readers
        reader = RfidReader(
            device.getDeviceName(),
            device.getDeviceVersion(),
            device.getSerialNum())

        self.attached_rfid_readers[reader.serial_no] = reader

        # notify clients
        self.wsregistry.broadcast('app', RfidReaderUpdatedMessage(self.attached_rfid_readers))

        logger.info('RFID reader %s Attached', device.getSerialNum())


    def rfidDetached(self, e):
        device = e.device
        serial_no = device.getSerialNum()

        # remove from attached list
        del self.attached_rfid_readers[serial_no]

        # notify clients
        self.wsregistry.broadcast('app', RfidReaderUpdatedMessage(self.attached_rfid_readers))

        logger.info('RFID reader %s Deteched', device.getSerialNum())


    def rfidError(self, e):
        try:
            source = e.device

            logger.error("RFID %i: Phidget Error %i: %s", source.getSerialNum(), e.eCode, e.description)
        except PhidgetException as e:
            logger.exception('Phidget Exception %i: %s', e.code, e.details)

    def rfidTagGained(self, e):
        source = e.device
        self.rfid.setLEDOn(1)
        logger.debug('RFID %i: Tag Read: %s', source.getSerialNum(), e.tag)


        # get existing mapping, if any
        tagmapping = self.store.find_by_tagid(e.tag)

        # broadcast to any listeners
        any_listeners = self.wsregistry.broadcast('rfid', RfidTagReadMessage(e.tag, tagmapping))

        # no listeners and resource was mapped, play resource
        if not any_listeners and tagmapping:
            self.spotify_session.play_link(tagmapping.spotifylink, None)

    def rfidTagLost(self, e):
        source = e.device
        self.rfid.setLEDOn(0)
        logger.debug('RFID %i: Tag Lost: %s', source.getSerialNum(), e.tag)


    def forget_spotify_credentials(self):
        # delete from store
        self.store.delete_configs(['spotify.username', 'spotify.password_blob'])

        # notify clients
        self.wsregistry.broadcast('app', SpotifyCredentialStoredMessage(self.store.get_config('spotify.password_blob') is not None))



    def start(self):
        logger.info('Starting appcontainer')

        # initialize store
        logger.info('Initializing store')
        self.store.initialize()

        # start spotify
        logger.info('Starting spotify session')
        self.spotify_session.start()

        # login using stored credentials
        username = self.store.get_config('spotify.username')
        password_blob = self.store.get_config('spotify.password_blob')

        if username and password_blob:
            logger.info('Logging in with username %s', username)
            self.spotify_session.login(username, None, False, password_blob)

        # start webapp
        listen_port = self.config['listen_port']
        logger.info('Starting web application, listening on port %s', listen_port)
        self.webapp.listen(listen_port)
        self.asyncioloop.start()

        # start rfid
        logger.info('Initializing RFID reader')
        self.rfid.openPhidget()

        logger.info('Appcontainer started')

    def stop(self):
        logger.info('Stopping appcontainer')

        # stop rfid
        logger.info('Closing RFID reader')
        self.rfid.closePhidget()

        # stop webapp
        logger.info('Stopping web application')
        self.asyncioloop.stop()

        # stop spotify
        logger.info('Stopping Spotify session')
        self.spotify_session.stop()

        logger.info('Appcontainer stopped')
Ejemplo n.º 5
0
 def process_item(self, item, spider):
     persistentstore = PersistentStore()
     persistentstore.add_parsed_item(item)
     return item