Exemple #1
0
    def _saveSecurityNews(self, headlineRows, xigniteSecurity):
        """ Store security news in the destination schema specified via
    _NEWS_SCHEMA member variable.

    :param headlineRows: rows of field values for target security news table
    :type headlineRows: sequence of dicts
    :param dict xigniteSecurity: Security info from xignite API results (e.g.,
      global security news, security bars, etc.)

    :returns: The count of new news rows that were saved; 0 if the news object
      has no headlines.
    """
        destSchema = self._NEWS_SCHEMA

        if not headlineRows:
            return 0

        if self.dryRun:
            g_log.info("%r.process(dryRun=True): security=%s, news=%s", self,
                       xigniteSecurity, headlineRows)
            return 0

        engine = collectorsdb.engineFactory()

        @collectorsdb.retryOnTransientErrors
        def saveNews():
            with engine.begin() as conn:
                # Save headlines
                newsIns = destSchema.insert().prefix_with("IGNORE",
                                                          dialect="mysql")
                return conn.execute(newsIns, headlineRows).rowcount

        try:
            return saveNews()
        except sql.exc.IntegrityError:
            # Most likely foreign key constraint violation against the
            # xignite_security table
            g_log.info("Inserting security row for symbol=%s",
                       xigniteSecurity["Symbol"])
            xignite_agent_utils.insertSecurity(engine, xigniteSecurity)

            # Re-insert news after resolving IntegrityError
            return saveNews()
    def addSecurity(symbol):
      self.addCleanup(_deleteSecurity, symbol)
      xignite_agent_utils.insertSecurity(
        engine=collectorsdb.engineFactory(),
        xigniteSecurity={
          "Symbol": symbol,
          "CIK": "CIK",
          "CUSIP": "CUSIP",
          "ISIN": "ISIN",
          "Valoren": "Valoren",
          "Name": "{sym} Inc.".format(sym=symbol),
          "Market": "Market",
          "MarketIdentificationCode": "mic1",
          "MostLiquidExchange": True,
          "CategoryOrIndustry": "CategoryOrIndustry"
        })

      self.assertTrue(securityExists(symbol),
                      "inserted {symbol} not found".format(symbol=symbol))
  def _saveSecurityNews(self, headlineRows, xigniteSecurity):
    """ Store security news in the destination schema specified via
    _NEWS_SCHEMA member variable.

    :param headlineRows: rows of field values for target security news table
    :type headlineRows: sequence of dicts
    :param dict xigniteSecurity: Security info from xignite API results (e.g.,
      global security news, security bars, etc.)

    :returns: The count of new news rows that were saved; 0 if the news object
      has no headlines.
    """
    destSchema = self._NEWS_SCHEMA

    if not headlineRows:
      return 0

    if self.dryRun:
      g_log.info("%r.process(dryRun=True): security=%s, news=%s", self,
                 xigniteSecurity, headlineRows)
      return 0

    engine = collectorsdb.engineFactory()

    @collectorsdb.retryOnTransientErrors
    def saveNews():
      with engine.begin() as conn:
        # Save headlines
        newsIns = destSchema.insert().prefix_with("IGNORE", dialect="mysql")
        return conn.execute(newsIns, headlineRows).rowcount

    try:
      return saveNews()
    except sql.exc.IntegrityError:
      # Most likely foreign key constraint violation against the
      # xignite_security table
      g_log.info("Inserting security row for symbol=%s",
                 xigniteSecurity["Symbol"])
      xignite_agent_utils.insertSecurity(engine, xigniteSecurity)

      # Re-insert news after resolving IntegrityError
      return saveNews()
        def addSecurity(symbol):
            self.addCleanup(_deleteSecurity, symbol)
            xignite_agent_utils.insertSecurity(
                engine=collectorsdb.engineFactory(),
                xigniteSecurity={
                    "Symbol": symbol,
                    "CIK": "CIK",
                    "CUSIP": "CUSIP",
                    "ISIN": "ISIN",
                    "Valoren": "Valoren",
                    "Name": "{sym} Inc.".format(sym=symbol),
                    "Market": "Market",
                    "MarketIdentificationCode": "mic1",
                    "MostLiquidExchange": True,
                    "CategoryOrIndustry": "CategoryOrIndustry"
                })

            self.assertTrue(
                securityExists(symbol),
                "inserted {symbol} not found".format(symbol=symbol))
def forward(metricSpecs, data, security, server=DEFAULT_SERVER,
            port=DEFAULT_PORT,
            dryrun=DEFAULT_DRYRUN):
  """ Forward stock data to Grok/Taurus instance via custom metric

  :param metricSpecs: Sequence of one or more StockMetricSpec objects associated
    with the same stock symbol for which polling was conducted
  :param list data: List of sample dicts
  :param dict security: Details of security from XIgnite API
  """
  try:
    symbol = security["Symbol"]

    engine = collectorsdb.engineFactory()

    lastSample = _getLatestSample(engine, symbol)
    if lastSample:
      localizedLastEndTime = (
        getEasternLocalizedEndTimestampFromSampleRow(lastSample))
    else:
      localizedLastEndTime = None

    # Implemented in two phases:
    #
    # 1. Buffer records to collectorsdb

    for sample in data:
      localizedSampleStartTime = (
        getEasternLocalizedTimestampFromSample(sample["StartDate"],
                                               sample["StartTime"],
                                               sample["UTCOffset"]))

      if localizedSampleStartTime.time() < NAIVE_MARKET_OPEN_TIME:
        # Ignore samples that preceed market open
        _LOG.info("Skipping data before market hours: %s @ %s sample=%s",
                  symbol, localizedSampleStartTime, sample)
        continue

      if localizedSampleStartTime.time() >= NAIVE_MARKET_CLOSE_TIME:
        # Ignore a quirk of the xignite API that duplicates some data at
        # end of trading day. This also excludes the closing auction on
        # NYSE.
        _LOG.info("Skipping data after market hours: %s @ %s sample=%s",
                  symbol, localizedSampleStartTime, sample)
        continue

      if not lastSample or (localizedSampleStartTime >= localizedLastEndTime):
        # Current sample starts at, or after last recorded timestamp ends
        localizedSampleEndTime = (
          getEasternLocalizedTimestampFromSample(sample["EndDate"],
                                                 sample["EndTime"],
                                                 sample["UTCOffset"]))

        ins = (xigniteSecurityBars
               .insert()
               .values(symbol=symbol,
                       StartDate=localizedSampleStartTime.date(),
                       StartTime=localizedSampleStartTime.time(),
                       EndDate=localizedSampleEndTime.date(),
                       EndTime=localizedSampleEndTime.time(),
                       UTCOffset=sample["UTCOffset"],
                       Open=sample["Open"],
                       High=sample["High"],
                       Low=sample["Low"],
                       Close=sample["Close"],
                       Volume=sample["Volume"],
                       Trades=sample["Trades"]))

        @collectorsdb.retryOnTransientErrors
        def _insertBar():
          engine.execute(ins)

        try:
          _insertBar()
        except IntegrityError:
          # Most likely foreign key constraint violation against the
          # xignite_security table
          _LOG.info("Inserting security row for symbol=%s", symbol)
          xignite_agent_utils.insertSecurity(engine, security)

          # Re-insert after resolving IntegrityError
          _insertBar()

    #  2. If in active mode, send ALL un-sent records to Taurus

    if g_opMode != ApplicationConfig.OP_MODE_ACTIVE:
      return

    transmitMetricData(metricSpecs=metricSpecs, symbol=symbol, engine=engine)

  except Exception:
    _LOG.exception("forward failed for metricSpecs=%s", metricSpecs)
    raise
Exemple #6
0
def forward(metricSpecs,
            data,
            security,
            server=DEFAULT_SERVER,
            port=DEFAULT_PORT,
            dryrun=DEFAULT_DRYRUN):
    """ Forward stock data to Grok/Taurus instance via custom metric

  :param metricSpecs: Sequence of one or more StockMetricSpec objects associated
    with the same stock symbol for which polling was conducted
  :param list data: List of sample dicts
  :param dict security: Details of security from XIgnite API
  """
    try:
        symbol = security["Symbol"]

        engine = collectorsdb.engineFactory()

        lastSample = _getLatestSample(engine, symbol)
        if lastSample:
            localizedLastEndTime = (
                getEasternLocalizedEndTimestampFromSampleRow(lastSample))
        else:
            localizedLastEndTime = None

        # Implemented in two phases:
        #
        # 1. Buffer records to collectorsdb

        for sample in data:
            localizedSampleStartTime = (getEasternLocalizedTimestampFromSample(
                sample["StartDate"], sample["StartTime"], sample["UTCOffset"]))

            if localizedSampleStartTime.time() < NAIVE_MARKET_OPEN_TIME:
                # Ignore samples that preceed market open
                _LOG.info(
                    "Skipping data before market hours: %s @ %s sample=%s",
                    symbol, localizedSampleStartTime, sample)
                continue

            if localizedSampleStartTime.time() >= NAIVE_MARKET_CLOSE_TIME:
                # Ignore a quirk of the xignite API that duplicates some data at
                # end of trading day. This also excludes the closing auction on
                # NYSE.
                _LOG.info(
                    "Skipping data after market hours: %s @ %s sample=%s",
                    symbol, localizedSampleStartTime, sample)
                continue

            if not lastSample or (localizedSampleStartTime >=
                                  localizedLastEndTime):
                # Current sample starts at, or after last recorded timestamp ends
                localizedSampleEndTime = (
                    getEasternLocalizedTimestampFromSample(
                        sample["EndDate"], sample["EndTime"],
                        sample["UTCOffset"]))

                ins = (xigniteSecurityBars.insert().values(
                    symbol=symbol,
                    StartDate=localizedSampleStartTime.date(),
                    StartTime=localizedSampleStartTime.time(),
                    EndDate=localizedSampleEndTime.date(),
                    EndTime=localizedSampleEndTime.time(),
                    UTCOffset=sample["UTCOffset"],
                    Open=sample["Open"],
                    High=sample["High"],
                    Low=sample["Low"],
                    Close=sample["Close"],
                    Volume=sample["Volume"],
                    Trades=sample["Trades"]))

                @collectorsdb.retryOnTransientErrors
                def _insertBar():
                    engine.execute(ins)

                try:
                    _insertBar()
                except IntegrityError:
                    # Most likely foreign key constraint violation against the
                    # xignite_security table
                    _LOG.info("Inserting security row for symbol=%s", symbol)
                    xignite_agent_utils.insertSecurity(engine, security)

                    # Re-insert after resolving IntegrityError
                    _insertBar()

        #  2. If in active mode, send ALL un-sent records to Taurus

        if g_opMode != ApplicationConfig.OP_MODE_ACTIVE:
            return

        transmitMetricData(metricSpecs=metricSpecs,
                           symbol=symbol,
                           engine=engine)

    except Exception:
        _LOG.exception("forward failed for metricSpecs=%s", metricSpecs)
        raise