Example #1
0
 def get_all_new_pastes(self):
     """
     Get all new pastes and set the last crawl time to now
     """
     page = 1
     oldest_timestamp = None
     new_pastes = []
     while oldest_timestamp is None or oldest_timestamp > self.__last_crawl:
         try:
             current_pastes = self._list_pastes(page)
         # Page does not exist
         except KeyError:
             break
         for paste_id in current_pastes:
             paste_json = self._get_paste(paste_id)
             if oldest_timestamp is None or \
                     Arrow.fromtimestamp(paste_json["timestamp"]).replace(tzinfo="UTC") < oldest_timestamp:
                 oldest_timestamp = Arrow.fromtimestamp(
                     paste_json["timestamp"])
             if oldest_timestamp > self.__last_crawl:
                 new_pastes.append(paste_json)
         page += 1
     self._logger.info("Found %d new pastes since %s" %
                       (len(new_pastes), self.__last_crawl.for_json()))
     self.__set_last_crawl()
     return new_pastes
Example #2
0
    def set_name(self, site: str = ''):
        self.site = site.lower()

        set1 = self.dbdata.find({
            'site': site,
            't0': {
                '$gt': self.start_timestamp()
            }
        })
        qtrhrs = set1.distinct('qtrhr')

        result = []
        for qh in sorted(qtrhrs):
            #
            qset = self.dbdata.find({'site': site, 'qtrhr': qh})
            detail = []
            for row in qset:
                reading = self.data_obj(row)
                detail.append(reading)

            minmax = self.determine_max_min(detail)

            qtrhour = {
                'ts': Arrow.fromtimestamp(timestamp=qh).for_json(),
                'dt': Arrow.fromtimestamp(timestamp=qh),
                'min': minmax['min'],
                'max': minmax['max'],
                'avg': minmax['avg'],
            }
            result.append(qtrhour)

        #
        # Prepare for plot
        #
        dev_x = []
        dev_y = []
        labels = []
        for r in result:
            time = r['dt'].strftime('%m/%d %H:%M')
            dev_x.append(time)
            labels.append(time)
            dev_y.append(r['avg'])

        n = labels.__len__()
        i = n - 1
        div_factor = math.floor(n / 10)
        while i > 0:
            if i % div_factor != 0:
                labels[i] = ''
            i = i - 1

        result = {'site': self.site, 'x': dev_x, 'y': dev_y, 'labels': labels}
        self.data = result
def user_action_stat(interval,user_out_name,song_out_name):
    #统计用户三日的数据量
    ## step 1: 读入数据
    base_time = 1426348800 #3-15-0-0-0的unix时间戳
    base_time_stamp = Arrow.fromtimestamp(base_time)
    interval_seconds = interval * 24 * 3600
    parts = load_csv_as_dict('%s/data_source/%s' %(PROJECT_PATH,'mars_tianchi_user_actions.csv'))
    user_dict = defaultdict(
        lambda: defaultdict(lambda: defaultdict(lambda: 0.0)))  # 默认dict的一个trick!
    song_dict = defaultdict(lambda:defaultdict(lambda:defaultdict(lambda:0.0)))
    count = 0
    ## step 2:统计数据
    for part in parts:
        user_id = part['user_id']
        song_id = part['song_id']
        action_type = part['action_type']
        gmt_create = eval(part['gmt_create'])
        date_interval_belong = int((Arrow.fromtimestamp(gmt_create) - base_time_stamp).total_seconds())/interval_seconds
        user_dict[user_id][date_interval_belong][action_type] += 1
        song_dict[song_id][date_interval_belong][action_type] += 1
        count += 1
        if count % 1000 == 0:
            print 'statistical %s records' % count
    print 'total users: %s' % len(user_dict)
    print 'total songs: %s' % len(song_dict)
    ## step 3:写入到feature文件
    fs = open('%s/feature/%s.csv' % (PROJECT_PATH,user_out_name),'w')
    fs.write('user_id,date_interval_%s ,plays,downloads,favors\n' % interval)
    count = 0
    for user in user_dict:
        date_dict = user_dict[user]
        for date in date_dict:
            action = date_dict[date]
            fs.write('%s,%s,%s,%s,%s\n' % (user,date,action['1'],action['2'],action['3']))
            count = count + 1
            if count % 1000 == 0:
                print 'write %s length' % count
    fs.close()
    print 'user_dict is write done'
    fs = open('%s/feature/%s.csv' % (PROJECT_PATH,song_out_name),'w')
    fs.write('song_id,date_interval_%s,plays,downloads,favors\n' % interval)
    count = 0
    for song in song_dict:
        date_dict = song_dict[song]
        for date in date_dict:
            action = date_dict[date]
            fs.write('%s,%s,%s,%s,%s\n' % (song,date,action['1'],action['2'],action['3']))
            count += 1
            if count % 1000 == 0:
                print 'write %s length' % count
    fs.close()
    print 'song_dict is write done'
    def dump(self, record, data):
        """Dump the data."""
        try:
            parent_data = dict_lookup(data, self.keys, parent=True)

            pd = parse_edtf(parent_data[self.key])
            parent_data[f"{self.key}_start"] = Arrow.fromtimestamp(
                calendar.timegm(pd.lower_strict()),
                tzinfo=utc).date().isoformat()
            parent_data[f"{self.key}_end"] = Arrow.fromtimestamp(
                calendar.timegm(pd.upper_strict()),
                tzinfo=utc).date().isoformat()
        except (KeyError, EDTFParseException):
            # The field does not exists or had wrong data
            return data  # FIXME: should log this in debug mode?
Example #5
0
 def evaluate_ct(self):
     try:
         print("Evaluating confirmation time (ECT), pre-notification at " +
               str(self.pre_latch_notif) + " out of " +
               str(self.data.confirmation_updates))
         last_holdoff = self.ct.holdoff
         # reset the timer if no notifications received for a while
         if (self.pre_latch_notif >= self.data.confirmation_updates):
             print("ECT: Reset confirmation time.")
             self.ct.time = 2
             self.ct.holdoff = False
         result = self.ct.update(self.latched_value)
         self.pre_latch_notif += 1
         if self.ct.holdoff:
             announcement = str(
                 (self.ct.delay - self.ct.time) *
                 self.data.confirmation_interval
             ) + " until " + self.data.enumeration[self.latched_value]
             print("ECT: in holdoff, publishing time announcement: " +
                   announcement)
             self.publish("tweeter/time_announce", announcement)
         if last_holdoff and not self.ct.holdoff:
             announcement = "is " + self.data.enumeration[result[1]]
             print("ECT: new value or value restored: " + announcement)
             self.publish("tweeter/time_announce", announcement)
         if result[0]:
             attime = arrow.fromtimestamp(time.time()).to(
                 self.data.timezone).strftime(self.data.time_format)
             self.tweet(
                 self.data.message.format(self.data.enumeration[result[1]],
                                          attime))
     except:
         traceback.print_ext()
         os._exit(1)
Example #6
0
    async def from_dataset(cls, client: "ProtosBot",
                           ds: "AutoPurgeJob") -> "Job":
        """
        Create a new job from a dataset.
        """

        channel = await client.fetch_channel(ds.channel_id)

        return Job(client, channel, ds.pattern, Arrow.fromtimestamp(ds.last))
Example #7
0
def sources():
    sources = get_collection('sources')
    limit = request.args.get('limit', 50)
    skip = request.args.get('skip', 0)
    skip = skip if skip != 0 else request.args.get(
        'offset', 0)  # allow another namespace 'offset'
    keyword = request.args.get('keyword', None)
    start_urls_keyword = request.args.get('start_urls_keyword', None)
    comment_keyword = request.args.get('comment_keyword', None)
    startwith = request.args.get('startwith', None)
    coop = request.args.get('coop', None)

    sql_filter = {}

    if keyword:
        sql_filter.update({"title": {'$regex': keyword, '$options': 'i'}})

    if start_urls_keyword:
        sql_filter.update(
            {"start_urls": {
                '$regex': start_urls_keyword,
                '$options': 'i'
            }})

    if comment_keyword:
        sql_filter.update(
            {"comments": {
                '$regex': comment_keyword,
                '$options': 'i'
            }})

    if startwith:
        sql_filter.update({'_id': {'$lte': ObjectId(startwith)}})

    if coop:
        sql_filter.update({'coops': coop})

    data = []
    data.extend([
        clean_id(i)
        for i in sources.find(sql_filter).sort('machine_name', 1).skip(
            int(skip)).limit(int(limit))
    ])

    for d in data:
        if d.get('latestArticleCreated'):
            last_time = Arrow.fromtimestamp(
                d['latestArticleCreated']).to('Asia/Shanghai')
            d['latestArticleCreatedHuman'] = u'最后时间:{0}'.format(
                last_time.humanize(locale='zh_cn'))
            if (Arrow.utcnow() - last_time).days > 1:
                d['latestArticleCreatedHuman'] = u'<span class="text-danger">{}</span>'.format(
                    d['latestArticleCreatedHuman'])

    ret = {'data': data, 'total': sources.find().count()}
    return jsonify(ret)
Example #8
0
def validate_token(token):
    if not isinstance(token, str):
        raise TypeError('token은 반드시 `str` 이여야합니다.')
    secret_key = get_secret_key()
    s = JSONWebSignatureSerializer(secret_key)
    try:
        data = s.loads(token.encode('utf-8'))
    except BadSignature as e:
        raise InvalidTokenError('잘못된 token입니다.')
    expired_at = Arrow.fromtimestamp(data['expired_at'])
    now = utcnow()
    if expired_at < now:
        raise ExpiredTokenError('만료된 token입니다.')
    return data
Example #9
0
 def _reformat_json(self, dict_to_write):
     """
     In-place reformatting method for a consistent format between objects.
     :param dict dict_to_write: Formats all JSONs to write out
     :return: None, changes the JSON in place
     """
     self._logger.debug("Reformatting JSON: %s" % str(dict_to_write))
     for key in dict_to_write:
         if key == "timestamp":
             arrow_date = Arrow.fromtimestamp(
                 dict_to_write[key]).replace(tzinfo=self.timezone)
             val = arrow_date.for_json()
         elif key == "author":
             author = dict_to_write[key]
             val = author if str(author).lower(
             ) not in TinyWriter._unknown_authors else "Unknown"
         else:
             val = str(dict_to_write[key]).strip() if self.should_strip and type(dict_to_write[key]) == str\
                 else dict_to_write[key]
         if type(val) == str:
             val.replace("\r\n", "\n")
         dict_to_write[key] = val
Example #10
0
def names():
    items = get_collection('items')

    if request.method == 'GET':
        name_list = items.distinct('name')
        current = []
        if os.path.exists('filter.json'):
            with open('filter.json', 'r') as f:
                current = json.load(f)
        msgs = {}
        for name in name_list:
            item = [
                i for i in items.find({
                    'name': name
                }, {
                    '_id': 0
                }).sort([('ctime', -1)]).limit(1)
            ]
            if item:
                item = item[0]
                last_time = Arrow.fromtimestamp(
                    item['ctime']).to('Asia/Shanghai')
                msgs[name] = u'最后时间:{0}'.format(
                    last_time.humanize(locale='zh_cn'))
                if (Arrow.utcnow() - last_time).days > 1:
                    msgs[name] = u'<span class="text-danger">{}</span>'.format(
                        msgs[name])
        return jsonp({'all': name_list, 'current': current, 'msg': msgs})
    elif request.method == 'POST':
        code = 500
        if request.json:
            with open('filter.json', 'w') as f:
                f.write(
                    json.dumps(request.json,
                               ensure_ascii=False).encode('utf8'))
                code = 200
        return Response('', code)
def test_generate_backtest_stats(default_conf, testdatadir):
    default_conf.update({'strategy': 'DefaultStrategy'})
    StrategyResolver.load_strategy(default_conf)

    results = {
        'DefStrat': {
            'results':
            pd.DataFrame({
                "pair": [
                    "UNITTEST/BTC", "UNITTEST/BTC", "UNITTEST/BTC",
                    "UNITTEST/BTC"
                ],
                "profit_ratio": [0.003312, 0.010801, 0.013803, 0.002780],
                "profit_abs": [0.000003, 0.000011, 0.000014, 0.000003],
                "open_date": [
                    Arrow(2017, 11, 14, 19, 32, 00).datetime,
                    Arrow(2017, 11, 14, 21, 36, 00).datetime,
                    Arrow(2017, 11, 14, 22, 12, 00).datetime,
                    Arrow(2017, 11, 14, 22, 44, 00).datetime
                ],
                "close_date": [
                    Arrow(2017, 11, 14, 21, 35, 00).datetime,
                    Arrow(2017, 11, 14, 22, 10, 00).datetime,
                    Arrow(2017, 11, 14, 22, 43, 00).datetime,
                    Arrow(2017, 11, 14, 22, 58, 00).datetime
                ],
                "open_rate": [0.002543, 0.003003, 0.003089, 0.003214],
                "close_rate": [0.002546, 0.003014, 0.003103, 0.003217],
                "trade_duration": [123, 34, 31, 14],
                "is_open": [False, False, False, True],
                "sell_reason": [
                    SellType.ROI, SellType.STOP_LOSS, SellType.ROI,
                    SellType.FORCE_SELL
                ]
            }),
            'config':
            default_conf,
            'locks': [],
            'backtest_start_time':
            Arrow.utcnow().int_timestamp,
            'backtest_end_time':
            Arrow.utcnow().int_timestamp,
        }
    }
    timerange = TimeRange.parse_timerange('1510688220-1510700340')
    min_date = Arrow.fromtimestamp(1510688220)
    max_date = Arrow.fromtimestamp(1510700340)
    btdata = history.load_data(testdatadir,
                               '1m', ['UNITTEST/BTC'],
                               timerange=timerange,
                               fill_up_missing=True)

    stats = generate_backtest_stats(btdata, results, min_date, max_date)
    assert isinstance(stats, dict)
    assert 'strategy' in stats
    assert 'DefStrat' in stats['strategy']
    assert 'strategy_comparison' in stats
    strat_stats = stats['strategy']['DefStrat']
    assert strat_stats['backtest_start'] == min_date.datetime
    assert strat_stats['backtest_end'] == max_date.datetime
    assert strat_stats['total_trades'] == len(results['DefStrat']['results'])
    # Above sample had no loosing trade
    assert strat_stats['max_drawdown'] == 0.0

    results = {
        'DefStrat': {
            'results':
            pd.DataFrame({
                "pair": [
                    "UNITTEST/BTC", "UNITTEST/BTC", "UNITTEST/BTC",
                    "UNITTEST/BTC"
                ],
                "profit_ratio": [0.003312, 0.010801, -0.013803, 0.002780],
                "profit_abs": [0.000003, 0.000011, -0.000014, 0.000003],
                "open_date": [
                    Arrow(2017, 11, 14, 19, 32, 00).datetime,
                    Arrow(2017, 11, 14, 21, 36, 00).datetime,
                    Arrow(2017, 11, 14, 22, 12, 00).datetime,
                    Arrow(2017, 11, 14, 22, 44, 00).datetime
                ],
                "close_date": [
                    Arrow(2017, 11, 14, 21, 35, 00).datetime,
                    Arrow(2017, 11, 14, 22, 10, 00).datetime,
                    Arrow(2017, 11, 14, 22, 43, 00).datetime,
                    Arrow(2017, 11, 14, 22, 58, 00).datetime
                ],
                "open_rate": [0.002543, 0.003003, 0.003089, 0.003214],
                "close_rate": [0.002546, 0.003014, 0.0032903, 0.003217],
                "trade_duration": [123, 34, 31, 14],
                "open_at_end": [False, False, False, True],
                "sell_reason": [
                    SellType.ROI, SellType.STOP_LOSS, SellType.ROI,
                    SellType.FORCE_SELL
                ]
            }),
            'config':
            default_conf
        }
    }

    assert strat_stats['max_drawdown'] == 0.0
    assert strat_stats['drawdown_start'] == datetime(1970,
                                                     1,
                                                     1,
                                                     tzinfo=timezone.utc)
    assert strat_stats['drawdown_end'] == datetime(1970,
                                                   1,
                                                   1,
                                                   tzinfo=timezone.utc)
    assert strat_stats['drawdown_end_ts'] == 0
    assert strat_stats['drawdown_start_ts'] == 0
    assert strat_stats['pairlist'] == ['UNITTEST/BTC']

    # Test storing stats
    filename = Path(testdatadir / 'btresult.json')
    filename_last = Path(testdatadir / LAST_BT_RESULT_FN)
    _backup_file(filename_last, copy_file=True)
    assert not filename.is_file()

    store_backtest_stats(filename, stats)

    # get real Filename (it's btresult-<date>.json)
    last_fn = get_latest_backtest_filename(filename_last.parent)
    assert re.match(r"btresult-.*\.json", last_fn)

    filename1 = (testdatadir / last_fn)
    assert filename1.is_file()
    content = filename1.read_text()
    assert 'max_drawdown' in content
    assert 'strategy' in content
    assert 'pairlist' in content

    assert filename_last.is_file()

    _clean_test_file(filename_last)
    filename1.unlink()
Example #12
0
def _format_date(date):
    """Format the given date into ISO format."""
    arrow = Arrow.fromtimestamp(calendar.timegm(date), tzinfo=utc)
    return arrow.date().isoformat()
Example #13
0
def test_generate_backtest_stats(default_conf, testdatadir, tmpdir):
    default_conf.update({'strategy': CURRENT_TEST_STRATEGY})
    StrategyResolver.load_strategy(default_conf)

    results = {
        'DefStrat': {
            'results':
            pd.DataFrame({
                "pair": [
                    "UNITTEST/BTC", "UNITTEST/BTC", "UNITTEST/BTC",
                    "UNITTEST/BTC"
                ],
                "profit_ratio": [0.003312, 0.010801, 0.013803, 0.002780],
                "profit_abs": [0.000003, 0.000011, 0.000014, 0.000003],
                "open_date": [
                    Arrow(2017, 11, 14, 19, 32, 00).datetime,
                    Arrow(2017, 11, 14, 21, 36, 00).datetime,
                    Arrow(2017, 11, 14, 22, 12, 00).datetime,
                    Arrow(2017, 11, 14, 22, 44, 00).datetime
                ],
                "close_date": [
                    Arrow(2017, 11, 14, 21, 35, 00).datetime,
                    Arrow(2017, 11, 14, 22, 10, 00).datetime,
                    Arrow(2017, 11, 14, 22, 43, 00).datetime,
                    Arrow(2017, 11, 14, 22, 58, 00).datetime
                ],
                "open_rate": [0.002543, 0.003003, 0.003089, 0.003214],
                "close_rate": [0.002546, 0.003014, 0.003103, 0.003217],
                "trade_duration": [123, 34, 31, 14],
                "is_open": [False, False, False, True],
                "is_short": [False, False, False, False],
                "stake_amount": [0.01, 0.01, 0.01, 0.01],
                "exit_reason": [
                    ExitType.ROI, ExitType.STOP_LOSS, ExitType.ROI,
                    ExitType.FORCE_EXIT
                ]
            }),
            'config':
            default_conf,
            'locks': [],
            'final_balance':
            1000.02,
            'rejected_signals':
            20,
            'timedout_entry_orders':
            0,
            'timedout_exit_orders':
            0,
            'backtest_start_time':
            Arrow.utcnow().int_timestamp,
            'backtest_end_time':
            Arrow.utcnow().int_timestamp,
            'run_id':
            '123',
        }
    }
    timerange = TimeRange.parse_timerange('1510688220-1510700340')
    min_date = Arrow.fromtimestamp(1510688220)
    max_date = Arrow.fromtimestamp(1510700340)
    btdata = history.load_data(testdatadir,
                               '1m', ['UNITTEST/BTC'],
                               timerange=timerange,
                               fill_up_missing=True)

    stats = generate_backtest_stats(btdata, results, min_date, max_date)
    assert isinstance(stats, dict)
    assert 'strategy' in stats
    assert 'DefStrat' in stats['strategy']
    assert 'strategy_comparison' in stats
    strat_stats = stats['strategy']['DefStrat']
    assert strat_stats['backtest_start'] == min_date.strftime(
        DATETIME_PRINT_FORMAT)
    assert strat_stats['backtest_end'] == max_date.strftime(
        DATETIME_PRINT_FORMAT)
    assert strat_stats['total_trades'] == len(results['DefStrat']['results'])
    # Above sample had no loosing trade
    assert strat_stats['max_drawdown_account'] == 0.0

    # Retry with losing trade
    results = {
        'DefStrat': {
            'results':
            pd.DataFrame({
                "pair": [
                    "UNITTEST/BTC", "UNITTEST/BTC", "UNITTEST/BTC",
                    "UNITTEST/BTC"
                ],
                "profit_ratio": [0.003312, 0.010801, -0.013803, 0.002780],
                "profit_abs": [0.000003, 0.000011, -0.000014, 0.000003],
                "open_date": [
                    Arrow(2017, 11, 14, 19, 32, 00).datetime,
                    Arrow(2017, 11, 14, 21, 36, 00).datetime,
                    Arrow(2017, 11, 14, 22, 12, 00).datetime,
                    Arrow(2017, 11, 14, 22, 44, 00).datetime
                ],
                "close_date": [
                    Arrow(2017, 11, 14, 21, 35, 00).datetime,
                    Arrow(2017, 11, 14, 22, 10, 00).datetime,
                    Arrow(2017, 11, 14, 22, 43, 00).datetime,
                    Arrow(2017, 11, 14, 22, 58, 00).datetime
                ],
                "open_rate": [0.002543, 0.003003, 0.003089, 0.003214],
                "close_rate": [0.002546, 0.003014, 0.0032903, 0.003217],
                "trade_duration": [123, 34, 31, 14],
                "is_open": [False, False, False, True],
                "is_short": [False, False, False, False],
                "stake_amount": [0.01, 0.01, 0.01, 0.01],
                "exit_reason": [
                    ExitType.ROI, ExitType.ROI, ExitType.STOP_LOSS,
                    ExitType.FORCE_EXIT
                ]
            }),
            'config':
            default_conf,
            'locks': [],
            'final_balance':
            1000.02,
            'rejected_signals':
            20,
            'timedout_entry_orders':
            0,
            'timedout_exit_orders':
            0,
            'backtest_start_time':
            Arrow.utcnow().int_timestamp,
            'backtest_end_time':
            Arrow.utcnow().int_timestamp,
            'run_id':
            '124',
        }
    }

    stats = generate_backtest_stats(btdata, results, min_date, max_date)
    assert isinstance(stats, dict)
    assert 'strategy' in stats
    assert 'DefStrat' in stats['strategy']
    assert 'strategy_comparison' in stats
    strat_stats = stats['strategy']['DefStrat']

    assert pytest.approx(strat_stats['max_drawdown_account']) == 1.399999e-08
    assert strat_stats['drawdown_start'] == '2017-11-14 22:10:00'
    assert strat_stats['drawdown_end'] == '2017-11-14 22:43:00'
    assert strat_stats['drawdown_end_ts'] == 1510699380000
    assert strat_stats['drawdown_start_ts'] == 1510697400000
    assert strat_stats['pairlist'] == ['UNITTEST/BTC']

    # Test storing stats
    filename = Path(tmpdir / 'btresult.json')
    filename_last = Path(tmpdir / LAST_BT_RESULT_FN)
    _backup_file(filename_last, copy_file=True)
    assert not filename.is_file()

    store_backtest_stats(filename, stats)

    # get real Filename (it's btresult-<date>.json)
    last_fn = get_latest_backtest_filename(filename_last.parent)
    assert re.match(r"btresult-.*\.json", last_fn)

    filename1 = Path(tmpdir / last_fn)
    assert filename1.is_file()
    content = filename1.read_text()
    assert 'max_drawdown_account' in content
    assert 'strategy' in content
    assert 'pairlist' in content

    assert filename_last.is_file()

    _clean_test_file(filename_last)
    filename1.unlink()
Example #14
0
 def __set_last_crawl(self, timestamp=str(time())):
     if type(timestamp) == Arrow:
         self.__last_crawl = timestamp
     else:
         self.__last_crawl = Arrow.fromtimestamp(
             str(timestamp)).replace(tzinfo="UTC")
Example #15
0
    def process_data(self):
        try:
            logger.info("Processing WindsSpots data...")
            result = requests.get("http://api.windspots.com/windmobile/stationinfos?allStation=true",
                                  timeout=(self.connect_timeout, self.read_timeout))

            for windspots_station in result.json()['stationInfo']:
                station_id = None
                try:
                    windspots_id = windspots_station['@id'][10:]
                    station_id = self.get_station_id(windspots_id)
                    station = self.save_station(
                        station_id,
                        windspots_station['@shortName'],
                        windspots_station['@name'],
                        windspots_station['@wgs84Latitude'],
                        windspots_station['@wgs84Longitude'],
                        windspots_station['@maintenanceStatus'],
                        altitude=windspots_station['@altitude'],
                        url=urllib.parse.urljoin(self.provider_url, "/spots"))

                    try:
                        # Asking 2 days of data
                        result = requests.get(
                            "http://api.windspots.com/windmobile/stationdatas/windspots:{windspots_id}/60"
                            .format(windspots_id=windspots_id), timeout=(self.connect_timeout, self.read_timeout))
                        try:
                            windspots_measure = result.json()
                        except ValueError:
                            raise ProviderException("Action=Data return invalid json response")

                        measures_collection = self.measures_collection(station_id)

                        new_measures = []
                        try:
                            key = arrow.get(windspots_measure['@lastUpdate']).timestamp
                        except arrow.parser.ParserError:
                            raise ProviderException("Unable to parse measure date: '{0}".format(
                                windspots_measure['@lastUpdate']))

                        wind_direction_last = windspots_measure['windDirectionChart']['serie']['points'][0]
                        wind_direction_key = int(wind_direction_last['date']) / 1000
                        if key != wind_direction_key:
                            logger.warn(
                                "{name} ({id}): wind direction measure '{direction}' is inconsistent with key '{key}'"
                                .format(
                                    name=station['short'],
                                    id=station_id,
                                    key=Arrow.fromtimestamp(key).format('DD-MM-YY HH:mm:ssZZ'),
                                    direction=Arrow.fromtimestamp(wind_direction_key).format('DD-MM-YY HH:mm:ssZZ')))

                        if not measures_collection.find_one(key):
                            try:
                                measure = self.create_measure(
                                    key,
                                    wind_direction_last['value'],
                                    windspots_measure.get('windAverage'),
                                    windspots_measure.get('windMax'),
                                    windspots_measure.get('airTemperature'),
                                    windspots_measure.get('airHumidity'))
                                new_measures.append(measure)
                            except ProviderException as e:
                                logger.warn("Error while processing measure '{0}' for station '{1}': {2}"
                                            .format(key, station_id, e))
                            except Exception as e:
                                logger.exception("Error while processing measure '{0}' for station '{1}': {2}"
                                                 .format(key, station_id, e))
                                self.raven_client.captureException()

                        self.insert_new_measures(measures_collection, station, new_measures, logger)

                    except ProviderException as e:
                        logger.warn("Error while processing measure for station '{0}': {1}".format(station_id, e))
                    except Exception as e:
                        logger.exception("Error while processing measure for station '{0}': {1}".format(station_id, e))
                        self.raven_client.captureException()

                    self.add_last_measure(station_id)

                except ProviderException as e:
                    logger.warn("Error while processing station '{0}': {1}".format(station_id, e))
                except Exception as e:
                    logger.exception("Error while processing station '{0}': {1}".format(station_id, e))
                    self.raven_client.captureException()

        except ProviderException as e:
            logger.warn("Error while processing Windspots: {0}".format(e))
        except Exception as e:
            logger.exception("Error while processing Windspots: {0}".format(e))
            self.raven_client.captureException()

        logger.info("Done !")
Example #16
0
def generate_strategy_stats(btdata: Dict[str, DataFrame], strategy: str,
                            content: Dict[str, Any], min_date: Arrow,
                            max_date: Arrow,
                            market_change: float) -> Dict[str, Any]:
    """
    :param btdata: Backtest data
    :param strategy: Strategy name
    :param content: Backtest result data in the format:
                    {'results: results, 'config: config}}.
    :param min_date: Backtest start date
    :param max_date: Backtest end date
    :param market_change: float indicating the market change
    :return: Dictionary containing results per strategy and a stratgy summary.
    """
    results: Dict[str, DataFrame] = content['results']
    if not isinstance(results, DataFrame):
        return {}
    config = content['config']
    max_open_trades = min(config['max_open_trades'], len(btdata.keys()))
    starting_balance = config['dry_run_wallet']
    stake_currency = config['stake_currency']

    pair_results = generate_pair_metrics(btdata,
                                         stake_currency=stake_currency,
                                         starting_balance=starting_balance,
                                         results=results,
                                         skip_nan=False)
    sell_reason_stats = generate_sell_reason_stats(
        starting_balance=starting_balance, results=results)
    left_open_results = generate_pair_metrics(
        btdata,
        stake_currency=stake_currency,
        starting_balance=starting_balance,
        results=results.loc[results['is_open']],
        skip_nan=True)
    daily_stats = generate_daily_stats(results)
    trade_stats = generate_trading_stats(results)
    best_pair = max(
        [pair for pair in pair_results if pair['key'] != 'TOTAL'],
        key=lambda x: x['profit_sum']) if len(pair_results) > 1 else None
    worst_pair = min(
        [pair for pair in pair_results if pair['key'] != 'TOTAL'],
        key=lambda x: x['profit_sum']) if len(pair_results) > 1 else None
    results['open_timestamp'] = results['open_date'].astype(int64) // 1e6
    results['close_timestamp'] = results['close_date'].astype(int64) // 1e6

    max_date_real = Arrow.fromtimestamp(max(results['close_timestamp'])).to(
        'utc') if not results['close_timestamp'].empty else max_date
    ended_early = False
    if (max_date_real < max_date):
        max_date = max_date_real
        ended_early = True

    backtest_days = (max_date - min_date).days
    strat_stats = {
        'trades':
        results.to_dict(orient='records'),
        'config':
        config,
        'locks': [lock.to_json() for lock in content['locks']],
        'best_pair':
        best_pair,
        'worst_pair':
        worst_pair,
        'results_per_pair':
        pair_results,
        'sell_reason_summary':
        sell_reason_stats,
        'left_open_trades':
        left_open_results,
        'total_trades':
        len(results),
        'total_volume':
        float(results['stake_amount'].sum()),
        'avg_stake_amount':
        results['stake_amount'].mean() if len(results) > 0 else 0,
        'profit_mean':
        results['profit_ratio'].mean() if len(results) > 0 else 0,
        'profit_median':
        results['profit_ratio'].median() if len(results) > 0 else 0,
        'profit_total':
        results['profit_abs'].sum() / starting_balance,
        'profit_total_abs':
        results['profit_abs'].sum(),
        'backtest_start':
        min_date.datetime,
        'backtest_start_ts':
        min_date.int_timestamp * 1000,
        'backtest_end':
        max_date.datetime,
        'backtest_end_ts':
        max_date.int_timestamp * 1000,
        'backtest_days':
        backtest_days,
        'early_end':
        '* ' if ended_early else '',
        'win_loss_ratio':
        round(
            len(results[results['profit_abs'] > 0]) /
            len(results[results['profit_abs'] < 0]), 4)
        if len(results[results['profit_abs'] < 0]) > 0 else 0.0,
        'backtest_run_start_ts':
        content['backtest_start_time'],
        'backtest_run_end_ts':
        content['backtest_end_time'],
        'trades_per_day':
        round(len(results) / backtest_days, 2) if backtest_days > 0 else 0,
        'market_change':
        market_change,
        'pairlist':
        list(btdata.keys()),
        'stake_amount':
        config['stake_amount'],
        'stake_currency':
        config['stake_currency'],
        'stake_currency_decimals':
        decimals_per_coin(config['stake_currency']),
        'starting_balance':
        starting_balance,
        'dry_run_wallet':
        starting_balance,
        'final_balance':
        content['final_balance'],
        'max_open_trades':
        max_open_trades,
        'max_open_trades_setting':
        (config['max_open_trades']
         if config['max_open_trades'] != float('inf') else -1),
        'timeframe':
        config['timeframe'],
        'timerange':
        config.get('timerange', ''),
        'enable_protections':
        config.get('enable_protections', False),
        'strategy_name':
        strategy,
        # Parameters relevant for backtesting
        'stoploss':
        config['stoploss'],
        'trailing_stop':
        config.get('trailing_stop', False),
        'trailing_stop_positive':
        config.get('trailing_stop_positive'),
        'trailing_stop_positive_offset':
        config.get('trailing_stop_positive_offset', 0.0),
        'trailing_only_offset_is_reached':
        config.get('trailing_only_offset_is_reached', False),
        'use_custom_stoploss':
        config.get('use_custom_stoploss', False),
        'minimal_roi':
        config['minimal_roi'],
        'use_sell_signal':
        config['ask_strategy']['use_sell_signal'],
        'sell_profit_only':
        config['ask_strategy']['sell_profit_only'],
        'sell_profit_offset':
        config['ask_strategy']['sell_profit_offset'],
        'ignore_roi_if_buy_signal':
        config['ask_strategy']['ignore_roi_if_buy_signal'],
        **daily_stats,
        **trade_stats
    }

    try:
        max_drawdown, _, _, _, _ = calculate_max_drawdown(
            results, value_col='profit_ratio')
        drawdown_abs, drawdown_start, drawdown_end, high_val, low_val = calculate_max_drawdown(
            results, value_col='profit_abs')
        strat_stats.update({
            'max_drawdown':
            max_drawdown,
            'max_drawdown_pct':
            drawdown_abs / (high_val + starting_balance),
            'max_drawdown_abs':
            drawdown_abs,
            'drawdown_start':
            drawdown_start,
            'drawdown_start_ts':
            drawdown_start.timestamp() * 1000,
            'drawdown_end':
            drawdown_end,
            'drawdown_end_ts':
            drawdown_end.timestamp() * 1000,
            'max_drawdown_low':
            low_val + starting_balance,
            'max_drawdown_high':
            high_val + starting_balance,
        })

        csum_min, csum_max = calculate_csum(results, starting_balance)
        strat_stats.update({'csum_min': csum_min, 'csum_max': csum_max})

    except ValueError:
        strat_stats.update({
            'max_drawdown':
            0.0,
            'max_drawdown_pct':
            0.0,
            'max_drawdown_abs':
            0.0,
            'max_drawdown_low':
            0.0,
            'max_drawdown_high':
            0.0,
            'drawdown_start':
            datetime(1970, 1, 1, tzinfo=timezone.utc),
            'drawdown_start_ts':
            0,
            'drawdown_end':
            datetime(1970, 1, 1, tzinfo=timezone.utc),
            'drawdown_end_ts':
            0,
            'csum_min':
            0,
            'csum_max':
            0
        })

    return strat_stats