def test_date_range_BST(tickstore_lib): DUMMY_DATA = [ {'a': 1., 'b': 2., 'index': dt(2013, 6, 1, 12, 00, tzinfo=mktz('Europe/London')) }, {'a': 3., 'b': 4., 'index': dt(2013, 6, 1, 13, 00, tzinfo=mktz('Europe/London')) }, ] tickstore_lib.chunk_size = 1 tickstore_lib.write('SYM', DUMMY_DATA) df = tickstore_lib.read('SYM', columns=None) assert_array_equal(df['b'].values, np.array([2., 4.])) # df = tickstore_lib.read('SYM', columns=None, date_range=DateRange(dt(2013, 6, 1, 12), # dt(2013, 6, 1, 13))) # assert_array_equal(df['b'].values, np.array([2., 4.])) df = tickstore_lib.read('SYM', columns=None, date_range=DateRange(dt(2013, 6, 1, 12, tzinfo=mktz('Europe/London')), dt(2013, 6, 1, 13, tzinfo=mktz('Europe/London')))) assert_array_equal(df['b'].values, np.array([2., 4.])) df = tickstore_lib.read('SYM', columns=None, date_range=DateRange(dt(2013, 6, 1, 12, tzinfo=mktz('UTC')), dt(2013, 6, 1, 13, tzinfo=mktz('UTC')))) assert_array_equal(df['b'].values, np.array([4., ]))
def test_delete_daterange(tickstore_lib): DUMMY_DATA = [ {'a': 1., 'b': 2., 'index': dt(2013, 1, 1, tzinfo=mktz('Europe/London')) }, {'a': 3., 'b': 4., 'index': dt(2013, 2, 1, tzinfo=mktz('Europe/London')) }, ] tickstore_lib._chunk_size = 1 tickstore_lib.write('SYM', DUMMY_DATA) # Delete with a date-range deleted = tickstore_lib.delete( 'SYM', DateRange( dt(2013, 1, 1, tzinfo=mktz('Europe/London')), dt(2013, 2, 1, tzinfo=mktz('Europe/London')), CLOSED_OPEN ) ) assert deleted.deleted_count == 1 df = tickstore_lib.read('SYM', columns=None) assert np.allclose(df['b'].values, np.array([4.]))
def test_date_range_default_timezone(tickstore_lib, tz_name): """ We assume naive datetimes are user-local """ DUMMY_DATA = [ {'a': 1., 'b': 2., 'index': dt(2013, 1, 1, tzinfo=mktz(tz_name)) }, # Half-way through the year {'b': 3., 'c': 4., 'index': dt(2013, 7, 1, tzinfo=mktz(tz_name)) }, ] with patch('arctic.date._mktz.DEFAULT_TIME_ZONE_NAME', tz_name): tickstore_lib._chunk_size = 1 tickstore_lib.write('SYM', DUMMY_DATA) df = tickstore_lib.read('SYM', date_range=DateRange(20130101, 20130701), columns=None) assert len(df) == 2 assert df.index[1] == dt(2013, 7, 1, tzinfo=mktz(tz_name)) assert df.index.tz == mktz(tz_name) df = tickstore_lib.read('SYM', date_range=DateRange(20130101, 20130101), columns=None) assert len(df) == 1 df = tickstore_lib.read('SYM', date_range=DateRange(20130701, 20130701), columns=None) assert len(df) == 1
def test_delete(tickstore_lib): DUMMY_DATA = [ {'a': 1., 'b': 2., 'index': dt(2013, 1, 1, tzinfo=mktz('Europe/London')) }, {'a': 3., 'b': 4., 'index': dt(2013, 1, 30, tzinfo=mktz('Europe/London')) }, ] tickstore_lib._chunk_size = 1 tickstore_lib.write('SYM', DUMMY_DATA) deleted = tickstore_lib.delete('SYM') assert deleted.deleted_count == 2 with pytest.raises(NoDataFoundException): tickstore_lib.read('SYM', date_range=DateRange(20130102), columns=None) # Delete with a date-range tickstore_lib.write('SYM', DUMMY_DATA) deleted = tickstore_lib.delete( 'SYM', DateRange( dt(2013, 1, 1, tzinfo=mktz('Europe/London')), dt(2013, 1, 2, tzinfo=mktz('Europe/London')) ) ) assert deleted.deleted_count == 1 df = tickstore_lib.read('SYM', columns=None) assert np.allclose(df['b'].values, np.array([4.]))
def test_datetime_roundtrip_est_tz(): pdt = datetime.datetime(2012, 6, 12, 12, 12, 12, 123000, tzinfo=mktz('EST')) pdt2 = ms_to_datetime(datetime_to_ms(pdt)) assert pdt2.replace(tzinfo=mktz()) == pdt pdt = datetime.datetime(2012, 1, 12, 12, 12, 12, 123000, tzinfo=mktz('EST')) pdt2 = ms_to_datetime(datetime_to_ms(pdt)) assert pdt2.replace(tzinfo=mktz()) == pdt
def test_datetime_roundtrip_lon_tz(): pdt = datetime.datetime(2012, 6, 12, 12, 12, 12, 123000, tzinfo=mktz('Europe/London')) pdt2 = ms_to_datetime(datetime_to_ms(pdt)) assert pdt2 == pdt.replace(tzinfo=None) pdt = datetime.datetime(2012, 1, 12, 12, 12, 12, 123000, tzinfo=mktz('Europe/London')) pdt2 = ms_to_datetime(datetime_to_ms(pdt)) assert pdt2 == pdt.replace(tzinfo=None)
def test_daterange_closedclosed_no_tz(): date_range = DateRange(dt(2013, 1, 1), dt(2014, 2, 1), OPEN_OPEN) expected = DateRange(dt(2013, 1, 1, 0, 0, 0, 1000, tzinfo=mktz()), dt(2014, 1, 31, 23, 59, 59, 999000, tzinfo=mktz()), CLOSED_CLOSED) act = to_pandas_closed_closed(date_range) assert act == expected
def test_read_strings(tickstore_lib): df = pd.DataFrame(data={'data': ['A', 'B', 'C']}, index=pd.Index(data=[dt(2016, 1, 1, 00, tzinfo=mktz('UTC')), dt(2016, 1, 2, 00, tzinfo=mktz('UTC')), dt(2016, 1, 3, 00, tzinfo=mktz('UTC'))], name='date')) tickstore_lib.write('test', df) read_df = tickstore_lib.read('test') assert(all(read_df['data'].values == df['data'].values))
def test_datetime_roundtrip_local_tz(): pdt = datetime.datetime(2012, 6, 12, 12, 12, 12, 123000, tzinfo=mktz(DEFAULT_TIME_ZONE_NAME)) pdt2 = ms_to_datetime(datetime_to_ms(pdt)) assert pdt2 == pdt.replace(tzinfo=None) pdt = datetime.datetime(2012, 1, 12, 12, 12, 12, 123000, tzinfo=mktz(DEFAULT_TIME_ZONE_NAME)) pdt2 = ms_to_datetime(datetime_to_ms(pdt)) assert pdt2 == pdt.replace(tzinfo=None)
def test_read_unicode_strings(tickstore_lib): df = pd.DataFrame(data={'data': [u'一', u'二', u'三']}, # Chinese character [one, two , three] index=pd.Index(data=[dt(2016, 1, 1, 00, tzinfo=mktz('UTC')), dt(2016, 1, 2, 00, tzinfo=mktz('UTC')), dt(2016, 1, 3, 00, tzinfo=mktz('UTC'))], name='date')) tickstore_lib.write('test', df) read_df = tickstore_lib.read('test') assert(all(read_df['data'].values == df['data'].values))
def test_datetime_roundtrip_local_tz(): pdt = datetime.datetime(2012, 6, 12, 12, 12, 12, 123000, tzinfo=mktz()) pdt2 = ms_to_datetime(datetime_to_ms(pdt)) assert pdt2 == pdt pdt = datetime.datetime(2012, 1, 12, 12, 12, 12, 123000, tzinfo=mktz()) pdt2 = ms_to_datetime(datetime_to_ms(pdt)) assert pdt2 == pdt
def test_ts_write_pandas(tickstore_lib): data = DUMMY_DATA tickstore_lib.write('SYM', data) data = tickstore_lib.read('SYM', columns=None).tz_localize(mktz('Europe/London')) tickstore_lib.delete('SYM') tickstore_lib.write('SYM', data) read = tickstore_lib.read('SYM', columns=None).tz_localize(mktz('Europe/London')) assert_frame_equal(read, data, check_names=False)
def assert_roundtrip(tz): ts = datetime.datetime(1982, 7, 1, 16, 5) ts1 = ts.replace(tzinfo=tz) ts2 = ms_to_datetime(datetime_to_ms(ts1.astimezone(mktz("UTC"))), tz) ts1 = ts1.replace(tzinfo=None) if tz == mktz() else ts1 #logger.info(ts2.tzinfo) assert(ts2.hour == ts1.hour) # assert(ts2.tzinfo == ts1.tzinfo) assert ts2 == ts1
def test_mktz(): tz = mktz("Europe/London") d = dt(2012, 2, 2, tzinfo=tz) assert d.tzname() == 'GMT' d = dt(2012, 7, 2, tzinfo=tz) assert d.tzname() == 'BST' tz = mktz('UTC') d = dt(2012, 2, 2, tzinfo=tz) assert d.tzname() == 'UTC' d = dt(2012, 7, 2, tzinfo=tz) assert d.tzname() == 'UTC' # --------replace_empty_timezones_with_default -----------------
def test_read_out_of_order(tickstore_lib): data = [{'A': 120, 'D': 1}, {'A': 122, 'B': 2.0}, {'A': 3, 'B': 3.0, 'D': 1}] tick_index = [dt(2013, 6, 1, 12, 00, tzinfo=mktz('UTC')), dt(2013, 6, 1, 11, 00, tzinfo=mktz('UTC')), # Out-of-order dt(2013, 6, 1, 13, 00, tzinfo=mktz('UTC'))] data = pd.DataFrame(data, index=tick_index) tickstore_lib._chunk_size = 3 tickstore_lib.write('SYM', data) tickstore_lib.read('SYM', columns=None) assert len(tickstore_lib.read('SYM', columns=None, date_range=DateRange(dt(2013, 6, 1, tzinfo=mktz('UTC')), dt(2013, 6, 2, tzinfo=mktz('UTC'))))) == 3 assert len(tickstore_lib.read('SYM', columns=None, date_range=DateRange(dt(2013, 6, 1, tzinfo=mktz('UTC')), dt(2013, 6, 1, 12, tzinfo=mktz('UTC'))))) == 2
def test_read_with_image(tickstore_lib): DUMMY_DATA = [ {'a': 1., 'index': dt(2013, 1, 1, 11, 00, tzinfo=mktz('Europe/London')) }, { 'b': 4., 'index': dt(2013, 1, 1, 12, 00, tzinfo=mktz('Europe/London')) }, ] # Add an image tickstore_lib.write('SYM', DUMMY_DATA) tickstore_lib._collection.update_one({}, {'$set': {'im': {'i': {'a': 37., 'c': 2., }, 't': dt(2013, 1, 1, 10, tzinfo=mktz('Europe/London')) } } } ) dr = DateRange(dt(2013, 1, 1), dt(2013, 1, 2)) # tickstore_lib.read('SYM', columns=None) df = tickstore_lib.read('SYM', columns=None, date_range=dr) assert df['a'][0] == 1 # Read with the image as well df = tickstore_lib.read('SYM', columns=None, date_range=dr, include_images=True) assert set(df.columns) == set(('a', 'b', 'c')) assert_array_equal(df['a'].values, np.array([37, 1, np.nan])) assert_array_equal(df['b'].values, np.array([np.nan, np.nan, 4])) assert_array_equal(df['c'].values, np.array([2, np.nan, np.nan])) assert df.index[0] == dt(2013, 1, 1, 10) assert df.index[1] == dt(2013, 1, 1, 11) assert df.index[2] == dt(2013, 1, 1, 12) df = tickstore_lib.read('SYM', columns=('a', 'b'), date_range=dr, include_images=True) assert set(df.columns) == set(('a', 'b')) assert_array_equal(df['a'].values, np.array([37, 1, np.nan])) assert_array_equal(df['b'].values, np.array([np.nan, np.nan, 4])) assert df.index[0] == dt(2013, 1, 1, 10) assert df.index[1] == dt(2013, 1, 1, 11) assert df.index[2] == dt(2013, 1, 1, 12) df = tickstore_lib.read('SYM', columns=['c'], date_range=dr, include_images=True) assert set(df.columns) == set(['c']) assert_array_equal(df['c'].values, np.array([2, np.nan, np.nan])) assert df.index[0] == dt(2013, 1, 1, 10) assert df.index[1] == dt(2013, 1, 1, 11) assert df.index[2] == dt(2013, 1, 1, 12)
def test_should_list_symbols_from_the_underlying_library(toplevel_tickstore, arctic, start, end, startr, endr): arctic.initialize_library('FEED_2010.LEVEL1', tickstore.TICK_STORE_TYPE) arctic.initialize_library('FEED_2011.LEVEL1', tickstore.TICK_STORE_TYPE) toplevel_tickstore.add(DateRange(start=dt(2010, 1, 1), end=dt(2010, 12, 31, 23, 59, 59, 999000)), 'FEED_2010.LEVEL1') toplevel_tickstore.add(DateRange(start=dt(2011, 1, 1), end=dt(2011, 12, 31, 23, 59, 59, 999000)), 'FEED_2011.LEVEL1') dtstart = dt(2010, 1, 1, tzinfo=mktz('Europe/London')) for i in range(10): dates = pd.date_range(dtstart, periods=50, tz=mktz('Europe/London')) df = pd.DataFrame(np.random.randn(50, 4), index=dates, columns=list('ABCD')) dtstart = dates[-1] + dtd(days=1) toplevel_tickstore.write('sym' + str(i), df) expected_symbols = ['sym' + str(i) for i in range(startr, endr)] assert expected_symbols == toplevel_tickstore.list_symbols(DateRange(start=start, end=end))
def test_should_write_top_level_with_list_of_dicts(arctic): arctic.initialize_library('FEED_2010.LEVEL1', tickstore.TICK_STORE_TYPE) arctic.initialize_library('FEED_2011.LEVEL1', tickstore.TICK_STORE_TYPE) arctic.initialize_library('FEED.LEVEL1', toplevel.TICK_STORE_TYPE) toplevel_tickstore = arctic['FEED.LEVEL1'] dates = pd.date_range('20101201', periods=57, tz=mktz('Europe/London')) data = [{'index': dates[i], 'a': i} for i in range(len(dates))] expected = pd.DataFrame(np.arange(57, dtype=np.float64), index=dates, columns=list('a')) toplevel_tickstore.write('blah', data) res = toplevel_tickstore.read('blah', DateRange(start=dt(2010, 12, 1), end=dt(2011, 2, 1)), columns=list('a')) assert_frame_equal(expected, res.tz_convert(mktz('Europe/London'))) lib2010 = arctic['FEED_2010.LEVEL1'] res = lib2010.read('blah', DateRange(start=dt(2010, 12, 1), end=dt(2011, 1, 1))) assert_frame_equal(expected[dt(2010, 12, 1): dt(2010, 12, 31)], res.tz_convert(mktz('Europe/London')))
def test_objects_fail(tickstore_lib): class Fake(object): def __init__(self, val): self.val = val def fake(self): return self.val df = pd.DataFrame(data={'data': [Fake(1), Fake(2)]}, index=pd.Index(data=[dt(2016, 1, 1, 00, tzinfo=mktz('UTC')), dt(2016, 1, 2, 00, tzinfo=mktz('UTC'))], name='date')) with pytest.raises(Exception) as e: tickstore_lib.write('test', df) assert('Casting object column to string failed' in str(e))
def test_read_utf8_strings(tickstore_lib): data = ['一', '二', '三'] # Chinese character [one, two , three] if six.PY2: utf8_data = data unicode_data = [s.decode('utf8') for s in data] else: utf8_data = [s.encode('utf8') for s in data] unicode_data = data df = pd.DataFrame(data={'data': utf8_data}, index=pd.Index(data=[dt(2016, 1, 1, 00, tzinfo=mktz('UTC')), dt(2016, 1, 2, 00, tzinfo=mktz('UTC')), dt(2016, 1, 3, 00, tzinfo=mktz('UTC'))], name='date')) tickstore_lib.write('test', df) read_df = tickstore_lib.read('test') assert(all(read_df['data'].values == np.array(unicode_data)))
def test_should_return_data_when_date_range_falls_in_a_single_underlying_library(toplevel_tickstore, arctic): arctic.initialize_library('FEED_2010.LEVEL1', tickstore.TICK_STORE_TYPE) tstore = arctic['FEED_2010.LEVEL1'] arctic.initialize_library('test_current.toplevel_tickstore', tickstore.TICK_STORE_TYPE) tickstore_current = arctic['test_current.toplevel_tickstore'] toplevel_tickstore._collection.insert_one({'start': dt(2010, 1, 1), 'end': dt(2010, 12, 31, 23, 59, 59), 'library_name': 'FEED_2010.LEVEL1'}) dates = pd.date_range('20100101', periods=6, tz=mktz('Europe/London')) df = pd.DataFrame(np.random.randn(6, 4), index=dates, columns=list('ABCD')) tstore.write('blah', df) tickstore_current.write('blah', df) res = toplevel_tickstore.read('blah', DateRange(start=dt(2010, 1, 1), end=dt(2010, 1, 6)), list('ABCD')) assert_frame_equal(df, res.tz_convert(mktz('Europe/London')))
def test_should_write_top_level_with_correct_timezone(arctic): # Write timezone aware data and read back in UTC utc = mktz('UTC') arctic.initialize_library('FEED_2010.LEVEL1', tickstore.TICK_STORE_TYPE) arctic.initialize_library('FEED_2011.LEVEL1', tickstore.TICK_STORE_TYPE) arctic.initialize_library('FEED.LEVEL1', toplevel.TICK_STORE_TYPE) toplevel_tickstore = arctic['FEED.LEVEL1'] dates = pd.date_range('20101230220000', periods=10, tz=mktz('America/New_York')) # 10pm New York time is 3am next day UTC data = [{'index': dates[i], 'a': i} for i in range(len(dates))] expected = pd.DataFrame(np.arange(len(dates), dtype=np.float64), index=dates.tz_convert(utc), columns=list('a')) toplevel_tickstore.write('blah', data) res = toplevel_tickstore.read('blah', DateRange(start=dt(2010, 1, 1), end=dt(2011, 12, 31)), columns=list('a')).tz_convert(utc) assert_frame_equal(expected, res) lib2010 = arctic['FEED_2010.LEVEL1'] # Check that only one point was written into 2010 being 3am on 31st assert len(lib2010.read('blah', DateRange(start=dt(2010, 12, 1), end=dt(2011, 1, 1)))) == 1
def test_should_return_data_when_date_range_spans_libraries(toplevel_tickstore, arctic): arctic.initialize_library('FEED_2010.LEVEL1', tickstore.TICK_STORE_TYPE) arctic.initialize_library('FEED_2011.LEVEL1', tickstore.TICK_STORE_TYPE) tickstore_2010 = arctic['FEED_2010.LEVEL1'] tickstore_2011 = arctic['FEED_2011.LEVEL1'] toplevel_tickstore.add(DateRange(start=dt(2010, 1, 1), end=dt(2010, 12, 31, 23, 59, 59, 999000)), 'FEED_2010.LEVEL1') toplevel_tickstore.add(DateRange(start=dt(2011, 1, 1), end=dt(2011, 12, 31, 23, 59, 59, 999000)), 'FEED_2011.LEVEL1') dates = pd.date_range('20100101', periods=6, tz=mktz('Europe/London')) df_10 = pd.DataFrame(np.random.randn(6, 4), index=dates, columns=list('ABCD')) tickstore_2010.write('blah', df_10) dates = pd.date_range('20110101', periods=6, tz=mktz('Europe/London')) df_11 = pd.DataFrame(np.random.randn(6, 4), index=dates, columns=list('ABCD')) tickstore_2011.write('blah', df_11) res = toplevel_tickstore.read('blah', DateRange(start=dt(2010, 1, 2), end=dt(2011, 1, 4)), list('ABCD')) expected_df = pd.concat([df_10[1:], df_11[:4]]) assert_frame_equal(expected_df, res.tz_convert(mktz('Europe/London')))
def test_read(tickstore_lib): data = [{'ASK': 1545.25, 'ASKSIZE': 1002.0, 'BID': 1545.0, 'BIDSIZE': 55.0, 'CUMVOL': 2187387.0, 'DELETED_TIME': 0, 'INSTRTYPE': 'FUT', 'PRICE': 1545.0, 'SIZE': 1.0, 'TICK_STATUS': 0, 'TRADEHIGH': 1561.75, 'TRADELOW': 1537.25, 'index': 1185076787070}, {'CUMVOL': 354.0, 'DELETED_TIME': 0, 'PRICE': 1543.75, 'SIZE': 354.0, 'TRADEHIGH': 1543.75, 'TRADELOW': 1543.75, 'index': 1185141600600}] tickstore_lib.write('FEED::SYMBOL', data) df = tickstore_lib.read('FEED::SYMBOL', columns=['BID', 'ASK', 'PRICE']) assert_array_equal(df['ASK'].values, np.array([1545.25, np.nan])) assert_array_equal(df['BID'].values, np.array([1545, np.nan])) assert_array_equal(df['PRICE'].values, np.array([1545, 1543.75])) assert_array_equal(df.index.values.astype('object'), np.array([1185076787070000000, 1185141600600000000])) assert tickstore_lib._collection.find_one()['c'] == 2 assert df.index.tzinfo == mktz()
async def process_order(orders): async for order in orders.group_by(Order.model_id): print("1. ORDER recevied.") if not order.testing: # [order.symbol] model_account = orders_for_account[order.model_id] print("1. Send to Arctic DB") data = order.asdict() data["index"] = datetime.datetime.fromtimestamp( int(order.timestamps) * 1000, tzinfo=mktz("UTC+8")) lib.write(symbol=f"ORDER:{Order.model_id}", data=[order.asdict()]) await test_order.order_send_to_db.send(order) account = model_account.account if account.get(order.symbol, None) is None: account[order.symbol] = Position(amount=0, price=0) + order else: account[order.symbol] += order model_account.account = account orders_for_account[order.model_id] = model_account await test_order.order_executed.send(order.id) yield order
def test_list_versions_localTime(): # Object ID's are stored in UTC. We need to ensure that the returned times # for versions are in the local TimeZone vs = create_autospec(VersionStore, instance=True, _versions=Mock()) vs._find_snapshots.return_value = 'snap' date = dt(2013, 4, 1, 9, 0) vs._versions.find.return_value = [{'_id': bson.ObjectId.from_datetime(date), 'symbol': 's', 'version': 10}] version = list(VersionStore.list_versions(vs, "symbol"))[0] local_date = date.replace(tzinfo=mktz("UTC")).astimezone(mktz()).replace(tzinfo=None) assert version == {'symbol': version['symbol'], 'version': version['version'], # We return naive datetimes in 'default' time, which is London for us 'date': local_date, 'snapshots': 'snap'}
def test_list_versions_localTime(): # Object ID's are stored in UTC. We need to ensure that the returned times # for versions are in the local TimeZone vs = create_autospec(VersionStore, instance=True, _versions=Mock(), _snapshots=Mock()) mocked_snap_resp = [{'_id': 'abcde', 'name': 'snap'}] vs._snapshots.find.return_value = mocked_snap_resp vs._snapshots.find_one.return_value = mocked_snap_resp date = dt(2013, 4, 1, 9, 0) vs._versions.find.return_value = [{ '_id': bson.ObjectId.from_datetime(date), 'symbol': 's', 'version': 10, 'metadata': None, 'parent': [mocked_snap_resp[0]['_id']] }] version = list(VersionStore.list_versions(vs, "symbol"))[0] local_date = date.replace(tzinfo=mktz("UTC")) assert version == { 'symbol': version['symbol'], 'version': version['version'], # We return naive datetimes in 'default' time, which is London for us 'date': local_date, 'snapshots': ['snap'], 'deleted': False }
def test_should_return_data_when_date_range_spans_libraries_even_if_one_returns_nothing(toplevel_tickstore, arctic): arctic.initialize_library('FEED_2010.LEVEL1', tickstore.TICK_STORE_TYPE) arctic.initialize_library('FEED_2011.LEVEL1', tickstore.TICK_STORE_TYPE) tickstore_2010 = arctic['FEED_2010.LEVEL1'] tickstore_2011 = arctic['FEED_2011.LEVEL1'] toplevel_tickstore.add(DateRange(start=dt(2010, 1, 1), end=dt(2010, 12, 31, 23, 59, 59, 999000)), 'FEED_2010.LEVEL1') toplevel_tickstore.add(DateRange(start=dt(2011, 1, 1), end=dt(2011, 12, 31, 23, 59, 59, 999000)), 'FEED_2011.LEVEL1') dates = pd.date_range('20100101', periods=6, tz=mktz('Europe/London')) df_10 = pd.DataFrame(np.random.randn(6, 4), index=dates, columns=list('ABCD')) tickstore_2010.write('blah', df_10) dates = pd.date_range('20110201', periods=6, tz=mktz('Europe/London')) df_11 = pd.DataFrame(np.random.randn(6, 4), index=dates, columns=list('ABCD')) tickstore_2011.write('blah', df_11) res = toplevel_tickstore.read('blah', DateRange(start=dt(2010, 1, 2), end=dt(2011, 1, 4)), list('ABCD')) expected_df = df_10[1:] assert_frame_equal(expected_df, res.tz_convert(mktz('Europe/London')))
def test_read_longs(tickstore_lib): DUMMY_DATA = [ {'a': 1, 'index': dt(2013, 6, 1, 12, 00, tzinfo=mktz('Europe/London')) }, { 'b': 4, 'index': dt(2013, 6, 1, 13, 00, tzinfo=mktz('Europe/London')) }, ] tickstore_lib.chunk_size = 3 tickstore_lib.write('SYM', DUMMY_DATA) tickstore_lib.read('SYM', columns=None) read = tickstore_lib.read('SYM', columns=None, date_range=DateRange(dt(2013, 6, 1), dt(2013, 6, 2))) assert read['a'][0] == 1 assert np.isnan(read['b'][0])
def test_utc_dt_to_local_dt(): with pytest.raises(ValueError): assert (utc_dt_to_local_dt( dt(2000, 1, 1, 0, 0, 0, tzinfo=mktz("Asia/Shanghai")))) utc_time = dt(2000, 1, 1, 10, 0, 0) pek_time = utc_dt_to_local_dt(utc_time) # GMT +0800 assert (pek_time.hour - utc_time.hour == 8)
def assert_roundtrip(tz): ts = datetime.datetime(1982, 7, 1, 16, 5) ts1 = ts.replace(tzinfo=tz) ts2 = ms_to_datetime(datetime_to_ms(ts1.astimezone(mktz("UTC"))), tz) assert(ts2.hour == ts1.hour) assert ts2 == ts1
def assert_roundtrip(tz): ts = datetime.datetime(1982, 7, 1, 16, 5) ts1 = ts.replace(tzinfo=tz) ts2 = ms_to_datetime(datetime_to_ms(ts1.astimezone(mktz("UTC"))), tz) assert (ts2.hour == ts1.hour) assert ts2 == ts1
def test_read_out_of_order(tickstore_lib): DUMMY_DATA = [ { 'a': 1., 'b': 2., 'index': dt(2013, 6, 1, 12, 00, tzinfo=mktz('UTC')) }, { 'a': 3., 'b': 4., 'index': dt(2013, 6, 1, 11, 00, tzinfo=mktz('UTC')) # Out-of-order }, { 'a': 3., 'b': 4., 'index': dt(2013, 6, 1, 13, 00, tzinfo=mktz('UTC')) }, ] tickstore_lib.chunk_size = 3 tickstore_lib.write('SYM', DUMMY_DATA) tickstore_lib.read('SYM', columns=None) assert len( tickstore_lib.read('SYM', columns=None, date_range=DateRange( dt(2013, 6, 1, tzinfo=mktz('UTC')), dt(2013, 6, 2, tzinfo=mktz('UTC'))))) == 3 assert len( tickstore_lib.read('SYM', columns=None, date_range=DateRange( dt(2013, 6, 1, tzinfo=mktz('UTC')), dt(2013, 6, 1, 12, tzinfo=mktz('UTC'))))) == 2
def test_read_out_of_order(tickstore_lib): data = [{ 'A': 120, 'D': 1 }, { 'A': 122, 'B': 2.0 }, { 'A': 3, 'B': 3.0, 'D': 1 }] tick_index = [ dt(2013, 6, 1, 12, 00, tzinfo=mktz('UTC')), dt(2013, 6, 1, 11, 00, tzinfo=mktz('UTC')), # Out-of-order dt(2013, 6, 1, 13, 00, tzinfo=mktz('UTC')) ] data = pd.DataFrame(data, index=tick_index) tickstore_lib._chunk_size = 3 tickstore_lib.write('SYM', data) tickstore_lib.read('SYM', columns=None) assert len( tickstore_lib.read('SYM', columns=None, date_range=DateRange( dt(2013, 6, 1, tzinfo=mktz('UTC')), dt(2013, 6, 2, tzinfo=mktz('UTC'))))) == 3 assert len( tickstore_lib.read('SYM', columns=None, date_range=DateRange( dt(2013, 6, 1, tzinfo=mktz('UTC')), dt(2013, 6, 1, 12, tzinfo=mktz('UTC'))))) == 2
def test_read_as_of_NotNaive(): # When we do a read, with naive as_of, that as_of is treated in London Time. vs = create_autospec(VersionStore, instance=True, _versions=Mock(), _allow_secondary=False) VersionStore._read_metadata(vs, 'symbol', dt(2013, 4, 1, 9, 0, tzinfo=mktz('Europe/Paris'))) versions = vs._versions.with_options.return_value versions.find_one.assert_called_once_with({'symbol':'symbol', '_id': {'$lt': bson.ObjectId.from_datetime(dt(2013, 4, 1, 9, 0, tzinfo=mktz('Europe/Paris')) + dtd(seconds=1))}}, sort=[('_id', pymongo.DESCENDING)])
def test_utc_dt_to_local_dt(): with pytest.raises(ValueError): assert(utc_dt_to_local_dt( dt(2000, 1, 1, 0, 0, 0, tzinfo=mktz("Asia/Shanghai")) )) utc_time = dt(2000, 1, 1, 10, 0, 0) pek_time = utc_dt_to_local_dt(utc_time) # GMT +0800 assert(pek_time.hour - utc_time.hour == 8)
def test_objects_fail(tickstore_lib): class Fake(object): def __init__(self, val): self.val = val def fake(self): return self.val df = pd.DataFrame(data={'data': [Fake(1), Fake(2)]}, index=pd.Index(data=[ dt(2016, 1, 1, 00, tzinfo=mktz('UTC')), dt(2016, 1, 2, 00, tzinfo=mktz('UTC')) ], name='date')) with pytest.raises(Exception) as e: tickstore_lib.write('test', df) assert ('Casting object column to string failed' in str(e.value))
def test_delete_daterange(tickstore_lib): DUMMY_DATA = [ {'a': 1., 'b': 2., 'index': dt(2013, 1, 1, tzinfo=mktz('Europe/London')) }, {'a': 3., 'b': 4., 'index': dt(2013, 2, 1, tzinfo=mktz('Europe/London')) }, ] tickstore_lib.chunk_size = 1 tickstore_lib.write('SYM', DUMMY_DATA) # Delete with a date-range tickstore_lib.delete('SYM', DateRange(dt(2013, 1, 1, tzinfo=mktz('Europe/London')), dt(2013, 2, 1, tzinfo=mktz('Europe/London')), CLOSED_OPEN)) df = tickstore_lib.read('SYM', columns=None) assert np.allclose(df['b'].values, np.array([4.]))
def test_date_range_end_not_in_range(tickstore_lib): DUMMY_DATA = [ {'a': 1., 'b': 2., 'index': dt(2013, 1, 1, tzinfo=mktz('Europe/London')) }, {'b': 3., 'c': 4., 'index': dt(2013, 1, 2, 10, 1, tzinfo=mktz('Europe/London')) }, ] tickstore_lib.chunk_size = 1 tickstore_lib.write('SYM', DUMMY_DATA) with patch.object(tickstore_lib._collection, 'find', side_effect=tickstore_lib._collection.find) as f: df = tickstore_lib.read('SYM', date_range=DateRange(20130101, dt(2013, 1, 2, 9, 0)), columns=None) assert_array_equal(df['b'].values, np.array([2.])) assert tickstore_lib._collection.find(f.call_args_list[-1][0][0]).count() == 1
def test_date_range_end_not_in_range(tickstore_lib): DUMMY_DATA = [ {'a': 1., 'b': 2., 'index': dt(2013, 1, 1, tzinfo=mktz('Europe/London')) }, {'b': 3., 'c': 4., 'index': dt(2013, 1, 2, 10, 1, tzinfo=mktz('Europe/London')) }, ] tickstore_lib._chunk_size = 1 tickstore_lib.write('SYM', DUMMY_DATA) with patch.object(tickstore_lib._collection, 'find', side_effect=tickstore_lib._collection.find) as f: df = tickstore_lib.read('SYM', date_range=DateRange(20130101, dt(2013, 1, 2, 9, 0)), columns=None) assert_array_equal(df['b'].values, np.array([2.])) assert tickstore_lib._collection.find(f.call_args_list[-1][0][0]).count() == 1
def test_read_utf8_strings(tickstore_lib): data = ['一', '二', '三'] # Chinese character [one, two , three] if six.PY2: utf8_data = data unicode_data = [s.decode('utf8') for s in data] else: utf8_data = [s.encode('utf8') for s in data] unicode_data = data df = pd.DataFrame(data={'data': utf8_data}, index=pd.Index(data=[ dt(2016, 1, 1, 00, tzinfo=mktz('UTC')), dt(2016, 1, 2, 00, tzinfo=mktz('UTC')), dt(2016, 1, 3, 00, tzinfo=mktz('UTC')) ], name='date')) tickstore_lib.write('test', df) read_df = tickstore_lib.read('test') assert (all(read_df['data'].values == np.array(unicode_data)))
def test_should_successfully_do_a_roundtrip_write_and_read_spanning_multiple_underlying_libraries(toplevel_tickstore, arctic): arctic.initialize_library('FEED_2010.LEVEL1', tickstore.TICK_STORE_TYPE) arctic.initialize_library('FEED_2011.LEVEL1', tickstore.TICK_STORE_TYPE) arctic.initialize_library('test_current.toplevel_tickstore', tickstore.TICK_STORE_TYPE) toplevel_tickstore.add(DateRange(start=dt(2010, 1, 1), end=dt(2010, 12, 31, 23, 59, 59, 999000)), 'FEED_2010.LEVEL1') toplevel_tickstore.add(DateRange(start=dt(2011, 1, 1), end=dt(2011, 12, 31, 23, 59, 59, 999000)), 'FEED_2011.LEVEL1') tickstore_current = arctic['test_current.toplevel_tickstore'] dates = pd.date_range('20101201', periods=57, tz=mktz('Europe/London')) data = pd.DataFrame(np.random.randn(57, 4), index=dates, columns=list('ABCD')) toplevel_tickstore.write('blah', data) tickstore_current.write('blah', data) res = toplevel_tickstore.read('blah', DateRange(start=dt(2010, 12, 1), end=dt(2011, 2, 1)), columns=list('ABCD')) assert_frame_equal(data, res.tz_convert(mktz('Europe/London'))) lib2010 = arctic['FEED_2010.LEVEL1'] res = lib2010.read('blah', DateRange(start=dt(2010, 12, 1), end=dt(2011, 1, 1)), columns=list('ABCD')) assert_frame_equal(data[dt(2010, 12, 1): dt(2010, 12, 31)], res.tz_convert(mktz('Europe/London'))) lib2011 = arctic['FEED_2011.LEVEL1'] res = lib2011.read('blah', DateRange(start=dt(2011, 1, 1), end=dt(2011, 2, 1)), columns=list('ABCD')) assert_frame_equal(data[dt(2011, 1, 1): dt(2011, 2, 1)], res.tz_convert(mktz('Europe/London')))
def test_read_with_image(tickstore_lib): DUMMY_DATA = [ { 'a': 1., 'index': dt(2013, 6, 1, 12, 00, tzinfo=mktz('Europe/London')) }, { 'b': 4., 'index': dt(2013, 6, 1, 13, 00, tzinfo=mktz('Europe/London')) }, ] # Add an image tickstore_lib.write('SYM', DUMMY_DATA) tickstore_lib._collection.update_one({}, { '$set': { 'im': { 'i': { 'a': 37., 'c': 2., }, 't': dt(2013, 6, 1, 11, 0) } } }) tickstore_lib.read('SYM', columns=None) read = tickstore_lib.read('SYM', columns=None, date_range=DateRange(dt(2013, 6, 1), dt(2013, 6, 2))) assert read['a'][0] == 1 # Read with the image as well read = tickstore_lib.read('SYM', columns=None, date_range=DateRange(dt(2013, 6, 1), dt(2013, 6, 2)), include_images=True) assert read['a'][0] == 37 assert read['a'][1] == 1 assert np.isnan(read['b'][0]) assert read['b'][2] == 4 assert read.index[0] == dt(2013, 6, 1, 11)
def test_read_longs(tickstore_lib): DUMMY_DATA = [ { 'a': 1, 'index': dt(2013, 6, 1, 12, 00, tzinfo=mktz('Europe/London')) }, { 'b': 4, 'index': dt(2013, 6, 1, 13, 00, tzinfo=mktz('Europe/London')) }, ] tickstore_lib._chunk_size = 3 tickstore_lib.write('SYM', DUMMY_DATA) tickstore_lib.read('SYM', columns=None) read = tickstore_lib.read('SYM', columns=None, date_range=DateRange(dt(2013, 6, 1), dt(2013, 6, 2))) assert read['a'][0] == 1 assert np.isnan(read['b'][0])
def test_no_min_max_date(arctic): arctic.initialize_library('FEED_2010.LEVEL1', tickstore.TICK_STORE_TYPE) tstore = arctic['FEED_2010.LEVEL1'] dates = pd.date_range('20100101', periods=6, tz=mktz('Europe/London')) df = pd.DataFrame(np.random.randn(6, 4), index=dates, columns=list('ABCD')) tstore.write('blah', df) with pytest.raises(NoDataFoundException): tstore.min_date('unknown-symbol') with pytest.raises(NoDataFoundException): tstore.max_date('unknown-symbol')
def test_min_max_date(arctic): arctic.initialize_library('FEED_2010.LEVEL1', tickstore.TICK_STORE_TYPE) tstore = arctic['FEED_2010.LEVEL1'] dates = pd.date_range('20100101', periods=6, tz=mktz('Europe/London')) df = pd.DataFrame(np.random.randn(6, 4), index=dates, columns=list('ABCD')) tstore.write('blah', df) min_date = tstore.min_date('blah') max_date = tstore.max_date('blah') assert min_date == dates[0].to_pydatetime() assert max_date == dates[-1].to_pydatetime()
def test_read_with_metadata(tickstore_lib): metadata = {'metadata': 'important data'} tickstore_lib.write( 'test', [{ 'index': dt(2013, 6, 1, 13, 00, tzinfo=mktz('Europe/London')), 'price': 100.50, 'ticker': 'QQQ' }], metadata=metadata) m = tickstore_lib.read_metadata('test') assert (metadata == m)
def test_read_all_cols_all_dtypes(tickstore_lib, chunk_size): data = [ { 'f': 0.1, 'of': 0.2, 's': 's', 'os': 'os', 'l': 1, 'ol': 2, 'index': dt(1970, 1, 1, tzinfo=mktz('UTC')), }, { 'f': 0.3, 'nf': 0.4, 's': 't', 'ns': 'ns', 'l': 3, 'nl': 4, 'index': dt(1970, 1, 1, 0, 0, 1, tzinfo=mktz('UTC')), }, ] tickstore_lib._chunk_size = chunk_size tickstore_lib.write('sym', data) df = tickstore_lib.read('sym', columns=None) # The below is probably more trouble than it's worth, but we *should* # be able to roundtrip data and get the same answer... # Ints become floats data[0]['l'] = float(data[0]['l']) # Treat missing strings as None data[0]['ns'] = None data[1]['os'] = None index = DatetimeIndex([ dt(1970, 1, 1, tzinfo=mktz('UTC')), dt(1970, 1, 1, 0, 0, 1, tzinfo=mktz('UTC')) ], ) index.tz = mktz() expected = pd.DataFrame(data, index=index) expected = expected[df.columns] assert_frame_equal(expected, df, check_names=False)
def test_ts_write_pandas(tickstore_lib): data = DUMMY_DATA tickstore_lib.write('SYM', data) data = tickstore_lib.read('SYM', columns=None) assert data.index[0] == dt(2013, 1, 1, tzinfo=mktz('Europe/London')) assert data.a[0] == 1 tickstore_lib.delete('SYM') tickstore_lib.write('SYM', data) read = tickstore_lib.read('SYM', columns=None) assert_frame_equal(read, data, check_names=False)
def test_should_write_top_level_with_list_of_dicts(arctic): arctic.initialize_library('FEED_2010.LEVEL1', tickstore.TICK_STORE_TYPE) arctic.initialize_library('FEED_2011.LEVEL1', tickstore.TICK_STORE_TYPE) arctic.initialize_library('FEED.LEVEL1', toplevel.TICK_STORE_TYPE) toplevel_tickstore = arctic['FEED.LEVEL1'] dates = pd.date_range('20101201', periods=57, tz=mktz('Europe/London')) data = [{'index': dates[i], 'a': i} for i in range(len(dates))] expected = pd.DataFrame(np.arange(57, dtype=np.float64), index=dates, columns=list('a')) toplevel_tickstore.write('blah', data) res = toplevel_tickstore.read('blah', DateRange(start=dt(2010, 12, 1), end=dt(2011, 2, 1)), columns=list('a')) assert_frame_equal(expected, res.tz_convert(mktz('Europe/London'))) lib2010 = arctic['FEED_2010.LEVEL1'] res = lib2010.read('blah', DateRange(start=dt(2010, 12, 1), end=dt(2011, 1, 1))) assert_frame_equal(expected[dt(2010, 12, 1):dt(2010, 12, 31)], res.tz_convert(mktz('Europe/London')))
def test_get_library_metadata(arctic): arctic.initialize_library('FEED_2010.LEVEL1', tickstore.TICK_STORE_TYPE) arctic.initialize_library('FEED_2011.LEVEL1', tickstore.TICK_STORE_TYPE) arctic.initialize_library('FEED.LEVEL1', toplevel.TICK_STORE_TYPE) toplevel_tickstore = arctic['FEED.LEVEL1'] symbol = "USD" tzinfo=mktz('Asia/Chongqing') with pytest.raises(NoDataFoundException): toplevel_tickstore.read(symbol, DateRange(start=dt(2010, 1, 1, tzinfo=tzinfo), end=dt(2011, 1, 2, tzinfo=tzinfo)), columns=None)
def test_date_range_default_timezone(tickstore_lib, tz_name): """ We assume naive datetimes are user-local """ DUMMY_DATA = [ {'a': 1., 'b': 2., 'index': dt(2013, 1, 1, tzinfo=mktz(tz_name)) }, # Half-way through the year {'b': 3., 'c': 4., 'index': dt(2013, 7, 1, tzinfo=mktz(tz_name)) }, ] with patch('tzlocal.get_localzone', return_value=Mock(zone=tz_name)): tickstore_lib._chunk_size = 1 tickstore_lib.write('SYM', DUMMY_DATA) df = tickstore_lib.read('SYM', date_range=DateRange(20130101, 20130701), columns=None) assert df.index.tzinfo == mktz() assert len(df) == 2 assert df.index[1] == dt(2013, 7, 1, tzinfo=mktz(tz_name)) df = tickstore_lib.read('SYM', date_range=DateRange(20130101, 20130101), columns=None) assert len(df) == 1 assert df.index.tzinfo == mktz() df = tickstore_lib.read('SYM', date_range=DateRange(20130701, 20130701), columns=None) assert len(df) == 1 assert df.index.tzinfo == mktz()
def test_list_versions_localTime(): # Object ID's are stored in UTC. We need to ensure that the returned times # for versions are in the local TimeZone vs = create_autospec(VersionStore, instance=True, _versions=Mock()) vs._find_snapshots.return_value = 'snap' date = dt(2013, 4, 1, 9, 0) vs._versions.find.return_value = [{ '_id': bson.ObjectId.from_datetime(date), 'symbol': 's', 'version': 10 }] version = list(VersionStore.list_versions(vs, "symbol"))[0] local_date = date.replace(tzinfo=mktz("UTC")).astimezone( mktz()).replace(tzinfo=None) assert version == { 'symbol': version['symbol'], 'version': version['version'], # We return naive datetimes in 'default' time, which is London for us 'date': local_date, 'snapshots': 'snap' }
def test_delete(tickstore_lib): DUMMY_DATA = [ {'a': 1., 'b': 2., 'index': dt(2013, 1, 1, tzinfo=mktz('Europe/London')) }, {'a': 3., 'b': 4., 'index': dt(2013, 1, 30, tzinfo=mktz('Europe/London')) }, ] tickstore_lib.chunk_size = 1 tickstore_lib.write('SYM', DUMMY_DATA) tickstore_lib.delete('SYM') with pytest.raises(NoDataFoundException): tickstore_lib.read('SYM', date_range=DateRange(20130102), columns=None) # Delete with a date-range tickstore_lib.write('SYM', DUMMY_DATA) tickstore_lib.delete('SYM', DateRange(dt(2013, 1, 1, tzinfo=mktz('Europe/London')), dt(2013, 1, 2, tzinfo=mktz('Europe/London')))) df = tickstore_lib.read('SYM', columns=None) assert np.allclose(df['b'].values, np.array([4.]))
def test_datetime_roundtrip_est_tz(): pdt = datetime.datetime(2012, 6, 12, 12, 12, 12, 123000, tzinfo=mktz('EST')) pdt2 = ms_to_datetime(datetime_to_ms(pdt)) assert pdt2.replace(tzinfo=mktz(DEFAULT_TIME_ZONE_NAME)) == pdt pdt = datetime.datetime(2012, 1, 12, 12, 12, 12, 123000, tzinfo=mktz('EST')) pdt2 = ms_to_datetime(datetime_to_ms(pdt)) assert pdt2.replace(tzinfo=mktz(DEFAULT_TIME_ZONE_NAME)) == pdt
def test_datetime_roundtrip_local_tz(): pdt = datetime.datetime(2012, 6, 12, 12, 12, 12, 123000, tzinfo=mktz(DEFAULT_TIME_ZONE_NAME)) pdt2 = ms_to_datetime(datetime_to_ms(pdt)) assert pdt2 == pdt pdt = datetime.datetime(2012, 1, 12, 12, 12, 12, 123000, tzinfo=mktz(DEFAULT_TIME_ZONE_NAME)) pdt2 = ms_to_datetime(datetime_to_ms(pdt)) assert pdt2 == pdt
def test_get_libraries_no_data_raises_exception_tzinfo_given( toplevel_tickstore, arctic): tzinfo = mktz('Asia/Chongqing') date_range = DateRange(start=dt(2009, 1, 1, tzinfo=tzinfo), end=dt(2010, 12, 31, 23, 59, 59, 999000, tzinfo=tzinfo)) with pytest.raises(NoDataFoundException): toplevel_tickstore._get_libraries(date_range)
def test_datetime_to_ms_and_back_microseconds(): pdt = dt(2012, 8, 1, 12, 34, 56, 999999, tzinfo=mktz()) i = datetime_to_ms(pdt) pdt2 = ms_to_datetime(i) assert pdt != pdt2 assert pdt.year == pdt2.year assert pdt.month == pdt2.month assert pdt.day == pdt2.day assert pdt.hour == pdt2.hour assert pdt.minute == pdt2.minute assert pdt.second == pdt2.second # Microsecond precision loss inevitable. assert pdt.microsecond // 1000 == pdt2.microsecond // 1000