def setUp(self): self.application = ImiguApplication(helper.datadir, SH()) self.application.dbm._mtime = 1291167000 self.open_time = 1291167000 self.close_time = 1291186800 key = helper.sample_key() sample = helper.sample() sample[key]['timestamp'] = 1291167000 self.application.dbm.reportstore.update(sample)
def main(): tornado.options.parse_command_line() app = ImiguApplication(options.datadir, SH()) server = Server(app, auth_password=config.AUTH_PASSWORD) server.listen(options.port) io_loop = tornado.ioloop.IOLoop.instance() check_time = 1 * 1000 # every second scheduler = ioloop.PeriodicCallback(app.periodic_job, check_time, io_loop=io_loop) def shutdown(signum, frame): print 'Signal handler called with signal', signum io_loop.stop() scheduler.stop() server.log_stats() logging.info("==> Exiting datafeed.") signal.signal(signal.SIGTERM, shutdown) signal.signal(signal.SIGINT, shutdown) scheduler.start() io_loop.start()
class ImiguApplicationTest(unittest.TestCase): def setUp(self): self.application = ImiguApplication(helper.datadir, SH()) self.application.dbm._mtime = 1291167000 self.open_time = 1291167000 self.close_time = 1291186800 key = helper.sample_key() sample = helper.sample() sample[key]['timestamp'] = 1291167000 self.application.dbm.reportstore.update(sample) @patch.object(time, 'time') def test_archive_day_09_29(self, mock_time): mock_time.return_value = self.open_time - 1 # not open today = datetime.datetime.today() ret = self.application.scheduled_archive_day(today) self.assertFalse(ret) @patch.object(time, 'time') def test_archive_day_15_05_no_data(self, mock_time): mock_time.return_value = self.close_time + 300 self.application.dbm._mtime = self.close_time - 86400 today = datetime.datetime.today() ret = self.application.scheduled_archive_day(today) self.assertFalse(ret) @patch.object(time, 'time') def test_archive_day_15_05_01(self, mock_time): mock_time.return_value = self.close_time + 181 # closed more than 3 minutes self.application.dbm._mtime = self.close_time + 180 + 1 today = datetime.datetime.today() ret = self.application.scheduled_archive_day(today) self.assertTrue(ret) @patch.object(time, 'time') def test_archive_day_15_05_01_archived_before(self, mock_time): mock_time.return_value = self.close_time + 181 # closed more than 3 minutes self.application.archive_day_time = self.close_time + 180 today = datetime.datetime.today() ret = self.application.scheduled_archive_day(today) self.assertFalse(ret) @patch.object(time, 'time') def test_archive_minute_09_29(self, mock_time): mock_time.return_value = self.open_time - 1 # before open today = datetime.datetime.today() ret = self.application.scheduled_archive_minute(today) self.assertFalse(ret) @patch.object(time, 'time') def test_archive_minute_09_30(self, mock_time): mock_time.return_value = self.open_time today = datetime.datetime.today() ret = self.application.scheduled_archive_minute(today) self.assertTrue(ret) @patch.object(time, 'time') def test_archive_minute_14_30(self, mock_time): mock_time.return_value = self.close_time - 1800 # in session today = datetime.datetime.today() ret = self.application.scheduled_archive_minute(today) self.assertTrue(ret) @patch.object(time, 'time') def test_archive_minute_14_30_05_if_not_archived(self, mock_time): mock_time.return_value = self.close_time - 1795 # in session self.application.archive_minute_time = self.close_time - 1860 today = datetime.datetime.today() ret = self.application.scheduled_archive_minute(today) self.assertTrue(ret) @patch.object(time, 'time') def test_archive_minute_14_30_05_if_archived(self, mock_time): mock_time.return_value = self.close_time - 1795 # in session self.application.archive_minute_time = self.close_time - 1800 today = datetime.datetime.today() ret = self.application.scheduled_archive_minute(today) self.assertFalse(ret) @patch.object(time, 'time') def test_archive_minute_15_00(self, mock_time): mock_time.return_value = self.close_time today = datetime.datetime.today() ret = self.application.scheduled_archive_minute(today) self.assertTrue(ret) @patch.object(time, 'time') def test_archive_minute_15_03(self, mock_time): mock_time.return_value = self.close_time + 180 today = datetime.datetime.today() ret = self.application.scheduled_archive_minute(today) self.assertTrue(ret) @patch.object(time, 'time') def test_archive_minute_15_05_01(self, mock_time): mock_time.return_value = self.close_time + 300 + 1 # closed today = datetime.datetime.today() ret = self.application.scheduled_archive_minute(today) self.assertFalse(ret) @patch.object(time, 'time') def test_crontab_08_00_00(self, mock_time): mock_time.return_value = self.open_time - 3600 - 1800 today = datetime.datetime.fromtimestamp(time.time()) ret = self.application.scheduled_crontab_daily(today) self.assertTrue(ret) @patch.object(time, 'time') def test_crontab_08_00_01_if_not_running(self, mock_time): mock_time.return_value = self.open_time - 3600 - 1799 self.application.crontab_time = self.open_time - 86400 - 7200 today = datetime.datetime.fromtimestamp(time.time()) ret = self.application.scheduled_crontab_daily(today) self.assertTrue(ret) @patch.object(time, 'time') def test_crontab_09_30(self, mock_time): mock_time.return_value = self.open_time today = datetime.datetime.fromtimestamp(time.time()) ret = self.application.scheduled_crontab_daily(today) self.assertFalse(ret) def test_archive_day(self): r = { 'amount': 84596203520.0, 'close': 2856.9899999999998, 'high': 2880.5599999999999, 'low': 2851.9499999999998, 'name': u'\u4e0a\u8bc1\u6307\u6570', 'open': 2868.73, 'preclose': 2875.8600000000001, 'price': 2856.9899999999998, 'symbol': 'SH000001', 'volume': 75147848.0 } day = datetime.datetime.today() ts = time.mktime((day.year, day.month, day.day, 15, 0, 0, 0, 0, 0)) day_ts = time.mktime((day.year, day.month, day.day, 0, 0, 0, 0, 0, 0)) r['timestamp'] = ts r['time'] = str(datetime.datetime.fromtimestamp(ts)) data = {'SH000001': r} import zlib import marshal data = zlib.compress(marshal.dumps(data)) request = Request(None, 'put_reports', data) self.application(request) request = Request(None, 'archive_day') self.application(request) y = self.application.dbm.daystore.get('SH000001', 1) self.assertEqual(y[0]['time'], day_ts) self.assertTrue((y[0]['open'] - 2868.73) < 0.1 ** 6) @patch.object(ImiguHandler, 'get_snapshot_index') def test_fix_report_when_archive(self, mock_index): # set to after hours: 15:30 implicates error data # some datafeed still sending data even market was closed. day = datetime.datetime.today() ts = time.mktime((day.year, day.month, day.day, 15, 30, 0, 0, 0, 0)) mock_index.return_value = (ts, 360) r = { 'amount': 84596203520.0, 'close': 2856.9899999999998, 'high': 2880.5599999999999, 'low': 2851.9499999999998, 'name': u'\u4e0a\u8bc1\u6307\u6570', 'open': 2868.73, 'preclose': 2875.8600000000001, 'price': 2856.9899999999998, 'symbol': 'SH000001', 'time': '2010-12-08 14:02:57', 'timestamp': 1291788177, 'volume': 75147848.0 } r['timestamp'] = ts r['time'] = str(datetime.datetime.fromtimestamp(ts)) data = {'SH000001': r} import zlib import marshal data = zlib.compress(marshal.dumps(data)) request = Request(None, 'put_reports', data) self.application(request) close_time = time.mktime((day.year, day.month, day.day, 15, 0, 0, 0, 0, 0)) request = Request(None, 'archive_minute', data) self.application(request) r = self.application.dbm.get_report('SH000001') self.assertEqual(r['timestamp'], close_time) self.assertEqual(r['open'], 2868.73) @patch.object(ImiguHandler, 'get_snapshot_index') def test_archive_minute_at_open_time(self, mock_index): # set data time to pre-market(centralized competitive pricing) day = datetime.datetime.today() t1 = time.mktime((day.year, day.month, day.day, 9, 26, 0, 0, 0, 0)) open_time = time.mktime((day.year, day.month, day.day, 9, 30, 0, 0, 0, 0)) mock_index.return_value = (open_time, 0) r = { 'amount': 10000.0, 'close': 0.0, 'high': 3000.0, 'low': 3000.0, 'name': u'\u4e0a\u8bc1\u6307\u6570', 'open': 3000.0, 'preclose': 2875.0, 'price': 3000.0, 'symbol': 'SH000001', 'volume': 900000.0 } r['timestamp'] = t1 r['time'] = str(datetime.datetime.fromtimestamp(t1)) data = {'SH000001': r} import zlib import marshal data = zlib.compress(marshal.dumps(data)) request = Request(None, 'put_reports', data) self.application(request) self.assertEqual(self.application.dbm.mtime, t1) request = Request(None, 'archive_minute') self.application(request) y = self.application.dbm.minutestore.get('SH000001') self.assertEqual(y[0]['time'], open_time) self.assertEqual(y[0]['price'], 3000.0) @patch.object(ImiguHandler, 'get_snapshot_index') def test_archive_minute_raise_at_wrong_index(self, mock_index): # set data time to pre-market(centralized competitive pricing) day = datetime.datetime.today() t1 = time.mktime((day.year, day.month, day.day, 9, 26, 0, 0, 0, 0)) mock_index.return_value = (t1, -4) request = Request(None, 'archive_minute') self.assertRaises(SnapshotIndexError, self.application, request) @patch.object(time, 'time') def test_get_snapshot_index(self, mock_time): mock_time.return_value = 1309829400 report_time = 1309829160 mintime, index = ImiguHandler.get_snapshot_index(1309829400, report_time) self.assertEqual(mintime, 1309829400) self.assertEqual(index, 0)
class ImiguApplicationTest(unittest.TestCase): def setUp(self): self.application = ImiguApplication(helper.datadir, SH()) self.application.dbm._mtime = 1291167000 self.open_time = 1291167000 self.close_time = 1291186800 key = helper.sample_key() sample = helper.sample() sample[key]['timestamp'] = 1291167000 self.application.dbm.reportstore.update(sample) @patch.object(time, 'time') def test_archive_day_09_29(self, mock_time): mock_time.return_value = self.open_time - 1 # not open today = datetime.datetime.today() ret = self.application.scheduled_archive_day(today) self.assertFalse(ret) @patch.object(time, 'time') def test_archive_day_15_05_no_data(self, mock_time): mock_time.return_value = self.close_time + 300 self.application.dbm._mtime = self.close_time - 86400 today = datetime.datetime.today() ret = self.application.scheduled_archive_day(today) self.assertFalse(ret) @patch.object(time, 'time') def test_archive_day_15_05_01(self, mock_time): mock_time.return_value = self.close_time + 181 # closed more than 3 minutes self.application.dbm._mtime = self.close_time + 180 + 1 today = datetime.datetime.today() ret = self.application.scheduled_archive_day(today) self.assertTrue(ret) @patch.object(time, 'time') def test_archive_day_15_05_01_archived_before(self, mock_time): mock_time.return_value = self.close_time + 181 # closed more than 3 minutes self.application.archive_day_time = self.close_time + 180 today = datetime.datetime.today() ret = self.application.scheduled_archive_day(today) self.assertFalse(ret) @patch.object(time, 'time') def test_archive_minute_09_29(self, mock_time): mock_time.return_value = self.open_time - 1 # before open today = datetime.datetime.today() ret = self.application.scheduled_archive_minute(today) self.assertFalse(ret) @patch.object(time, 'time') def test_archive_minute_09_30(self, mock_time): mock_time.return_value = self.open_time today = datetime.datetime.today() ret = self.application.scheduled_archive_minute(today) self.assertTrue(ret) @patch.object(time, 'time') def test_archive_minute_14_30(self, mock_time): mock_time.return_value = self.close_time - 1800 # in session today = datetime.datetime.today() ret = self.application.scheduled_archive_minute(today) self.assertTrue(ret) @patch.object(time, 'time') def test_archive_minute_14_30_05_if_not_archived(self, mock_time): mock_time.return_value = self.close_time - 1795 # in session self.application.archive_minute_time = self.close_time - 1860 today = datetime.datetime.today() ret = self.application.scheduled_archive_minute(today) self.assertTrue(ret) @patch.object(time, 'time') def test_archive_minute_14_30_05_if_archived(self, mock_time): mock_time.return_value = self.close_time - 1795 # in session self.application.archive_minute_time = self.close_time - 1800 today = datetime.datetime.today() ret = self.application.scheduled_archive_minute(today) self.assertFalse(ret) @patch.object(time, 'time') def test_archive_minute_15_00(self, mock_time): mock_time.return_value = self.close_time today = datetime.datetime.today() ret = self.application.scheduled_archive_minute(today) self.assertTrue(ret) @patch.object(time, 'time') def test_archive_minute_15_03(self, mock_time): mock_time.return_value = self.close_time + 180 today = datetime.datetime.today() ret = self.application.scheduled_archive_minute(today) self.assertTrue(ret) @patch.object(time, 'time') def test_archive_minute_15_05_01(self, mock_time): mock_time.return_value = self.close_time + 300 + 1 # closed today = datetime.datetime.today() ret = self.application.scheduled_archive_minute(today) self.assertFalse(ret) @patch.object(time, 'time') def test_crontab_08_00_00(self, mock_time): mock_time.return_value = self.open_time - 3600 - 1800 today = datetime.datetime.fromtimestamp(time.time()) ret = self.application.scheduled_crontab_daily(today) self.assertTrue(ret) @patch.object(time, 'time') def test_crontab_08_00_01_if_not_running(self, mock_time): mock_time.return_value = self.open_time - 3600 - 1799 self.application.crontab_time = self.open_time - 86400 - 7200 today = datetime.datetime.fromtimestamp(time.time()) ret = self.application.scheduled_crontab_daily(today) self.assertTrue(ret) @patch.object(time, 'time') def test_crontab_09_30(self, mock_time): mock_time.return_value = self.open_time today = datetime.datetime.fromtimestamp(time.time()) ret = self.application.scheduled_crontab_daily(today) self.assertFalse(ret) def test_archive_day(self): r = { 'amount': 84596203520.0, 'close': 2856.9899999999998, 'high': 2880.5599999999999, 'low': 2851.9499999999998, 'name': u'\u4e0a\u8bc1\u6307\u6570', 'open': 2868.73, 'preclose': 2875.8600000000001, 'price': 2856.9899999999998, 'symbol': 'SH000001', 'volume': 75147848.0 } day = datetime.datetime.today() ts = time.mktime((day.year, day.month, day.day, 15, 0, 0, 0, 0, 0)) day_ts = time.mktime((day.year, day.month, day.day, 0, 0, 0, 0, 0, 0)) r['timestamp'] = ts r['time'] = str(datetime.datetime.fromtimestamp(ts)) data = {'SH000001': r} import zlib import marshal data = zlib.compress(marshal.dumps(data)) request = Request(None, 'put_reports', data) self.application(request) request = Request(None, 'archive_day') self.application(request) y = self.application.dbm.daystore.get('SH000001', 1) self.assertEqual(y[0]['time'], day_ts) self.assertTrue((y[0]['open'] - 2868.73) < 0.1**6) @patch.object(ImiguHandler, 'get_snapshot_index') def test_fix_report_when_archive(self, mock_index): # set to after hours: 15:30 implicates error data # some datafeed still sending data even market was closed. day = datetime.datetime.today() ts = time.mktime((day.year, day.month, day.day, 15, 30, 0, 0, 0, 0)) mock_index.return_value = (ts, 360) r = { 'amount': 84596203520.0, 'close': 2856.9899999999998, 'high': 2880.5599999999999, 'low': 2851.9499999999998, 'name': u'\u4e0a\u8bc1\u6307\u6570', 'open': 2868.73, 'preclose': 2875.8600000000001, 'price': 2856.9899999999998, 'symbol': 'SH000001', 'time': '2010-12-08 14:02:57', 'timestamp': 1291788177, 'volume': 75147848.0 } r['timestamp'] = ts r['time'] = str(datetime.datetime.fromtimestamp(ts)) data = {'SH000001': r} import zlib import marshal data = zlib.compress(marshal.dumps(data)) request = Request(None, 'put_reports', data) self.application(request) close_time = time.mktime( (day.year, day.month, day.day, 15, 0, 0, 0, 0, 0)) request = Request(None, 'archive_minute', data) self.application(request) r = self.application.dbm.get_report('SH000001') self.assertEqual(r['timestamp'], close_time) self.assertEqual(r['open'], 2868.73) @patch.object(ImiguHandler, 'get_snapshot_index') def test_archive_minute_at_open_time(self, mock_index): # set data time to pre-market(centralized competitive pricing) day = datetime.datetime.today() t1 = time.mktime((day.year, day.month, day.day, 9, 26, 0, 0, 0, 0)) open_time = time.mktime( (day.year, day.month, day.day, 9, 30, 0, 0, 0, 0)) mock_index.return_value = (open_time, 0) r = { 'amount': 10000.0, 'close': 0.0, 'high': 3000.0, 'low': 3000.0, 'name': u'\u4e0a\u8bc1\u6307\u6570', 'open': 3000.0, 'preclose': 2875.0, 'price': 3000.0, 'symbol': 'SH000001', 'volume': 900000.0 } r['timestamp'] = t1 r['time'] = str(datetime.datetime.fromtimestamp(t1)) data = {'SH000001': r} import zlib import marshal data = zlib.compress(marshal.dumps(data)) request = Request(None, 'put_reports', data) self.application(request) self.assertEqual(self.application.dbm.mtime, t1) request = Request(None, 'archive_minute') self.application(request) y = self.application.dbm.minutestore.get('SH000001') self.assertEqual(y[0]['time'], open_time) self.assertEqual(y[0]['price'], 3000.0) @patch.object(ImiguHandler, 'get_snapshot_index') def test_archive_minute_raise_at_wrong_index(self, mock_index): # set data time to pre-market(centralized competitive pricing) day = datetime.datetime.today() t1 = time.mktime((day.year, day.month, day.day, 9, 26, 0, 0, 0, 0)) mock_index.return_value = (t1, -4) request = Request(None, 'archive_minute') self.assertRaises(SnapshotIndexError, self.application, request) @patch.object(time, 'time') def test_get_snapshot_index(self, mock_time): mock_time.return_value = 1309829400 report_time = 1309829160 mintime, index = ImiguHandler.get_snapshot_index( 1309829400, report_time) self.assertEqual(mintime, 1309829400) self.assertEqual(index, 0)
import os import sys import time import tornado from tornado import ioloop from tornado.options import define, options sys.path[0:0] = ['..'] from datafeed.exchange import SH from datafeed.imiguserver import ImiguApplication from datafeed.server import Server, Request tornado.options.parse_command_line() app = ImiguApplication('/tmp/df', SH()) today = datetime.datetime.today() timestamp = int( time.mktime((today.year, today.month, today.day, 15, 0, 0, 0, 0, 0))) dt = datetime.datetime.fromtimestamp(timestamp) d = { 'SH000001': { 'amount': 84596203520.0, 'close': 2856.9899999999998, 'high': 2880.5599999999999, 'low': 2851.9499999999998, 'name': u'\u4e0a\u8bc1\u6307\u6570', 'open': 2868.73, 'preclose': 2875.8600000000001,