def test_mid_state_start(self): new_state = {StateKey.POSITION:203, StateKey.TIMESTAMP:self.base_timestamp+2} self.stream_handle = StringIO(CtdpfkParserUnitTestCase.TEST_DATA) self.parser = CtdpfkParser(self.config, new_state, self.stream_handle, self.pos_callback, self.pub_callback) # last one is the link to the data source result = self.parser.get_records(1) self.assert_result(result, 231, self.base_timestamp+3, self.particle_d)
def test_happy_path(self): """ Test the happy path of operations where the parser takes the input and spits out a valid data particle given the stream. """ self.stream_handle = StringIO(CtdpfkParserUnitTestCase.TEST_DATA) self.parser = CtdpfkParser(self.config, self.position, self.stream_handle, self.pos_callback, self.pub_callback) # last one is the link to the data source result = self.parser.get_records(1) self.assert_result(result, 147, self.base_timestamp, self.particle_a) result = self.parser.get_records(1) self.assert_result(result, 175, self.base_timestamp+1, self.particle_b) result = self.parser.get_records(1) self.assert_result(result, 203, self.base_timestamp+2, self.particle_c) result = self.parser.get_records(1) self.assert_result(result, 231, self.base_timestamp+3, self.particle_d) # no data left, dont move the position result = self.parser.get_records(1) self.assertEqual(result, []) self.assertEqual(self.parser._state[StateKey.POSITION], 231) self.assertEqual(self.position_callback_value[StateKey.POSITION], 231) self.assertEqual(self.position_callback_value[StateKey.TIMESTAMP], self.base_timestamp+3) self.assert_(isinstance(self.publish_callback_value, list)) self.assertEqual(self.publish_callback_value[0], self.particle_d)
def test_no_timestamp(self): """ There's no timestamp in the data! Ack! """ self.stream_handle = StringIO(CtdpfkParserUnitTestCase.NO_TIME_TEST_DATA) self.parser = CtdpfkParser(self.config, self.position, self.stream_handle, self.pos_callback, self.pub_callback) # last one is the link to the data source with self.assertRaises(SampleException): self.parser.get_records(1)
def test_bad_data(self): """ There's a bad sample in the data! Ack! Skip it! """ self.stream_handle = StringIO(CtdpfkParserUnitTestCase.BAD_TEST_DATA) self.parser = CtdpfkParser(self.config, self.position, self.stream_handle, self.pos_callback, self.pub_callback) # last one is the link to the data source result = self.parser.get_records(1) self.assert_result(result, 205, self.base_timestamp, self.particle_a)
def test_long_stream(self): self.stream_handle = StringIO(CtdpfkParserUnitTestCase.LONG_DATA) self.parser = CtdpfkParser(self.config, self.position, self.stream_handle, self.pos_callback, self.pub_callback) # last one is the link to the data source result = self.parser.get_records(44) self.assertEqual(result[-1], self.particle_z) self.assertEqual(self.parser._state[StateKey.POSITION], 1352) self.assertEqual(self.position_callback_value[StateKey.POSITION], 1352) self.assertEqual(self.position_callback_value[StateKey.TIMESTAMP], self.base_timestamp+43) self.assertEqual(self.publish_callback_value[-1], self.particle_z)
def test_set_state(self): new_state = {StateKey.POSITION:174, StateKey.TIMESTAMP:self.base_timestamp+1} self.stream_handle = StringIO(CtdpfkParserUnitTestCase.TEST_DATA) self.parser = CtdpfkParser(self.config, self.position, self.stream_handle, self.pos_callback, self.pub_callback) # last one is the link to the data source result = self.parser.get_records(1) self.assert_result(result, 147, self.base_timestamp, self.particle_a) self.parser.set_state(new_state) result = self.parser.get_records(1) self.assert_result(result, 218, self.base_timestamp+2, self.particle_c) result = self.parser.get_records(1) self.assert_result(result, 246, self.base_timestamp+3, self.particle_d)
def test_get_many(self): self.stream_handle = StringIO(CtdpfkParserUnitTestCase.TEST_DATA) self.parser = CtdpfkParser(self.config, self.position, self.stream_handle, self.pos_callback, self.pub_callback) # last one is the link to the data source result = self.parser.get_records(2) self.assertEqual(result, [self.particle_a, self.particle_b]) self.assertEqual(self.parser._state[StateKey.POSITION], 175) self.assertEqual(self.position_callback_value[StateKey.POSITION], 175) self.assertEqual(self.position_callback_value[StateKey.TIMESTAMP], self.base_timestamp+1) self.assertEqual(self.publish_callback_value[0], self.particle_a) self.assertEqual(self.publish_callback_value[1], self.particle_b)
def _build_parser(self, parser_state, infile): config = self._parser_config config.update({ 'particle_module': 'mi.dataset.parser.ctdpfk', 'particle_class': 'CtdpfkParserDataParticle' }) self._parser = CtdpfkParser(config, parser_state, infile, self._save_parser_state, self._data_callback) return self._parser
class CtdpfkParserUnitTestCase(ParserUnitTestCase): """ CTDPFK Parser unit test suite """ TEST_DATA = """ CTD turned on at 04/16/2013 01:00:02 CTD turned off at 04/16/2013 01:05:16 Profile 181 mmho/cm,Celsius,dbars,hz 1.5370,16.3184,12.630, 3122 1.5370,16.3174,12.640, 3127 1.5371,16.3170,12.640, 3132 1.5371,16.3171,12.640, 3137 Profile 181 """ LONG_DATA = """ CTD turned on at 04/16/2013 01:00:02 CTD turned off at 04/16/2013 01:05:16 Profile 181 mmho/cm,Celsius,dbars,hz 1.5370,16.3184,12.630, 3122 1.5370,16.3174,12.640, 3127 1.5371,16.3170,12.640, 3132 1.5371,16.3171,12.640, 3137 1.5371,16.3169,12.640, 3141 1.5371,16.3160,12.640, 3146 1.5372,16.3160,12.640, 3151 1.5372,16.3152,12.640, 3157 1.5372,16.3145,12.640, 3161 1.5372,16.3143,12.640, 3165 1.5372,16.3139,12.640, 3170 1.5373,16.3139,12.640, 3175 1.5373,16.3132,12.640, 3180 1.5373,16.3132,12.640, 3184 1.5373,16.3125,12.640, 3187 1.5373,16.3121,12.640, 3192 1.5373,16.3127,12.640, 3196 1.5373,16.3118,12.630, 3200 1.5374,16.3117,12.640, 3205 1.5374,16.3108,12.640, 3209 1.5374,16.3111,12.630, 3213 1.5374,16.3102,12.630, 3217 1.5374,16.3106,12.640, 3221 1.5375,16.3095,12.640, 3226 1.5374,16.3096,12.640, 3229 1.5375,16.3097,12.640, 3234 1.5375,16.3092,12.640, 3237 1.5375,16.3086,12.640, 3242 1.5375,16.3079,12.640, 3246 1.5376,16.3081,12.640, 3249 1.5376,16.3081,12.640, 3253 1.5376,16.3077,12.640, 3256 1.5376,16.3072,12.640, 3260 1.5376,16.3069,12.640, 3264 1.5376,16.3072,12.640, 3268 1.5376,16.3059,12.640, 3272 1.5376,16.3067,12.640, 3275 1.5377,16.3063,12.640, 3279 1.5377,16.3057,12.640, 3282 1.5377,16.3058,12.640, 3286 1.5377,16.3053,12.640, 3289 1.5377,16.3052,12.640, 3294 1.5377,16.3045,12.640, 3297 1.5371,16.3160,12.640, 3146 Profile 181 """ LEFTOVER = """ 1.5377,16.3051,12.640, 3303 1.5377,16.3042,12.640, 3307 1.5377,16.3041,12.640, 3310 1.5377,16.3034,12.640, 3313 1.5377,16.3039,12.640, 3317 1.5377,16.3038,12.640, 3319 1.5377,16.3036,12.640, 3322 1.5378,16.3030,12.640, 3326 1.5378,16.3030,12.640, 3329 1.5378,16.3028,12.640, 3332 1.5378,16.3029,12.640, 3336 1.5378,16.3026,12.640, 3340 1.5378,16.3020,12.640, 3343 1.5379,16.3026,12.640, 3345 1.5379,16.3018,12.640, 3349 1.5378,16.3017,12.640, 3352 1.5378,16.3017,12.640, 3355 1.5379,16.3010,12.640, 3358 1.5378,16.3016,12.640, 3361 1.5379,16.3006,12.640, 3363 1.5379,16.3009,12.640, 3367 1.5378,16.3009,12.640, 3369 1.5378,16.3008,12.640, 3372 1.5379,16.3006,12.640, 3374 1.5379,16.3005,12.640, 3377 1.5379,16.3002,12.640, 3379 1.5379,16.3007,12.640, 3382 1.5379,16.2999,12.640, 3383 1.5379,16.2998,12.640, 3385 1.5379,16.2995,12.630, 3388 1.5379,16.3002,12.640, 3391 1.5371,16.3160,12.640, 3146 """ BAD_TEST_DATA = """ CTD turned on at 04/16/2013 01:00:02 CTD turnfed off at 04/16/2013 01:05:16 Profile 181 mmho/cm,Celsius,dbars,hz 1.5283,1f6.3574,0.970, 2739 1.5283,16.3596,x0.970, 2745 1.5370,16.3184,12.630, 3122 1.5282,16.3604,0.970, 2f752 Profiled 181.3 """ NO_TIME_TEST_DATA = """ Profile 181 mmho/cm,Celsius,dbars,hz 1.5282,16.3541,0.970, 2724 1.5283,16.3574,0.970, 2739 1.5283,16.3596,0.970, 2745 1.5283,16.3603,0.970, 2749 1.5282,16.3604,0.970, 2752 """ def pos_callback(self, pos, file_ingested): """ Call back method to watch what comes in via the position callback """ self.position_callback_value = pos self.file_ingested = file_ingested def pub_callback(self, pub): """ Call back method to watch what comes in via the publish callback """ self.publish_callback_value = pub def setUp(self): ParserUnitTestCase.setUp(self) self.config = { DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.ctdpfk', DataSetDriverConfigKeys.PARTICLE_CLASS: 'CtdpfkParserDataParticle' } # not a DataSourceLocation...its just the parser self.position = {StateKey.POSITION:0, StateKey.TIMESTAMP:0.0} # Gonna need the test cases to make some of these # self.stream_handle = StringIO(TEST_DATA) # self.parser = CtdpfParser(config, self.position, self.stream_handle) # last one is the link to the data source self.base_timestamp = 3575062802.0 self.particle_a = CtdpfkParserDataParticle("1.5370,16.3184,12.630, 3122", internal_timestamp=self.base_timestamp) self.particle_b = CtdpfkParserDataParticle("1.5370,16.3174,12.640, 3127", internal_timestamp=self.base_timestamp+1) self.particle_c = CtdpfkParserDataParticle("1.5371,16.3170,12.640, 3132", internal_timestamp=self.base_timestamp+2) self.particle_d = CtdpfkParserDataParticle("1.5371,16.3171,12.640, 3137", internal_timestamp=self.base_timestamp+3) self.particle_e = CtdpfkParserDataParticle("1.5371,16.3169,12.640, 3141", internal_timestamp=self.base_timestamp+60) self.particle_z = CtdpfkParserDataParticle("1.5371,16.3160,12.640, 3146", internal_timestamp=self.base_timestamp+43) self.position_callback_value = None self.publish_callback_value = None def assert_result(self, result, position, timestamp, particle): self.assertEqual(result, [particle]) self.assertEqual(self.parser._state[StateKey.POSITION], position) self.assertEqual(self.position_callback_value[StateKey.POSITION], position) self.assertEqual(self.position_callback_value[StateKey.TIMESTAMP], timestamp) self.assert_(isinstance(self.publish_callback_value, list)) self.assertEqual(self.publish_callback_value[0], particle) def test_happy_path(self): """ Test the happy path of operations where the parser takes the input and spits out a valid data particle given the stream. """ self.stream_handle = StringIO(CtdpfkParserUnitTestCase.TEST_DATA) self.parser = CtdpfkParser(self.config, self.position, self.stream_handle, self.pos_callback, self.pub_callback) # last one is the link to the data source result = self.parser.get_records(1) self.assert_result(result, 147, self.base_timestamp, self.particle_a) result = self.parser.get_records(1) self.assert_result(result, 175, self.base_timestamp+1, self.particle_b) result = self.parser.get_records(1) self.assert_result(result, 203, self.base_timestamp+2, self.particle_c) result = self.parser.get_records(1) self.assert_result(result, 231, self.base_timestamp+3, self.particle_d) # no data left, dont move the position result = self.parser.get_records(1) self.assertEqual(result, []) self.assertEqual(self.parser._state[StateKey.POSITION], 231) self.assertEqual(self.position_callback_value[StateKey.POSITION], 231) self.assertEqual(self.position_callback_value[StateKey.TIMESTAMP], self.base_timestamp+3) self.assert_(isinstance(self.publish_callback_value, list)) self.assertEqual(self.publish_callback_value[0], self.particle_d) def test_get_many(self): self.stream_handle = StringIO(CtdpfkParserUnitTestCase.TEST_DATA) self.parser = CtdpfkParser(self.config, self.position, self.stream_handle, self.pos_callback, self.pub_callback) # last one is the link to the data source result = self.parser.get_records(2) self.assertEqual(result, [self.particle_a, self.particle_b]) self.assertEqual(self.parser._state[StateKey.POSITION], 175) self.assertEqual(self.position_callback_value[StateKey.POSITION], 175) self.assertEqual(self.position_callback_value[StateKey.TIMESTAMP], self.base_timestamp+1) self.assertEqual(self.publish_callback_value[0], self.particle_a) self.assertEqual(self.publish_callback_value[1], self.particle_b) def test_bad_data(self): """ There's a bad sample in the data! Ack! Skip it! """ self.stream_handle = StringIO(CtdpfkParserUnitTestCase.BAD_TEST_DATA) self.parser = CtdpfkParser(self.config, self.position, self.stream_handle, self.pos_callback, self.pub_callback) # last one is the link to the data source result = self.parser.get_records(1) self.assert_result(result, 205, self.base_timestamp, self.particle_a) def test_no_timestamp(self): """ There's no timestamp in the data! Ack! """ self.stream_handle = StringIO(CtdpfkParserUnitTestCase.NO_TIME_TEST_DATA) self.parser = CtdpfkParser(self.config, self.position, self.stream_handle, self.pos_callback, self.pub_callback) # last one is the link to the data source with self.assertRaises(SampleException): self.parser.get_records(1) def test_long_stream(self): self.stream_handle = StringIO(CtdpfkParserUnitTestCase.LONG_DATA) self.parser = CtdpfkParser(self.config, self.position, self.stream_handle, self.pos_callback, self.pub_callback) # last one is the link to the data source result = self.parser.get_records(44) self.assertEqual(result[-1], self.particle_z) self.assertEqual(self.parser._state[StateKey.POSITION], 1352) self.assertEqual(self.position_callback_value[StateKey.POSITION], 1352) self.assertEqual(self.position_callback_value[StateKey.TIMESTAMP], self.base_timestamp+43) self.assertEqual(self.publish_callback_value[-1], self.particle_z) def test_mid_state_start(self): new_state = {StateKey.POSITION:203, StateKey.TIMESTAMP:self.base_timestamp+2} self.stream_handle = StringIO(CtdpfkParserUnitTestCase.TEST_DATA) self.parser = CtdpfkParser(self.config, new_state, self.stream_handle, self.pos_callback, self.pub_callback) # last one is the link to the data source result = self.parser.get_records(1) self.assert_result(result, 231, self.base_timestamp+3, self.particle_d) def test_set_state(self): new_state = {StateKey.POSITION:174, StateKey.TIMESTAMP:self.base_timestamp+1} self.stream_handle = StringIO(CtdpfkParserUnitTestCase.TEST_DATA) self.parser = CtdpfkParser(self.config, self.position, self.stream_handle, self.pos_callback, self.pub_callback) # last one is the link to the data source result = self.parser.get_records(1) self.assert_result(result, 147, self.base_timestamp, self.particle_a) self.parser.set_state(new_state) result = self.parser.get_records(1) self.assert_result(result, 218, self.base_timestamp+2, self.particle_c) result = self.parser.get_records(1) self.assert_result(result, 246, self.base_timestamp+3, self.particle_d)