def test_nested_parser_template(self):
TickerModel = Processor({
'amount':
0.66075377,
'datetime':
Instances['datetime'],
'fee':
None,
'id':
None,
'info':
['659.100000', '0.66075377', 1517364874.3597, 's', 'l', ''],
})
TickerListProcessor = Processor({
'tickers': [TickerModel],
})
tmp = {
'tickers': [{
'amount': '0.66075377',
'datetime': '1517408042',
'fee': None,
'id': None,
'info': '[]',
}]
}
result = TickerListProcessor(tmp)
self.assertEqual(result['tickers'][0]['amount'], 0.66075377)
self.assertEqual(result['tickers'][0]['datetime'],
datetime(2018, 1, 31, 6, 14, 2))
self.assertEqual(result['tickers'][0]['info'], [])
self.assertEqual(result['tickers'][0]['fee'], None)
self.assertEqual(len(result['tickers']), 1)
def test_processor_obj_deep(self):
p = Processor({
'deep': {
'start': Instances['datetime'],
'end': Instances['datetime'],
}
})
childProcessor = Processor({
'start': Instances['datetime'],
'end': Instances['datetime'],
})
p3 = Processor({'deep': childProcessor})
p4 = Processor([p3])
tmp = {
'deep': {
'start': 1576226168.818243,
'end': 1576226168.818243,
}
}
resultFull = p(tmp)
resultNested = p3(tmp)
resultNestedArr = p4([tmp])
expected = {
'deep': {
'start': datetime(2019, 12, 13, 0, 36, 8, 818243),
'end': datetime(2019, 12, 13, 0, 36, 8, 818243),
}
}
self.assertEqual(resultNested, expected)
self.assertEqual(resultFull, expected)
self.assertEqual(resultNestedArr, [expected])
def test_processor_chain_complex(self):
p0 = Processor(lambda partial, options={}: '%s -> %s' %
(options, partial * 34))
p = Processor.flow([
float, int, lambda partial, options={}: '%s -> %s' %
(options, partial * 34)
])
self.assertEqual(p('1.0'), '1.0 -> 34')
def test_json_dump_datetime(self):
# Test the json datetime encoder
pt = Processor(Instances['datetime'])
tmp = pt(1517408265547)
self.assertEqual(datetime(2018, 1, 31, 6, 17, 45, 547000), tmp)
res = Parser('', {'datetime': tmp})
self.assertEqual('{"datetime": "2018-01-31T06:17:45.547"}', res)
def test_s_ts(self):
# Parse ms timestamp format (like from python)
pt = Processor(Instances['datetime'])
tmp = 1517408042.277897
self.assertEqual(datetime(2018, 1, 31, 6, 14, 2, 277897), pt(tmp))
tmp = 1517408042
self.assertEqual(datetime(2018, 1, 31, 6, 14, 2), pt(tmp))
def test_none(self):
pt = Processor({
'amount':
0.66075377,
'datetime':
Instances['datetime'],
'fee':
None,
'id':
None,
'price':
10000.0,
'info':
['659.100000', '0.66075377', 1517364874.3597, 's', 'l', ''],
})
tmp = {
'amount': 0.66075377,
'datetime': '2018-01-31T02:14:34.359Z',
'fee': None,
'id': None,
'price': '$10,000.00',
'info':
['659.100000', '0.66075377', 1517364874.3597, 's', 'l', ''],
}
self.assertEqual(pt(tmp), tmp)
def test_function_conversion(self):
# Executes the function passed in rather than using builtin
def convert_multiply(var):
return int(var) * 10
pt = Processor(convert_multiply)
self.assertEqual(10, pt('1'))
def test_processor_chain_lambda_original_value(self):
partA = lambda partial, options={}: {'total': sum(partial)}
def partB(partial, options={}):
partial.update({'fraction': options[1] / partial['total']})
return partial
def partC(partial, options={}):
return {'fraction': options[1] / partial['total']}
pReduce = Processor.flow([partA, partC], aggregator='reduce')
pReduceMerge = Processor.flow([partA, partB], aggregator='reduce')
pMerge = Processor.flow([partA, partC], aggregator='merge')
resultReduce = pReduce([10, 2, 100])
resultReduceMerge = pReduceMerge([10, 2, 100])
resultMerge = pMerge([10, 2, 100])
exp = {'total': 112, 'fraction': 2 / 112}
self.assertEqual({'fraction': 2 / 112}, resultReduce)
self.assertEqual(exp, resultReduceMerge)
self.assertEqual(exp, resultMerge)
self.assertEqual(resultReduceMerge, resultMerge)
def test_processor_obj(self):
p = Processor({
'start': Instances['datetime'],
'end': Instances['datetime'],
})
result = p({
'start': 1576226168.818243,
'end': 1576226168.818243,
})
expected = {
'start': datetime(2019, 12, 13, 0, 36, 8, 818243),
'end': datetime(2019, 12, 13, 0, 36, 8, 818243),
}
self.assertEqual(result, expected)
def test_processor_arr(self):
p = Processor([Instances['datetime']])
self.assertEqual(p([1576226168.818243]),
[datetime(2019, 12, 13, 0, 36, 8, 818243)])
def test_custom_processor_aggs(self):
getLastResult = lambda res: res[-1]['result']
proc = Processor(lambda p, options={}: {'result': [1, 2, 3]},
aggregator=getLastResult)
self.assertEqual(proc(None), [1, 2, 3])
def test_datetime_to_str_cast(self):
# Test encoding a datetime to a float
pt = Processor(str)
tmp = pt(datetime(2018, 1, 31, 6, 17, 45, 547000))
self.assertEqual("2018-01-31T06:17:45.547", tmp)
def test_processor_chain_complex(self):
p = Processor.flow([float, int, lambda x: x * 34])
self.assertEqual(p('1.0'), 34)
def test_processor_chain_then(self):
p = Processor(float)
self.assertEqual(p('1.0'), 1.0)
p.then(int)
self.assertEqual(p('1.0'), 1)
def test_ms_ts(self):
# Parse s timestamp format (like from python)
pt = Processor(Instances['datetime'])
tmp = 1517408265547
self.assertEqual(datetime(2018, 1, 31, 6, 17, 45, 547000), pt(tmp))
def test_lambda_conversion(self):
# Executes the lambda passed in rather than using builtin
pt = Processor(lambda x: int(x) * 10)
self.assertEqual(10, pt('1'))
def test_processor(self):
p = Processor(int)
self.assertEqual(p('1.0'), 1)
def test_datetime_to_int_cast(self):
# Test the timestamp encoding a datetime to int
pt = Processor(int)
tmp = pt(datetime(2018, 1, 31, 6, 17, 45, 547000))
self.assertEqual(1517408265547, tmp)
def test_processor_chain_flow(self):
p = Processor.flow([float, int])
self.assertEqual(p('1.0'), 1)
# Base case of 1 elem
p = Processor.flow([int])
self.assertEqual(p('1.0'), 1)
def test_datetime_to_float_cast(self):
# Test encoding a datetime to a float
pt = Processor(float)
tmp = pt(datetime(2018, 1, 31, 6, 14, 2, 277897))
self.assertEqual(1517408042.277897, tmp)
def test_processor_dt(self):
p = Processor(Instances['datetime'])
self.assertEqual(p(1576226168.818243),
datetime(2019, 12, 13, 0, 36, 8, 818243))
def test_json_encoder_processor_resolve(self):
res = json.dumps(
{'date': Processor(datetime(2018, 1, 31, 6, 17, 45, 547000))},
cls=JSONEncoder)
self.assertEqual(res, '{"date": null}')
