def load_from_dict(cls, msg):
if (isinstance(msg, dict) and 'v' in msg and 'action' in msg
and 'seq' in msg and 'irt' in msg and 'payload' in msg
and isinstance(msg['v'], int) and msg['v'] == cls._version_
and isinstance(msg['action'], str)
and msg['action'] == cls._action_.value
and isinstance(msg['payload'], dict)
and 'uri' in msg['payload']
and isinstance(msg['payload']['uri'], dict)
and 'uri' in msg['payload']['uri']
and 'type' in msg['payload']['uri']
and 'start' in msg['payload'] and 'end' in msg['payload']
and 'data' in msg['payload']):
if msg['payload']['uri']['type'] == Metrics.DATASOURCE.value:
m_type = Metrics.DATASOURCE
elif msg['payload']['uri']['type'] == Metrics.DATAPOINT.value:
m_type = Metrics.DATAPOINT
else:
raise TypeError('Invalid metric type')
start = timeuuid.TimeUUID(s=msg['payload']['start'])
end = timeuuid.TimeUUID(s=msg['payload']['end'])
data = msg['payload']['data']
seq = timeuuid.TimeUUID(s=msg['seq'])
irt = timeuuid.TimeUUID(
s=msg['irt']) if msg['irt'] != None else None
return cls(uri=msg['payload']['uri']['uri'],
m_type=m_type,
start=start,
end=end,
data=data,
seq=seq,
irt=irt)
else:
raise TypeError('Could not load message, invalid type')
async def test_metrics_updated_two_tm_activated_with_them(self):
''' metrics_updated should generate two new tasks '''
tm1 = transfer_methods.transfermethod(f=noop)
tm2 = transfer_methods.transfermethod(f=noop)
tm1._decorate_method(tm1._f)
tm2._decorate_method(tm2._f)
tmi = TransferMethodsIndex()
self.assertTrue(tmi.add_tm(tm1))
self.assertTrue(tmi.add_tm(tm2))
self.assertTrue(tm1.mid in tmi._disabled_methods)
self.assertTrue(tm2.mid in tmi._disabled_methods)
self.assertTrue(await tmi.enable_all())
tm1.schedule = OnUpdateSchedule(activation_metrics=Datasource('uri'))
tm2.schedule = OnUpdateSchedule(activation_metrics=Datasource('uri'))
self.assertTrue(tm1.mid in tmi._enabled_methods)
self.assertTrue(tm2.mid in tmi._enabled_methods)
self.assertIsNotNone(tmi._enabled_methods[tm1.mid]['first'])
self.assertIsNotNone(tmi._enabled_methods[tm2.mid]['first'])
metrics = [Datasource('uri'), Datasource('uri2')]
t = timeuuid.TimeUUID()
current_task = asyncio.Task.current_task()
tasks = asyncio.Task.all_tasks()
self.assertEqual(len(tasks), 1) #this task
tmi.metrics_updated(t=t, metrics=metrics, irt=timeuuid.TimeUUID())
tasks = asyncio.Task.all_tasks()
self.assertEqual(len(tasks), 3) #two tm activated
[
task.cancel() for task in asyncio.Task.all_tasks()
if task != current_task
]
def test_create_timeuuid_with_string(self):
''' creating a TimeUUID with a hex uuid1 should succeed '''
for i in range(1, 1000):
u = uuid.uuid1()
t = timeuuid.TimeUUID(s=u.hex)
self.assertEqual(t, u)
t = timeuuid.TimeUUID(s=str(u))
self.assertEqual(t, u)
async def _periodic_transfer_method_call(self, mid):
t = timeuuid.TimeUUID()
await asyncio.sleep(60 - t.timestamp % 60)
t = timeuuid.TimeUUID(t=t.timestamp + (60 - t.timestamp % 60))
localtime = time.localtime(t.timestamp)
tm_info = self.get_tm_info(mid)
if (tm_info and tm_info['enabled']):
if tm_info['tm'].schedule.meets(t=localtime):
logging.logger.debug('periodic_transfer_method_call ' +
mid.hex)
await tm_info['tm'].run(t=t, metrics=[])
asyncio.ensure_future(self._periodic_transfer_method_call(mid))
def test_create_timeuuid_with_uuid4_string_should_fail(self):
''' creating a TimeUUID with a hex uuid4 should fail'''
for i in range(1, 100):
u = uuid.uuid4()
with self.assertRaises(ValueError) as cm:
t = timeuuid.TimeUUID(s=u.hex)
self.assertEqual(str(cm.exception), 'Invalid UUID type')
for fn in [uuid.uuid3, uuid.uuid5]:
for i in range(1, 100):
u = fn(uuid.NAMESPACE_DNS, str(os.urandom(10)))
with self.assertRaises(ValueError) as cm:
t = timeuuid.TimeUUID(s=u.hex)
self.assertEqual(str(cm.exception), 'Invalid UUID type')
def test_create_timeuuid_with_timestamp_no_random(self):
''' creating a TimeUUID object with a timestamp and a predictable value '''
for i in range(1, 1000):
now = time.time()
t = timeuuid.TimeUUID(t=now, random=False)
self.assertTrue(isinstance(t, timeuuid.TimeUUID))
t2 = timeuuid.TimeUUID(t=now, random=False)
self.assertTrue(isinstance(t2, timeuuid.TimeUUID))
self.assertEqual(t, t2)
self.assertEqual(t.timestamp, int(now * 1e6) / 1e6)
self.assertTrue(t.clock_seq == 0 and t.node == 0)
self.assertFalse(t.clock_seq == 0x80 and t.node == 0x808080808080)
self.assertFalse(t.clock_seq == 0x3f7f
and t.node == 0x7f7f7f7f7f7f)
def data(self, data):
if (isinstance(data, list) and all(
isinstance(item, list) and len(item) == 2 and
(validation.validate_dp_value(item[1]) if self.m_type ==
Metrics.DATAPOINT else validation.validate_ds_value(item[1]))
for item in data)):
if self.m_type == Metrics.DATAPOINT:
self._data = [(timeuuid.TimeUUID(s=row[0]),
decimal.Decimal(str(row[1]))) for row in data]
else:
self._data = [(timeuuid.TimeUUID(s=row[0]), row[1])
for row in data]
else:
raise TypeError('Invalid data')
def test_magic_methods_comparisson(self):
''' Check magic methods comparisson works '''
for i in range(1, 10000):
t1 = timeuuid.TimeUUID(t=i, random=False)
t2 = timeuuid.TimeUUID(t=i, random=False)
t3 = timeuuid.TimeUUID(t=i + 1, random=False)
t4 = timeuuid.TimeUUID(t=i + 1, random=False)
self.assertTrue(t1 <= t2)
self.assertFalse(t1 < t2)
self.assertTrue(t1 < t3)
self.assertTrue(t1 <= t3)
self.assertTrue(t3 > t2)
self.assertTrue(t3 >= t2)
self.assertTrue(t3 >= t4)
self.assertFalse(t3 > t4)
def load_from_dict(cls, msg):
if (isinstance(msg, dict) and 'v' in msg and 'action' in msg
and 'seq' in msg and 'irt' in msg and 'payload' in msg
and isinstance(msg['v'], int) and msg['v'] == cls._version_
and isinstance(msg['action'], str)
and msg['action'] == cls._action_.value
and isinstance(msg['payload'], dict) and 't' in msg['payload']
and 'uris' in msg['payload']):
t = timeuuid.TimeUUID(s=msg['payload']['t'])
uris = msg['payload']['uris']
seq = timeuuid.TimeUUID(s=msg['seq'])
irt = timeuuid.TimeUUID(
s=msg['irt']) if msg['irt'] != None else None
return cls(t=t, uris=uris, seq=seq, irt=irt)
else:
raise TypeError('Could not load message, invalid type')
def test_is_message_sequence_success(self):
''' is_message_sequence should return False if param is no a valid sequence '''
params = [
timeuuid.TimeUUID(),
]
for param in params:
self.assertTrue(validation.is_message_sequence(param))
def test_TimeUUID_modify_timestamp_should_fail(self):
''' timestamp attribute cannot be modified '''
t = time.time()
tu = timeuuid.TimeUUID(t=t)
with self.assertRaises(TypeError) as cm:
tu.timestamp = 4
self.assertEqual(tu.timestamp, int(t * 1e6) / 1e6)
def test_create_Transaction_object(self):
''' Check creation of a new Transaction object '''
t = timeuuid.TimeUUID()
tr = Transaction(t=t)
self.assertTrue(tr.tm < time.monotonic())
self.assertTrue(isinstance(tr.tid, uuid.UUID))
self.assertEqual(tr.t, t)
self.assertEqual(tr._dirty, set())
def test_create_timeuuid_with_timestamp_highest(self):
''' creating a TimeUUID object with a timestamp and highest possible value '''
for i in range(1, 1000):
now = time.time()
t = timeuuid.TimeUUID(t=now, highest=True)
self.assertTrue(isinstance(t, timeuuid.TimeUUID))
self.assertEqual(t.timestamp, int(now * 1e6) / 1e6)
self.assertFalse(t.clock_seq == 0 and t.node == 0)
self.assertFalse(t.clock_seq == 0x80 and t.node == 0x808080808080)
self.assertTrue(t.clock_seq == 0x3f7f and t.node == 0x7f7f7f7f7f7f)
def test_create_timeuuid_with_timestamp(self):
''' creating a TimeUUID object with a timestamp and randomized '''
for i in range(1, 1000):
now = time.time()
t = timeuuid.TimeUUID(t=now)
self.assertTrue(isinstance(t, timeuuid.TimeUUID))
self.assertEqual(t.timestamp, int(now * 1e6) / 1e6)
# if this fails, you won the lottery
self.assertFalse(t.clock_seq == 0 and t.node == 0)
self.assertFalse(t.clock_seq == 0x80 and t.node == 0x808080808080)
self.assertFalse(t.clock_seq == 0x3f7f
and t.node == 0x7f7f7f7f7f7f)
def test_lowest_uuid_comparisson(self):
''' a TimeUUID object with lowest flag should never be greater than other with same t '''
now = time.time()
lowest = timeuuid.TimeUUID(t=now, lowest=True)
# some random tests
for i in range(1, 10000):
t2 = timeuuid.TimeUUID(t=now)
self.assertTrue(lowest <= t2)
# prepared tests
lowest = timeuuid.TimeUUID(s='00000000-0000-1000-8080-808080808080')
comparing_bytes = [
timeuuid.TimeUUID(s='00000000-0000-1000-8081-808080808080'),
timeuuid.TimeUUID(s='00000000-0000-1000-8080-818080808080'),
timeuuid.TimeUUID(s='00000000-0000-1000-8080-808180808080'),
timeuuid.TimeUUID(s='00000000-0000-1000-8080-808081808080'),
timeuuid.TimeUUID(s='00000000-0000-1000-8080-808080818080'),
timeuuid.TimeUUID(s='00000000-0000-1000-8080-808080808180'),
timeuuid.TimeUUID(s='00000000-0000-1000-8080-808080808081')
]
for other in comparing_bytes:
self.assertTrue(lowest < other)
def test_highest_uuid_comparisson(self):
''' a TimeUUID object with highest flag should never be less than other with same t '''
now = time.time()
highest = timeuuid.TimeUUID(t=now, highest=True)
# some random tests
for i in range(1, 10000):
t2 = timeuuid.TimeUUID(t=now)
self.assertTrue(highest >= t2)
# prepared tests
highest = timeuuid.TimeUUID(s='00000000-0000-1000-bfff-7f7f7f7f7f7f')
comparing_bytes = [
timeuuid.TimeUUID(s='00000000-0000-1000-bffe-7f7f7f7f7f7f'),
timeuuid.TimeUUID(s='00000000-0000-1000-bfff-7e7f7f7f7f7f'),
timeuuid.TimeUUID(s='00000000-0000-1000-bfff-7f7e7f7f7f7f'),
timeuuid.TimeUUID(s='00000000-0000-1000-bfff-7f7f7e7f7f7f'),
timeuuid.TimeUUID(s='00000000-0000-1000-bfff-7f7f7f7e7f7f'),
timeuuid.TimeUUID(s='00000000-0000-1000-bfff-7f7f7f7f7e7f'),
timeuuid.TimeUUID(s='00000000-0000-1000-bfff-7f7f7f7f7f7e'),
]
for other in comparing_bytes:
self.assertTrue(highest > other)
def test_create_timeuuid_defaults(self):
''' creating a TimeUUID object with defaults should generate an object with the current
timestamp, and randomized '''
for i in range(1, 1000):
now = time.time()
t = timeuuid.TimeUUID()
self.assertTrue(isinstance(t, timeuuid.TimeUUID))
# if this fails, you won the lottery
self.assertFalse(t.clock_seq == 0 and t.node == 0)
self.assertFalse(t.clock_seq == 0x80 and t.node == 0x808080808080)
self.assertFalse(t.clock_seq == 0x3f7f
and t.node == 0x7f7f7f7f7f7f)
def load_from_dict(cls, msg):
if (isinstance(msg, dict) and 'v' in msg and 'action' in msg
and 'seq' in msg and 'irt' in msg and 'payload' in msg
and isinstance(msg['v'], int)
and isinstance(msg['action'], str)
and msg['action'] == cls._action_.value
and isinstance(msg['payload'], dict)
and 'status' in msg['payload'] and 'error' in msg['payload']
and 'reason' in msg['payload']):
status = msg['payload']['status']
error = msg['payload']['error']
reason = msg['payload']['reason']
seq = timeuuid.TimeUUID(s=msg['seq'])
irt = timeuuid.TimeUUID(
s=msg['irt']) if msg['irt'] != None else None
return cls(status=status,
error=error,
reason=reason,
seq=seq,
irt=irt)
else:
raise TypeError('Could not load message, invalid type')
def test_add_dirty_item(self):
''' adding a dirty item should add the element to the _dirty set '''
class MyInterface:
def __init__(self):
self._counter = 0
def _tr_discard(self, tid):
self._counter -= 1
interfaces = [MyInterface() for _ in range(1, 10)]
t = timeuuid.TimeUUID()
tr = Transaction(t)
[tr.add_dirty_item(item) for item in interfaces]
self.assertEqual(len(tr._dirty), 9)
[tr.add_dirty_item(item) for item in interfaces]
self.assertEqual(len(tr._dirty), 9)
async def test_discard_transaction_automatically_if_out_of_context(self):
''' at context exit, discard() will be called on the Transaction object automatically '''
class MyInterface:
def __init__(self):
self._counter = 0
def _tr_discard(self, tid):
self._counter -= 1
interfaces = [MyInterface() for _ in range(1, 10)]
t = timeuuid.TimeUUID()
async with Transaction(t) as tr:
[self.assertEqual(item._counter, 0) for item in interfaces]
[tr.add_dirty_item(item) for item in interfaces]
self.assertNotEqual(tr._dirty, set())
[self.assertEqual(item._counter, -1) for item in interfaces]
self.assertEqual(tr._dirty, set())
async def test_commit_transaction(self):
''' Commit transaction should execute _tr_commit coroutines for each dirty item '''
class MyInterface:
def __init__(self):
self._counter = 0
async def _tr_commit(self, tid):
self._counter += 1
t = timeuuid.TimeUUID()
tr = Transaction(t=t)
interfaces = [MyInterface() for _ in range(1, 10)]
[self.assertEqual(item._counter, 0) for item in interfaces]
[tr.add_dirty_item(item) for item in interfaces]
await tr.commit()
[self.assertEqual(item._counter, 1) for item in interfaces]
self.assertEqual(tr._dirty, set())
async def hook(self, metric):
result = await prproc.hook_to_metric(metric)
if result['hooked']:
self._hooked.add(metric)
if result['exists']:
#sync future
now = timeuuid.TimeUUID()
await self.get(metric,
start=now,
end=timeuuid.MAX_TIMEUUID,
count=200)
else:
self._add_synced_range(metric,
t=time.monotonic(),
its=timeuuid.MIN_TIMEUUID,
ets=timeuuid.MAX_TIMEUUID)
return result
def test_is_message_sequence_failure(self):
''' is_message_sequence should return False if param is no a valid sequence '''
params = [
'NaN',
decimal.Decimal('23'),
decimal.Decimal(23),
-1,
-1.232,
-1e2,
2**32,
'1' * 2**7,
'1.1',
'1.5e+4',
'1.005E+43',
'-1.3e-34',
'1e4',
'-1e4',
'-1.3e-3',
'-1.3E-34',
'1E4',
'-1E4',
'-1.3E-3',
'+1.3E+3',
'1.5e4\n',
' 23 ',
' 32\n',
'32\n',
'1' * 2**7 + '1',
timeuuid.TimeUUID().hex,
uuid.uuid1(),
uuid.uuid1().hex,
uuid.uuid4().hex[0:20],
uuid.uuid1().hex[0:10],
uuid.uuid1().hex[0:20],
uuid.uuid4(),
{'set'},
{
'a': 'dict'
},
['a', 'list'],
('a', 'tuple'),
None,
1,
]
for param in params:
self.assertFalse(validation.is_message_sequence(param))
async def enable_tm(self, mid):
tm_info = self._disabled_methods.pop(mid, None)
if tm_info:
logging.logger.debug('enabling tm ' + mid.hex)
try:
logging.logger.debug('tm activation metrics: ' + str(
[m.uri
for m in tm_info['tm'].schedule.activation_metrics]))
for metric in tm_info['tm'].schedule.activation_metrics:
result = await metric.session.store.hook(metric=metric)
if result['hooked'] == False:
logging.logger.error(
'Error syncing metric {}. Aborting tm initialization'
.format(metric.uri))
return False
except (exceptions.SessionException,
exceptions.SessionNotFoundException) as e:
logging.logger.error(
'Error syncing metric {}. Aborting tm initialization'.
format(metric.uri))
logging.logger.error('Error: {}.'.format(e.msg))
self._disabled_methods[mid] = tm_info
if tm_info['first'] == None:
asyncio.ensure_future(self._retry_failed())
return False
else:
if tm_info['first'] == None:
now = pd.Timestamp('now', tz='utc')
tm_info['first'] = now
if tm_info['tm'].schedule.exec_on_load:
t = timeuuid.TimeUUID()
asyncio.ensure_future(tm_info['tm'].run(t=t,
metrics=[]))
self._enabled_methods[mid] = tm_info
if isinstance(tm_info['tm'].schedule, schedules.CronSchedule):
asyncio.ensure_future(self._periodic_transfer_method_call(mid))
return True
elif mid in self._enabled_methods:
logging.logger.debug('tm already enabled ' + mid.hex)
return False
else:
logging.logger.debug('tm not found ' + mid.hex)
return False
def test_discard_transaction(self):
''' Discarding a transaction should execute _tr_discard functions for each dirty item '''
class MyInterface:
def __init__(self):
self._counter = 0
async def _tr_commit(self, tid):
self._counter += 1
def _tr_discard(self, tid):
self._counter -= 1
t = timeuuid.TimeUUID()
tr = Transaction(t=t)
interfaces = [MyInterface() for _ in range(1, 10)]
[self.assertEqual(item._counter, 0) for item in interfaces]
[tr.add_dirty_item(item) for item in interfaces]
tr.discard()
[self.assertEqual(item._counter, -1) for item in interfaces]
self.assertEqual(tr._dirty, set())
async def test_discard_transaction_deletes_items_cannot_commit_after_that(
self):
''' Discarding a transaction will remove all dirty items and disable any later commit '''
class MyInterface:
def __init__(self):
self._counter = 0
async def _tr_commit(self, tid):
self._counter += 1
def _tr_discard(self, tid):
self._counter -= 1
t = timeuuid.TimeUUID()
tr = Transaction(t=t)
interfaces = [MyInterface() for _ in range(1, 10)]
[self.assertEqual(item._counter, 0) for item in interfaces]
[tr.add_dirty_item(item) for item in interfaces]
tr.discard()
[self.assertEqual(item._counter, -1) for item in interfaces]
self.assertEqual(tr._dirty, set())
await tr.commit()
[self.assertEqual(item._counter, -1) for item in interfaces]
def __init__(self, seq, irt):
self.seq = seq if seq != None else timeuuid.TimeUUID()
self.irt = irt
def test_TimeUUID_get_timestamp(self):
''' timestamp attribute should return the timestamp. We only keep microseconds precision '''
for i in range(1, 10000):
t = time.time()
tu = timeuuid.TimeUUID(t)
self.assertEqual(tu.timestamp, int(t * 1e6) / 1e6)
