def mock_read(measurement, begin_time_interval, end_time_interval,
influx_key):
nano_timestamp, nano_delta = nanotime.timestamp(mock_time()).nanoseconds(), \
nanotime.timestamp((self.__batch_duration)).nanoseconds()
if (begin_time_interval < nano_timestamp - nano_delta) and \
(end_time_interval > nano_timestamp - nano_delta):
return [{
"time": nano_timestamp - nano_delta,
"ip_size": 1111,
"ip_size_sum": 1111
}]
elif (begin_time_interval < nano_timestamp - 2 * nano_delta) and \
(end_time_interval > nano_timestamp - 2 * nano_delta):
return [{
"time": nano_timestamp - 2 * nano_delta,
"ip_size": 2222,
"ip_size_sum": 2222
}]
elif (begin_time_interval < nano_timestamp - 3 * nano_delta) and \
(end_time_interval > nano_timestamp - 3 * nano_delta):
return [{
"time": nano_timestamp - 3 * nano_delta,
"ip_size": 3333,
"ip_size_sum": 3333
}]
else:
return []
def test_index(self, mock_time):
mock_time.return_value = 1000.000
historical_data_repository_singleton = MagicMock()
def mock_read(measurement, begin_time_interval, end_time_interval,
influx_key):
nano_timestamp, nano_delta = nanotime.timestamp(mock_time()).nanoseconds(), \
nanotime.timestamp((self.__batch_duration)).nanoseconds()
if (begin_time_interval < nano_timestamp - nano_delta) and \
(end_time_interval > nano_timestamp - nano_delta):
return [{
"time": nano_timestamp - nano_delta,
"ip_size": 1111,
"ip_size_sum": 1111
}]
elif (begin_time_interval < nano_timestamp - 2 * nano_delta) and \
(end_time_interval > nano_timestamp - 2 * nano_delta):
return [{
"time": nano_timestamp - 2 * nano_delta,
"ip_size": 2222,
"ip_size_sum": 2222
}]
elif (begin_time_interval < nano_timestamp - 3 * nano_delta) and \
(end_time_interval > nano_timestamp - 3 * nano_delta):
return [{
"time": nano_timestamp - 3 * nano_delta,
"ip_size": 3333,
"ip_size_sum": 3333
}]
else:
return []
historical_data_repository_singleton.read.side_effect = mock_read
historical_data = HistoricalData(
historical_data_repository_singleton,
self._enumerate_output_aggregation_field, "test_measurement",
self._accuracy, self._key_fields_name, self.__batch_duration)
historical_data.set_zero_value((1111, 2222))
historical_data.set_key(("8.8.8.8", ))
self.assertDictEqual(historical_data[0], {
"ip_size": 1111,
"ip_size_sum": 2222,
'key': ('8.8.8.8', )
}, "Error in overload __getitem__")
self.assertDictEqual(
historical_data[3], {
"ip_size":
3333,
"ip_size_sum":
3333,
'key': ('8.8.8.8', ),
"time":
nanotime.timestamp(mock_time()).nanoseconds() -
3 * nanotime.timestamp((self.__batch_duration)).nanoseconds()
}, "Error in overload __getitem__")
self.assertDictEqual(historical_data[4], {},
"Error in overload __getitem__")
def test_construction(self):
def eq(time1, time2):
self.assertEqual(time1, time2)
self.assertEqual(int(time1), time2._ns)
self.assertEqual(time1._ns, int(time1))
self.assertEqual(time1._ns, int(time2))
self.assertEqual(time1._ns, time2._ns)
self.assertEqual(time1.nanoseconds(), time2.nanoseconds())
self.assertEqual(time1.microseconds(), time2.microseconds())
self.assertEqual(time1.milliseconds(), time2.milliseconds())
self.assertEqual(time1.seconds(), time2.seconds())
self.assertEqual(time1.minutes(), time2.minutes())
self.assertEqual(time1.hours(), time2.hours())
self.assertEqual(time1.days(), time2.days())
self.assertEqual(time1.timestamp(), time2.timestamp())
self.assertEqual(time1.datetime(), time2.datetime())
self.assertEqual(time1.unixtime(), time2.unixtime())
#self.assertEqual(time1, time1.unixtime())
#self.assertEqual(time1, time2.unixtime())
#self.assertEqual(time1, time1.timestamp())
#self.assertEqual(time1, time2.timestamp())
#self.assertEqual(time2, time1.unixtime())
#self.assertEqual(time2, time2.unixtime())
#self.assertEqual(time2, time1.timestamp())
#self.assertEqual(time2, time2.timestamp())
def close(x, y, epsilon=1e-6):
return abs(x - y) < epsilon
for i in range(0, 50):
# basic
eq(nanotime.seconds(1), nanotime.nanoseconds(1e9))
eq(nanotime.seconds(1), nanotime.microseconds(1e6))
eq(nanotime.seconds(1), nanotime.milliseconds(1e3))
eq(nanotime.seconds(1), nanotime.seconds(1))
eq(nanotime.seconds(1), nanotime.minutes(1.0/60))
eq(nanotime.seconds(1), nanotime.hours(1.0/3600))
eq(nanotime.seconds(1), nanotime.days(1.0/(3600 * 24)))
nt1 = nanotime.now()
self.assertTrue(nanotime.now() > nt1)
# timestamp
ts1 = time.time()
ts2 = nanotime.timestamp(ts1).timestamp()
eq(nanotime.timestamp(ts1), nanotime.timestamp(ts1))
self.assertTrue(close(ts2, ts1))
self.assertEqual(nanotime.timestamp(ts1), ts1)
# datetime
dt1 = datetime.datetime.now()
dt2 = nanotime.datetime(dt1).datetime()
eq(nanotime.datetime(dt1), nanotime.datetime(dt1))
self.assertTrue(close(dt1, dt2, datetime.timedelta(microseconds=1)))
def _do_update(self, path):
"""
Make sure the stored info for the given path is up to date.
"""
if not os.path.exists(path):
return (None, None)
mtime, size = self.mtime_and_size(path)
inode = self.inode(path)
cmd = 'SELECT * from {} WHERE inode={}'.format(self.STATE_TABLE,
self._to_sqlite(inode))
self._execute(cmd)
ret = self._fetchall()
if len(ret) == 0:
md5, info = self._collect(path)
cmd = 'INSERT INTO {}(inode, mtime, size, md5, timestamp) ' \
'VALUES ({}, "{}", "{}", "{}", "{}")'
self._execute(
cmd.format(self.STATE_TABLE, self._to_sqlite(inode), mtime,
size, md5, int(nanotime.timestamp(time.time()))))
self.inserts += 1
else:
assert len(ret) == 1
assert len(ret[0]) == 5
i, m, s, md5, timestamp = ret[0]
i = self._from_sqlite(i)
assert i == inode
msg = ("Inode '{}', mtime '{}', actual mtime '{}', "
"size '{}', actual size '{}'.")
logger.debug(msg.format(i, m, mtime, s, size))
if mtime != m or size != s:
md5, info = self._collect(path)
cmd = ('UPDATE {} SET '
'mtime = "{}", size = "{}", '
'md5 = "{}", timestamp = "{}" '
'WHERE inode = {}')
self._execute(
cmd.format(self.STATE_TABLE, mtime, size, md5,
int(nanotime.timestamp(time.time())),
self._to_sqlite(inode)))
else:
info = None
cmd = 'UPDATE {} SET timestamp = "{}" WHERE inode = {}'
self._execute(
cmd.format(self.STATE_TABLE,
int(nanotime.timestamp(time.time())),
self._to_sqlite(inode)))
return (md5, info)
def _do_update(self, path):
"""
Search for the current inode on the table or update it with
gathered information.
"""
if not os.path.exists(path):
return (None, None)
mtime = self.mtime(path)
inode = self.inode(path)
cmd = 'SELECT * from {} WHERE inode={}'.format(self.STATE_TABLE,
self._to_sqlite(inode))
self._execute(cmd)
ret = self._fetchall()
if len(ret) == 0:
md5, info = self._collect(path)
cmd = 'INSERT INTO {}(inode, mtime, md5, timestamp) ' \
'VALUES ({}, "{}", "{}", "{}")'
self._execute(
cmd.format(self.STATE_TABLE, self._to_sqlite(inode), mtime,
md5, int(nanotime.timestamp(time.time()))))
self.inserts += 1
else:
assert len(ret) == 1
assert len(ret[0]) == 4
i, m, md5, timestamp = ret[0]
i = self._from_sqlite(i)
assert i == inode
msg = "Inode '{}', mtime '{}', actual mtime '{}'."
logger.debug(msg.format(i, m, mtime))
if mtime != m:
md5, info = self._collect(path)
cmd = 'UPDATE {} SET ' \
'mtime = "{}", md5 = "{}", timestamp = "{}" ' \
'WHERE inode = {}'
self._execute(
cmd.format(self.STATE_TABLE, mtime, md5,
int(nanotime.timestamp(time.time())),
self._to_sqlite(inode)))
else:
info = None
cmd = 'UPDATE {} SET timestamp = "{}" WHERE inode = {}'
self._execute(
cmd.format(self.STATE_TABLE,
int(nanotime.timestamp(time.time())),
self._to_sqlite(inode)))
return (md5, info)
def get_mtime_and_size(path, fs, dvcignore=None):
import nanotime
if fs.isdir(path):
size = 0
files_mtimes = {}
if dvcignore:
walk_iterator = dvcignore.find(fs, path)
else:
walk_iterator = fs.find(path)
for file_path in walk_iterator:
try:
stats = fs.info(file_path)
except OSError as exc:
# NOTE: broken symlink case.
if exc.errno != errno.ENOENT:
raise
continue
size += stats["size"]
files_mtimes[file_path] = stats["mtime"]
# We track file changes and moves, which cannot be detected with simply
# max(mtime(f) for f in non_ignored_files)
mtime = dict_md5(files_mtimes)
else:
base_stat = fs.info(path)
size = base_stat["size"]
mtime = base_stat["mtime"]
mtime = int(nanotime.timestamp(mtime))
return str(mtime), size
def _update_state_for_path_changed(
self,
path,
actual_inode,
actual_mtime,
actual_size,
known_checksum=None,
):
if known_checksum:
md5, info = known_checksum, None
else:
md5, info = self._collect(path)
cmd = ("UPDATE {} SET "
'mtime = "{}", size = "{}", '
'md5 = "{}", timestamp = "{}" '
"WHERE inode = {}")
self._execute(
cmd.format(
self.STATE_TABLE,
actual_mtime,
actual_size,
md5,
int(nanotime.timestamp(time.time())),
self._to_sqlite(actual_inode),
))
return md5, info
def get_mtime_and_size(path, dvcignore):
if os.path.isdir(fspath_py35(path)):
size = 0
files_mtimes = {}
for file_path in walk_files(path, dvcignore):
try:
stat = os.stat(file_path)
except OSError as exc:
# NOTE: broken symlink case.
if exc.errno != errno.ENOENT:
raise
continue
size += stat.st_size
files_mtimes[file_path] = stat.st_mtime
# We track file changes and moves, which cannot be detected with simply
# max(mtime(f) for f in non_ignored_files)
mtime = dict_md5(files_mtimes)
else:
base_stat = os.stat(fspath_py35(path))
size = base_stat.st_size
mtime = base_stat.st_mtime
mtime = int(nanotime.timestamp(mtime))
# State of files handled by dvc is stored in db as TEXT.
# We cast results to string for later comparisons with stored values.
return str(mtime), str(size)
def __getitem__(self, index):
if index == 0:
self._zero_value["key"] = self._key
return self._zero_value
else:
if self._accuracy >= self._batch_duration:
logging.warning(
"Current accuracy {} is more or equal batch duration {}. You can get incorrect "
"results of analysis in this case ".format(
self._accuracy, self._batch_duration))
nano_timestamp, nano_delta, nano_accuracy = nanotime.timestamp(
time()), nanotime.seconds(
index * self._batch_duration), nanotime.seconds(
self._accuracy)
begin_time_interval = nano_timestamp - nano_delta - nano_accuracy
end_time_interval = nano_timestamp - nano_delta + nano_accuracy
influx_key = None
if self._key:
influx_key = dict(
map(lambda x, y: (x, y), self._key_fields_name, self._key))
historical_values = self._historical_data_repository_singleton.read(
self._measurement, begin_time_interval.nanoseconds(),
end_time_interval.nanoseconds(), influx_key)
if historical_values:
historical_values = sorted(historical_values,
key=lambda point: point["time"],
reverse=True)[0]
historical_values["key"] = self._key
return historical_values
else:
return {}
def _update_state_record_timestamp_for_inode(self, actual_inode):
cmd = 'UPDATE {} SET timestamp = "{}" WHERE inode = {}'
self._execute(
cmd.format(
self.STATE_TABLE,
int(nanotime.timestamp(time.time())),
self._to_sqlite(actual_inode),
))
def _do_update(self, path):
if not os.path.exists(path):
return (None, None)
mtime = self.mtime(path)
inode = self.inode(path)
cmd = 'SELECT * from {} WHERE inode={}'.format(self.STATE_TABLE,
self._to_sqlite(inode))
self._execute(cmd)
ret = self._fetchall()
if len(ret) == 0:
md5, info = self._collect(path)
cmd = 'INSERT INTO {}(inode, mtime, md5, timestamp) ' \
'VALUES ({}, "{}", "{}", "{}")'
self._execute(
cmd.format(self.STATE_TABLE, self._to_sqlite(inode), mtime,
md5, int(nanotime.timestamp(time.time()))))
self.inserts += 1
else:
assert len(ret) == 1
assert len(ret[0]) == 4
i, m, md5, timestamp = ret[0]
i = self._from_sqlite(i)
assert i == inode
if mtime != m:
md5, info = self._collect(path)
cmd = 'UPDATE {} SET ' \
'mtime = "{}", md5 = "{}", timestamp = "{}" ' \
'WHERE inode = {}'
self._execute(
cmd.format(self.STATE_TABLE, mtime, md5,
int(nanotime.timestamp(time.time())),
self._to_sqlite(inode)))
else:
info = None
cmd = 'UPDATE {} SET timestamp = "{}" WHERE inode = {}'
self._execute(
cmd.format(self.STATE_TABLE,
int(nanotime.timestamp(time.time())),
self._to_sqlite(inode)))
return (md5, info)
def mtime(path):
mtime = os.path.getmtime(path)
if os.path.isdir(path):
for root, dirs, files in os.walk(path):
for entry in list(dirs) + list(files):
p = os.path.join(root, entry)
m = os.path.getmtime(p)
if m > mtime:
mtime = m
return str(int(nanotime.timestamp(mtime)))
def _mtime_and_size(path):
size = os.path.getsize(path)
mtime = os.path.getmtime(path)
if os.path.isdir(path):
for root, dirs, files in os.walk(str(path)):
for name in dirs + files:
entry = os.path.join(root, name)
stat = os.stat(entry)
size += stat.st_size
entry_mtime = stat.st_mtime
if entry_mtime > mtime:
mtime = entry_mtime
# State of files handled by dvc is stored in db as TEXT.
# We cast results to string for later comparisons with stored values.
return str(int(nanotime.timestamp(mtime))), str(size)
def _insert_new_state_record(self, path, actual_inode, actual_mtime,
actual_size, known_checksum):
if known_checksum:
md5, info = known_checksum, None
else:
md5, info = self._collect(path)
cmd = ("INSERT INTO {}(inode, mtime, size, md5, timestamp) "
'VALUES ({}, "{}", "{}", "{}", "{}")')
self._execute(
cmd.format(
self.STATE_TABLE,
self._to_sqlite(actual_inode),
actual_mtime,
actual_size,
md5,
int(nanotime.timestamp(time.time())),
))
self.inserts += 1
return md5, info
def analysis_record(input_tuple_value, data_structure, time_delta, accuracy, rule, alert_sender_singleton,
historical_data_repository_singleton, measurement, key_fields_name=None, key=None):
"""
Checks data for deviations
:param key: This is the key to the data aggregation
:param input_tuple_value: input data to check
:return:
"""
nano_timestamp, nano_delta, nano_accuracy = nanotime.timestamp(time()), nanotime.seconds(
time_delta), nanotime.seconds(accuracy)
begin_time_interval = nano_timestamp - nano_delta - nano_accuracy
end_time_interval = nano_timestamp - nano_delta + nano_accuracy
# create a dictionary with a key equal to the name of field from rule and a value equal to index in the
# input tuple nfti (name field to index)
name_to_index = dict(map(lambda x: (x, data_structure[x]), rule.keys()))
# call for real historical data
influx_key = None
if key:
influx_key = dict(map(lambda x, y: (x, y), key_fields_name, key))
historical_values = historical_data_repository_singleton.read(measurement, begin_time_interval.nanoseconds(),
end_time_interval.nanoseconds(), influx_key)
if historical_values:
historical_values = sorted(historical_values, key=lambda point: point["time"])
# we analyse only first value from data, because next values will analysed in next moments
for field in rule.keys():
lower_bound = historical_values[0][field] * (1 - float(rule[field]) / 100)
upper_bound = historical_values[0][field] * (1 + float(rule[field]) / 100)
current_value = input_tuple_value[name_to_index[field]]
if (current_value < lower_bound) or (current_value > upper_bound):
alert_sender_singleton.send_message(timestamp=time(),
param={"key": key,
"field": field,
"lower_bound": lower_bound,
"upper_bound": upper_bound,
"value": current_value})
def mtime_and_size(path):
size = 0
mtime = os.path.getmtime(path)
if os.path.isdir(path):
for root, dirs, files in os.walk(path):
for dname in dirs:
p = os.path.join(root, dname)
m = os.path.getmtime(p)
if m > mtime:
mtime = m
for fname in files:
p = os.path.join(root, fname)
st = os.stat(p)
size += st.st_size
m = st.st_mtime
if m > mtime:
mtime = m
return (str(int(nanotime.timestamp(mtime))), size)
def get_mtime_and_size(path):
size = os.path.getsize(path)
mtime = os.path.getmtime(path)
if os.path.isdir(path):
for root, dirs, files in dvc_walk(str(path)):
for name in dirs + files:
entry = os.path.join(root, name)
try:
stat = os.stat(entry)
except OSError as exc:
# NOTE: broken symlink case.
if exc.errno != errno.ENOENT:
raise
continue
size += stat.st_size
entry_mtime = stat.st_mtime
if entry_mtime > mtime:
mtime = entry_mtime
# State of files handled by dvc is stored in db as TEXT.
# We cast results to string for later comparisons with stored values.
return str(int(nanotime.timestamp(mtime))), str(size)
def current_timestamp():
return int(nanotime.timestamp(time.time()))
def last_day():
now = nanotime.timestamp(time.time())
yesterday = datetime.datetime.utcnow() - datetime.timedelta(days=1)
yesterday = nanotime.datetime(yesterday)
return "{}-{}".format(yesterday.nanoseconds(), now.nanoseconds())
def _do_update(self, path):
"""
Make sure the stored info for the given path is up to date.
"""
if not os.path.exists(path):
return (None, None)
actual_mtime, actual_size = self._mtime_and_size(path)
actual_inode = self._inode(path)
cmd = "SELECT * from {} WHERE inode={}".format(
self.STATE_TABLE, self._to_sqlite(actual_inode)
)
self._execute(cmd)
ret = self._fetchall()
if not ret:
md5, info = self._collect(path)
cmd = (
"INSERT INTO {}(inode, mtime, size, md5, timestamp) "
'VALUES ({}, "{}", "{}", "{}", "{}")'
)
self._execute(
cmd.format(
self.STATE_TABLE,
self._to_sqlite(actual_inode),
actual_mtime,
actual_size,
md5,
int(nanotime.timestamp(time.time())),
)
)
self.inserts += 1
else:
assert len(ret) == 1
assert len(ret[0]) == 5
inode, mtime, size, md5, _ = ret[0]
inode = self._from_sqlite(inode)
assert inode == actual_inode
logger.debug(
"Inode '{}', mtime '{}', actual mtime '{}', size '{}', "
"actual size '{}'.".format(
inode, mtime, actual_mtime, size, actual_size
)
)
if actual_mtime != mtime or actual_size != size:
md5, info = self._collect(path)
cmd = (
"UPDATE {} SET "
'mtime = "{}", size = "{}", '
'md5 = "{}", timestamp = "{}" '
"WHERE inode = {}"
)
self._execute(
cmd.format(
self.STATE_TABLE,
actual_mtime,
actual_size,
md5,
int(nanotime.timestamp(time.time())),
self._to_sqlite(actual_inode),
)
)
else:
info = None
cmd = 'UPDATE {} SET timestamp = "{}" WHERE inode = {}'
self._execute(
cmd.format(
self.STATE_TABLE,
int(nanotime.timestamp(time.time())),
self._to_sqlite(actual_inode),
)
)
return (md5, info)
def current_timestamp():
import nanotime
return int(nanotime.timestamp(time.time()))
def mtime(path):
return str(int(nanotime.timestamp(os.path.getmtime(path))))
def since(since):
since = nanotime.datetime(since)
now = nanotime.timestamp(time.time())
return "{}-{}".format(since.nanoseconds(), now.nanoseconds())