def __init__(self, query, **kwargs):
super(TQL, self).__init__(**kwargs)
if self._start is None:
raise TQLException("Searches require at least a start time with either a timestamp or datemath expression: start='now-10m', end='now' | start='20160101', end='now'")
try:
self._query_raw = query
self._query = query
self._start_time = dm(self._start)
self._end_time = dm(self._end)
self._index = tattle.get_indexes(self._index_name_pattern, self._start_time, self._end_time, pattern=self._index_ts_pattern)
except TQLException as e:
raise TQLArgsException("Unable to set arguments for TQL, I am missing: %s" % (e))
self._start_time_iso_str = self._start_time.format(self._ISO_TS)
self._end_time_iso_str = self._end_time.format(self._ISO_TS)
self._start_time_pretty = self._start_time.format(self._PRETTY_TS)
self._end_time_pretty = self._end_time.format(self._PRETTY_TS)
self._start_time_epoch = int(self._start_time.format('X'))
self._start_time_epoch_millis = self._start_time_epoch * 1000
self._end_time_epoch = int(self._end_time.format('X'))
self._end_time_epoch_millis = self._end_time_epoch * 1000
self._start_time_unix = self._start_time_epoch
self._end_time_unix = self._end_time_epoch
self._qd = self.get_intentions(self._query_raw)
self.build_es_query()
def __init__(self, **kwargs):
self._ISO_TS = 'YYYY-MM-DDTHH:mm:ssZZ'
self._PRETTY_TS = 'MMM D YYYY, HH:mm:ss ZZ'
self.agg_size_from = 0
self.agg_size = 0
self.hit_size_from = 0
self.hit_size = 10000
self._query_raw = ''
self._query = ''
self._start = None
self._start_time = dm('now-1m')
self._end_time = dm('now')
self.exclude = ''
self._ts_field = '@timestamp'
self._index_ts_pattern = 'YYYY.MM.DD'
self._index_name_pattern = 'logstash-*'
for key, value in kwargs.items():
if key in ('start', 'from', '_from'):
self._start = value
elif key in ('end', 'to', '_to'):
self._end = value
setattr(self, key, value)
self._index = tattle.get_indexes(self.index, self._start_time, self._end_time, pattern=self._index_ts_pattern)
def __init__(self, **kwargs):
self._ISO_TS = 'YYYY-MM-DDTHH:mm:ssZZ'
self._PRETTY_TS = 'MMM D YYYY, HH:mm:ss ZZ'
self.agg_size_from = 0
self.agg_size = 0
self.hit_size_from = 0
self.hit_size = 10000
self._query_raw = ''
self._query = ''
self._start = None
self._start_time = dm('now-1m')
self._end_time = dm('now')
self.exclude = ''
self._ts_field = '@timestamp'
self._index_ts_pattern = 'YYYY.MM.DD'
self._index_name_pattern = 'logstash-*'
for key, value in kwargs.items():
if key in ('start', 'from', '_from'):
self._start = value
elif key in ('end', 'to', '_to'):
self._end = value
setattr(self, key, value)
self._index = tattle.get_indexes(self.index,
self._start_time,
self._end_time,
pattern=self._index_ts_pattern)
def __init__(self, query, **kwargs):
super(TQL, self).__init__(**kwargs)
if self._start is None:
raise TQLException(
"Searches require at least a start time with either a timestamp or datemath expression: start='now-10m', end='now' | start='20160101', end='now'"
)
try:
self._query_raw = query
self._query = query
self._start_time = dm(self._start)
self._end_time = dm(self._end)
self._index = tattle.get_indexes(self._index_name_pattern,
self._start_time,
self._end_time,
pattern=self._index_ts_pattern)
except TQLException as e:
raise TQLArgsException(
"Unable to set arguments for TQL, I am missing: %s" % (e))
self._start_time_iso_str = self._start_time.format(self._ISO_TS)
self._end_time_iso_str = self._end_time.format(self._ISO_TS)
self._start_time_pretty = self._start_time.format(self._PRETTY_TS)
self._end_time_pretty = self._end_time.format(self._PRETTY_TS)
self._start_time_epoch = int(self._start_time.format('X'))
self._start_time_epoch_millis = self._start_time_epoch * 1000
self._end_time_epoch = int(self._end_time.format('X'))
self._end_time_epoch_millis = self._end_time_epoch * 1000
self._start_time_unix = self._start_time_epoch
self._end_time_unix = self._end_time_epoch
self._qd = self.get_intentions(self._query_raw)
self.build_es_query()
def import_cti_event(self,
timestamp: datetime,
data: dict,
is_update: bool = False) -> dict:
try:
# Submit to Elastic index
_document = data
_document["@timestamp"] = timestamp
_write_idx = self.idx
# Render date-specific index if we're using logstash style indices
m = self.pattern.search(_write_idx)
if m is not None:
m = m.groupdict()
if m.get("modulo", None) is not None:
_fmt = m.get("format") or DM_DEFAULT_FMT
logger.debug(f"{m['modulo']} -> {_fmt}")
_val = dm(m.get("modulo"), now=timestamp).format(_fmt)
_write_idx = self.pattern.sub(_val, _write_idx)
# Submit to Elastic index
logger.debug(f"Indexing doc to {_write_idx}:\n {_document}")
self.es_client.index(
index=_write_idx,
id=data["x_opencti_id"],
body=_document,
)
except RequestError as err:
logger.error("Unexpected error:", err, data)
except Exception as err:
logger.error("Something else happened", err, data)
return data
def _validate_steps_count(self, start, end, gap):
with warnings.catch_warnings():
warnings.simplefilter("ignore")
gaps_count = None
ret_type = None
try:
start = int(start)
end = int(end)
gap = int(gap)
gaps_count = int((end - start) / gap)
ret_type = int
except ValueError:
pass
if not gaps_count:
if not (gap.startswith("-") or gap.startswith("+")):
raise SearchException(
"Gap must be preceded with either + or - sign.")
try:
parsed_start = dm(
self.datastore.to_pydatemath(start)).int_timestamp
parsed_end = dm(
self.datastore.to_pydatemath(end)).int_timestamp
parsed_gap = dm(self.datastore.to_pydatemath(
gap)).int_timestamp - dm('now').int_timestamp
gaps_count = int((parsed_end - parsed_start) / parsed_gap)
ret_type = str
except (DateMathException, AttributeError):
pass
if not gaps_count:
raise SearchException(
"Could not parse date ranges. (start='%s', end='%s', gap='%s')"
% (start, end, gap))
if gaps_count > self.MAX_FACET_LIMIT:
raise SearchException(
f'Histograms are limited to a maximum of {self.MAX_FACET_LIMIT} steps. '
f'Current settings would generate {gaps_count} steps')
return ret_type
import random
import string
import time
import warnings
from datemath import dm
from retrying import retry
import pytest
from assemblyline.datastore import Collection
with warnings.catch_warnings():
warnings.simplefilter("ignore")
test_map = {
'test1': {'expiry_dt': dm('now-2d/m').isoformat().replace('+00:00', '.001Z'),
'lvl_i': 400, 'test1_s': 'hello', 'tags_ss': ['a', 'b', 'c']},
'test2': {'expiry_dt': dm('now-1d/m').isoformat().replace('+00:00', '.001Z'),
'lvl_i': 100, 'test2_s': 'hello', 'tags_ss': ['a', 'b', 'f']},
'test3': {'expiry_dt': dm('now/m').isoformat().replace('+00:00', '.001Z'),
'lvl_i': 200, 'test3_s': 'hello', 'tags_ss': ['a', 'b', 'e']},
'test4': {'expiry_dt': dm('now-2d/m').isoformat().replace('+00:00', '.001Z'),
'lvl_i': 400, 'test4_s': 'hello', 'tags_ss': ['a', 'b', 'd']},
'dict1': {'expiry_dt': dm('now-2d/m').isoformat().replace('+00:00', '.001Z'),
'lvl_i': 400, 'classification_s': 'U', 'test1_s': 'hello', 'tags_ss': []},
'dict2': {'expiry_dt': dm('now/m').isoformat().replace('+00:00', '.001Z'),
'lvl_i': 100, 'classification_s': 'U', 'test2_s': 'hello', 'tags_ss': []},
'dict3': {'expiry_dt': dm('now-3d/m').isoformat().replace('+00:00', '.001Z'),
'lvl_i': 200, 'classification_s': 'C', 'test3_s': 'hello', 'tags_ss': []},
'dict4': {'expiry_dt': dm('now-1d/m').isoformat().replace('+00:00', '.001Z'),
'lvl_i': 400, 'classification_s': 'TS', 'test4_s': 'hello', 'tags_ss': []},
def testParse(self):
# Baisc dates
self.assertEqual(
dm('2016.01.02').format(iso8601), '2016-01-02T00:00:00-00:00')
self.assertEqual(
dm('2016-01-02').format(iso8601), '2016-01-02T00:00:00-00:00')
self.assertEqual(
dm('2016-01-02 01:00:00').format(iso8601),
'2016-01-02T01:00:00-00:00')
# Rounding Tests
self.assertEqual(
dm('2016-01-01||/d').format('YYYY-MM-DDTHH:mm:ssZZ'),
'2016-01-01T00:00:00-00:00')
self.assertEqual(
dm('2014-11-18||/y').format('YYYY-MM-DDTHH:mm:ssZZ'),
'2014-01-01T00:00:00-00:00')
self.assertEqual(
dm('2016-01-01 14:00:00||/w').format('YYYY-MM-DDTHH:mm:ssZZ'),
'2015-12-28T00:00:00-00:00')
self.assertEqual(
dm('2014-11||/M').format('YYYY-MM-DDTHH:mm:ssZZ'),
'2014-11-01T00:00:00-00:00')
self.assertEqual(
dm('2016-01-02||/M+1h+1m').format(iso8601),
'2016-01-01T01:01:00-00:00')
self.assertEqual(
dm('2016-01-02||/d+1h').format(iso8601),
'2016-01-02T01:00:00-00:00')
self.assertEqual(
dm('2016-01-02T14:02:00||/h').format(iso8601),
'2016-01-02T14:00:00-00:00')
self.assertEqual(
dm('2016-01-02T14:02:00||/H').format(iso8601),
'2016-01-02T14:00:00-00:00')
# Rounding Up Tests
self.assertEqual(
dm('2016-01-01||/d',
roundDown=False).format('YYYY-MM-DDTHH:mm:ssZZ'),
'2016-01-01T23:59:59-00:00')
self.assertEqual(
dm('2014-11-18||/y',
roundDown=False).format('YYYY-MM-DDTHH:mm:ssZZ'),
'2014-12-31T23:59:59-00:00')
# Timezone Tests
self.assertEqual(
dm('now', tz='US/Pacific').format(iso8601),
arrow.utcnow().to('US/Pacific').format(iso8601))
self.assertEqual(
dm('2017-09-22 10:20:00', tz='US/Pacific').datetime,
pydatetime(2017, 9, 22, 10, 20, 00, tzinfo=tz.gettz('US/Pacific')))
self.assertEqual(dm('2016-01-01', tz='UTC'),
arrow.get('2016-01-01').to('UTC'))
self.assertEqual(dm('2016-01-01', tz='US/Eastern'),
pydatetime(2016, 1, 1, tzinfo=tz.gettz('US/Eastern')))
self.assertEqual(
datemath('2016-01-01T01:00:00', tz='US/Central'),
pydatetime(2016, 1, 1, 1, 0, 0, tzinfo=tz.gettz('US/Central')))
# relitive formats
# addition
self.assertEqual(
dm('+1s').format(iso8601),
arrow.utcnow().replace(seconds=+1).format(iso8601))
self.assertEqual(
dm('+1m').format(iso8601),
arrow.utcnow().replace(minutes=+1).format(iso8601))
self.assertEqual(
dm('+1h').format(iso8601),
arrow.utcnow().replace(hours=+1).format(iso8601))
self.assertEqual(
dm('+1d').format(iso8601),
arrow.utcnow().replace(days=+1).format(iso8601))
self.assertEqual(
dm('+1w').format(iso8601),
arrow.utcnow().replace(weeks=+1).format(iso8601))
self.assertEqual(
dm('+1M').format(iso8601),
arrow.utcnow().replace(months=+1).format(iso8601))
self.assertEqual(
dm('+1Y').format(iso8601),
arrow.utcnow().replace(years=+1).format(iso8601))
self.assertEqual(
dm('+1y').format(iso8601),
arrow.utcnow().replace(years=+1).format(iso8601))
# subtraction
self.assertEqual(
dm('-1s').format(iso8601),
arrow.utcnow().replace(seconds=-1).format(iso8601))
self.assertEqual(
dm('-1m').format(iso8601),
arrow.utcnow().replace(minutes=-1).format(iso8601))
self.assertEqual(
dm('-1h').format(iso8601),
arrow.utcnow().replace(hours=-1).format(iso8601))
self.assertEqual(
dm('-1d').format(iso8601),
arrow.utcnow().replace(days=-1).format(iso8601))
self.assertEqual(
dm('-1w').format(iso8601),
arrow.utcnow().replace(weeks=-1).format(iso8601))
self.assertEqual(
dm('-1M').format(iso8601),
arrow.utcnow().replace(months=-1).format(iso8601))
self.assertEqual(
dm('-1Y').format(iso8601),
arrow.utcnow().replace(years=-1).format(iso8601))
self.assertEqual(
dm('-1y').format(iso8601),
arrow.utcnow().replace(years=-1).format(iso8601))
# rounding
self.assertEqual(
dm('/s').format(iso8601),
arrow.utcnow().floor('second').format(iso8601))
self.assertEqual(
dm('/m').format(iso8601),
arrow.utcnow().floor('minute').format(iso8601))
self.assertEqual(
dm('/h').format(iso8601),
arrow.utcnow().floor('hour').format(iso8601))
self.assertEqual(
dm('/d').format(iso8601),
arrow.utcnow().floor('day').format(iso8601))
self.assertEqual(
dm('/w').format(iso8601),
arrow.utcnow().floor('week').format(iso8601))
self.assertEqual(
dm('/M').format(iso8601),
arrow.utcnow().floor('month').format(iso8601))
self.assertEqual(
dm('/Y').format(iso8601),
arrow.utcnow().floor('year').format(iso8601))
self.assertEqual(
dm('/y').format(iso8601),
arrow.utcnow().floor('year').format(iso8601))
# rounding up
self.assertEqual(
dm('/s', roundDown=False).format(iso8601),
arrow.utcnow().ceil('second').format(iso8601))
self.assertEqual(
dm('/m', roundDown=False).format(iso8601),
arrow.utcnow().ceil('minute').format(iso8601))
self.assertEqual(
dm('/h', roundDown=False).format(iso8601),
arrow.utcnow().ceil('hour').format(iso8601))
self.assertEqual(
dm('/d', roundDown=False).format(iso8601),
arrow.utcnow().ceil('day').format(iso8601))
self.assertEqual(
dm('/w', roundDown=False).format(iso8601),
arrow.utcnow().ceil('week').format(iso8601))
self.assertEqual(
dm('/M', roundDown=False).format(iso8601),
arrow.utcnow().ceil('month').format(iso8601))
self.assertEqual(
dm('/Y', roundDown=False).format(iso8601),
arrow.utcnow().ceil('year').format(iso8601))
self.assertEqual(
dm('/y', roundDown=False).format(iso8601),
arrow.utcnow().ceil('year').format(iso8601))
# roundDown option tests
self.assertEqual(
dm('2016-01-01T14:00:00||/d').format(iso8601),
'2016-01-01T00:00:00-00:00')
self.assertEqual(
dm('2016-01-01T14:00:00||/d', roundDown=False).format(iso8601),
'2016-01-01T23:59:59-00:00')
# complicated date math
self.assertEqual(
dm('now/d-1h').format(iso8601),
arrow.utcnow().floor('day').replace(hours=-1).format(iso8601))
self.assertEqual(
dm('+1h').format(iso8601),
arrow.utcnow().replace(hours=+1).format(iso8601))
self.assertEqual(
dm('/M+2d').format(iso8601),
arrow.utcnow().floor('month').replace(days=+2).format(iso8601))
self.assertEqual(
dm('now/w+2d-2h').format(iso8601),
arrow.utcnow().floor('week').replace(days=+2,
hours=-2).format(iso8601))
self.assertEqual(
dm('now/M+1w-2h+10s').format(iso8601),
arrow.utcnow().floor('month').replace(weeks=+1,
hours=-2,
seconds=+10).format(iso8601))
self.assertEqual(
dm('now-1d/d').format(iso8601),
arrow.utcnow().replace(days=-1).floor('day').format(iso8601))
self.assertEqual(
dm('now+1d/d').format(iso8601),
arrow.utcnow().replace(days=1).floor('day').format(iso8601))
self.assertEqual(
dm('now-10d/d').format(iso8601),
arrow.utcnow().replace(days=-10).floor('day').format(iso8601))
self.assertEqual(
dm('now+10d/d').format(iso8601),
arrow.utcnow().replace(days=10).floor('day').format(iso8601))
# future
self.assertEqual(
dm('+1s').format(iso8601),
arrow.utcnow().replace(seconds=+1).format(iso8601))
self.assertEqual(
dm('+1s+2m+3h').format(iso8601),
arrow.utcnow().replace(seconds=+1, minutes=+2,
hours=+3).format(iso8601))
self.assertEqual(
dm('+1m').format(iso8601),
arrow.utcnow().replace(minutes=+1).format(iso8601))
self.assertEqual(
dm('+1m+5h').format(iso8601),
arrow.utcnow().replace(minutes=+1, hours=+5).format(iso8601))
self.assertEqual(
dm('/d+1m+5h').format(iso8601),
arrow.utcnow().floor('day').replace(minutes=+1,
hours=+5).format(iso8601))
self.assertEqual(
dm('+1h').format(iso8601),
arrow.utcnow().replace(hours=+1).format(iso8601))
self.assertEqual(
dm('+1w').format(iso8601),
arrow.utcnow().replace(weeks=+1).format(iso8601))
self.assertEqual(
dm('+1w+12d').format(iso8601),
arrow.utcnow().replace(weeks=+1, days=+12).format(iso8601))
self.assertEqual(
dm('+2y').format(iso8601),
arrow.utcnow().replace(years=+2).format(iso8601))
self.assertEqual(
dm('+2y+22d+4h').format(iso8601),
arrow.utcnow().replace(years=+2, days=+22,
hours=+4).format(iso8601))
# past
self.assertEqual(
dm('-3w').format(iso8601),
arrow.utcnow().replace(weeks=-3).format(iso8601))
self.assertEqual(
dm('-3W').format(iso8601),
arrow.utcnow().replace(weeks=-3).format(iso8601))
self.assertEqual(
dm('-3w-2d-6h').format(iso8601),
arrow.utcnow().replace(weeks=-3, days=-2,
hours=-6).format(iso8601))
self.assertEqual(
dm('-3w-2d-22h-36s').format(iso8601),
arrow.utcnow().replace(weeks=-3, days=-2, hours=-22,
seconds=-36).format(iso8601))
self.assertEqual(
dm('-6y-3w-2d-22h-36s').format(iso8601),
arrow.utcnow().replace(years=-6,
weeks=-3,
days=-2,
hours=-22,
seconds=-36).format(iso8601))
import datetime
delta = datetime.timedelta(seconds=1)
# datetime objects
self.assertAlmostEqual(dm('now').datetime,
arrow.utcnow().datetime,
delta=delta)
self.assertAlmostEqual(dm('now+1d').datetime,
arrow.utcnow().replace(days=+1).datetime,
delta=delta)
self.assertAlmostEqual(dm('/w').datetime,
arrow.utcnow().floor('week').datetime,
delta=delta)
# Floats
self.assertEqual(
dm('now-2.5h').format(iso8601),
arrow.utcnow().replace(hours=-2.5).format(iso8601))
self.assertEqual(
dm('now-2.5d').format(iso8601),
arrow.utcnow().replace(days=-2.5).format(iso8601))
def safe_date(pattern):
return dm(f'{pattern}/m').isoformat().replace('+00:00', '.001Z')
def testParse(self):
# Baisc dates
self.assertEqual(
dm('2016.01.02').format(iso8601), '2016-01-02T00:00:00-00:00')
self.assertEqual(
dm('2016-01-02').format(iso8601), '2016-01-02T00:00:00-00:00')
self.assertEqual(
dm('2016-01-02 01:00:00').format(iso8601),
'2016-01-02T01:00:00-00:00')
# Rounding Tests
self.assertEqual(
dm('2016-01-01||/d').format('YYYY-MM-DDTHH:mm:ssZZ'),
'2016-01-01T00:00:00-00:00')
self.assertEqual(
dm('2014-11-18||/y').format('YYYY-MM-DDTHH:mm:ssZZ'),
'2014-01-01T00:00:00-00:00')
self.assertEqual(
dm('2016-01-01 14:00:00||/w').format('YYYY-MM-DDTHH:mm:ssZZ'),
'2015-12-28T00:00:00-00:00')
self.assertEqual(
dm('2014-11||/M').format('YYYY-MM-DDTHH:mm:ssZZ'),
'2014-11-01T00:00:00-00:00')
self.assertEqual(
dm('2016-01-02||/M+1h+1m').format(iso8601),
'2016-01-01T01:01:00-00:00')
self.assertEqual(
dm('2016-01-02||/d+1h').format(iso8601),
'2016-01-02T01:00:00-00:00')
self.assertEqual(
dm('2016-01-02T14:02:00||/h').format(iso8601),
'2016-01-02T14:00:00-00:00')
self.assertEqual(
dm('2016-01-02T14:02:00||/H').format(iso8601),
'2016-01-02T14:00:00-00:00')
# relitive formats
# addition
self.assertEqual(
dm('+1s').format(iso8601),
arrow.utcnow().replace(seconds=+1).format(iso8601))
self.assertEqual(
dm('+1m').format(iso8601),
arrow.utcnow().replace(minutes=+1).format(iso8601))
self.assertEqual(
dm('+1h').format(iso8601),
arrow.utcnow().replace(hours=+1).format(iso8601))
self.assertEqual(
dm('+1d').format(iso8601),
arrow.utcnow().replace(days=+1).format(iso8601))
self.assertEqual(
dm('+1w').format(iso8601),
arrow.utcnow().replace(weeks=+1).format(iso8601))
self.assertEqual(
dm('+1M').format(iso8601),
arrow.utcnow().replace(months=+1).format(iso8601))
self.assertEqual(
dm('+1Y').format(iso8601),
arrow.utcnow().replace(years=+1).format(iso8601))
self.assertEqual(
dm('+1y').format(iso8601),
arrow.utcnow().replace(years=+1).format(iso8601))
# subtraction
self.assertEqual(
dm('-1s').format(iso8601),
arrow.utcnow().replace(seconds=-1).format(iso8601))
self.assertEqual(
dm('-1m').format(iso8601),
arrow.utcnow().replace(minutes=-1).format(iso8601))
self.assertEqual(
dm('-1h').format(iso8601),
arrow.utcnow().replace(hours=-1).format(iso8601))
self.assertEqual(
dm('-1d').format(iso8601),
arrow.utcnow().replace(days=-1).format(iso8601))
self.assertEqual(
dm('-1w').format(iso8601),
arrow.utcnow().replace(weeks=-1).format(iso8601))
self.assertEqual(
dm('-1M').format(iso8601),
arrow.utcnow().replace(months=-1).format(iso8601))
self.assertEqual(
dm('-1Y').format(iso8601),
arrow.utcnow().replace(years=-1).format(iso8601))
self.assertEqual(
dm('-1y').format(iso8601),
arrow.utcnow().replace(years=-1).format(iso8601))
# rounding
self.assertEqual(
dm('/s').format(iso8601),
arrow.utcnow().floor('second').format(iso8601))
self.assertEqual(
dm('/m').format(iso8601),
arrow.utcnow().floor('minute').format(iso8601))
self.assertEqual(
dm('/h').format(iso8601),
arrow.utcnow().floor('hour').format(iso8601))
self.assertEqual(
dm('/d').format(iso8601),
arrow.utcnow().floor('day').format(iso8601))
self.assertEqual(
dm('/w').format(iso8601),
arrow.utcnow().floor('week').format(iso8601))
self.assertEqual(
dm('/M').format(iso8601),
arrow.utcnow().floor('month').format(iso8601))
self.assertEqual(
dm('/Y').format(iso8601),
arrow.utcnow().floor('year').format(iso8601))
self.assertEqual(
dm('/y').format(iso8601),
arrow.utcnow().floor('year').format(iso8601))
# complicated date math
self.assertEqual(
dm('now/d-1h').format(iso8601),
arrow.utcnow().floor('day').replace(hours=-1).format(iso8601))
self.assertEqual(
dm('+1h').format(iso8601),
arrow.utcnow().replace(hours=+1).format(iso8601))
self.assertEqual(
dm('/M+2d').format(iso8601),
arrow.utcnow().floor('month').replace(days=+2).format(iso8601))
self.assertEqual(
dm('now/w+2d-2h').format(iso8601),
arrow.utcnow().floor('week').replace(days=+2,
hours=-2).format(iso8601))
self.assertEqual(
dm('now/M+1w-2h+10s').format(iso8601),
arrow.utcnow().floor('month').replace(weeks=+1,
hours=-2,
seconds=+10).format(iso8601))
self.assertEqual(
dm('now-1d/d').format(iso8601),
arrow.utcnow().replace(days=-1).floor('day').format(iso8601))
self.assertEqual(
dm('now+1d/d').format(iso8601),
arrow.utcnow().replace(days=1).floor('day').format(iso8601))
self.assertEqual(
dm('now-10d/d').format(iso8601),
arrow.utcnow().replace(days=-10).floor('day').format(iso8601))
self.assertEqual(
dm('now+10d/d').format(iso8601),
arrow.utcnow().replace(days=10).floor('day').format(iso8601))
# future
self.assertEqual(
dm('+1s').format(iso8601),
arrow.utcnow().replace(seconds=+1).format(iso8601))
self.assertEqual(
dm('+1s+2m+3h').format(iso8601),
arrow.utcnow().replace(seconds=+1, minutes=+2,
hours=+3).format(iso8601))
self.assertEqual(
dm('+1m').format(iso8601),
arrow.utcnow().replace(minutes=+1).format(iso8601))
self.assertEqual(
dm('+1m+5h').format(iso8601),
arrow.utcnow().replace(minutes=+1, hours=+5).format(iso8601))
self.assertEqual(
dm('/d+1m+5h').format(iso8601),
arrow.utcnow().floor('day').replace(minutes=+1,
hours=+5).format(iso8601))
self.assertEqual(
dm('+1h').format(iso8601),
arrow.utcnow().replace(hours=+1).format(iso8601))
self.assertEqual(
dm('+1w').format(iso8601),
arrow.utcnow().replace(weeks=+1).format(iso8601))
self.assertEqual(
dm('+1w+12d').format(iso8601),
arrow.utcnow().replace(weeks=+1, days=+12).format(iso8601))
self.assertEqual(
dm('+2y').format(iso8601),
arrow.utcnow().replace(years=+2).format(iso8601))
self.assertEqual(
dm('+2y+22d+4h').format(iso8601),
arrow.utcnow().replace(years=+2, days=+22,
hours=+4).format(iso8601))
# past
self.assertEqual(
dm('-3w').format(iso8601),
arrow.utcnow().replace(weeks=-3).format(iso8601))
self.assertEqual(
dm('-3W').format(iso8601),
arrow.utcnow().replace(weeks=-3).format(iso8601))
self.assertEqual(
dm('-3w-2d-6h').format(iso8601),
arrow.utcnow().replace(weeks=-3, days=-2,
hours=-6).format(iso8601))
self.assertEqual(
dm('-3w-2d-22h-36s').format(iso8601),
arrow.utcnow().replace(weeks=-3, days=-2, hours=-22,
seconds=-36).format(iso8601))
self.assertEqual(
dm('-6y-3w-2d-22h-36s').format(iso8601),
arrow.utcnow().replace(years=-6,
weeks=-3,
days=-2,
hours=-22,
seconds=-36).format(iso8601))
import datetime
delta = datetime.timedelta(seconds=1)
# datetime objects
self.assertAlmostEqual(dm('now').datetime,
arrow.utcnow().datetime,
delta=delta)
self.assertAlmostEqual(dm('now+1d').datetime,
arrow.utcnow().replace(days=+1).datetime,
delta=delta)
self.assertAlmostEqual(dm('/w').datetime,
arrow.utcnow().floor('week').datetime,
delta=delta)
# Floats
self.assertEqual(
dm('now-2.5h').format(iso8601),
arrow.utcnow().replace(hours=-2.5).format(iso8601))
self.assertEqual(
dm('now-2.5d').format(iso8601),
arrow.utcnow().replace(days=-2.5).format(iso8601))
def dm_convert(timestr):
from datemath import dm
return dm(timestr).timestamp
import random
import string
import time
import warnings
from datemath import dm
from retrying import retry
import pytest
from assemblyline.datastore import Collection
with warnings.catch_warnings():
warnings.simplefilter("ignore")
test_map = {
'test1': {
'expiry_dt': dm('now-2d/m').isoformat().replace('+00:00', '.001Z'),
'lvl_i': 400,
'test1_s': 'hello',
'tags_ss': ['a', 'b', 'c']
},
'test2': {
'expiry_dt': dm('now-1d/m').isoformat().replace('+00:00', '.001Z'),
'lvl_i': 100,
'test2_s': 'hello',
'tags_ss': ['a', 'b', 'f']
},
'test3': {
'expiry_dt': dm('now/m').isoformat().replace('+00:00', '.001Z'),
'lvl_i': 200,
'test3_s': 'hello',
'tags_ss': ['a', 'b', 'e']
def import_cti_event(self,
timestamp: datetime,
data: dict,
is_update: bool = False) -> dict:
logger.debug(f"Querying indicator: { data['x_opencti_id']}")
entity = self.helper.api.indicator.read(id=data["x_opencti_id"])
logger.debug(entity)
_result: dict = {}
_document: Cut = {}
if data["type"] != "indicator":
logger.error(
f"Data type unsupported: {data['type']}. Only 'indicators are currently supported."
)
return None
if is_update is True:
update_time: str = (datetime.now(
tz=timezone.utc).isoformat().replace("+00:00", "Z"))
try:
# Attempt to retreive existing document
logger.debug(f"Retrieving document id: {data['x_opencti_id']}")
_result = self.es_client.get(index=self.idx_pattern,
id=data["x_opencti_id"],
doc_type="_doc")
except NotFoundError:
logger.warn(
f"Document not found to update at /{self.idx}/_doc/{data['x_opencti_id']}"
)
logger.warn("Skipping")
return {}
except RequestError as err:
logger.error(
f"Unexpected error retreiving document at /{self.idx}/_doc/{data['x_opencti_id']}:",
err.__dict__,
)
if _result["found"] is True:
_document = Cut(_result["_source"])
_write_idx = _result["_index"]
if data.get("x_data_update", None):
if data["x_data_update"].get("replace", None):
if entity["pattern_type"] == "stix":
# Pull in any indicator updates
_indicator: dict = self._create_ecs_indicator_stix(
entity)
if _indicator == {}:
return {}
_document["threatintel.indicator"] = _indicator
if entity.get("killChainPhases", None):
phases = []
for phase in sorted(
entity["killChainPhases"],
key=lambda i: (
i["kill_chain_name"],
i["x_opencti_order"],
),
):
phases.append({
"killchain_name":
phase["kill_chain_name"],
"phase_name":
phase["phase_name"],
"opencti_phase_order":
phase["x_opencti_order"],
})
_document.setdefault(
"threatintel.opencti.killchain_phases", phases)
else:
logger.warning(
f"Unsupported indicator pattern type: {entity['pattern_type']}. Skipping."
)
return _document
for k, v in data["x_data_update"].get("replace",
{}).items():
logger.debug(
f"Updating field {k} -> {entity_field_mapping.get(k)} to {v}"
)
try:
_field = entity_field_mapping.get(k)
_document.setdefault(_field, v)
_document[_field] = v
except KeyError as err:
logger.error(
f"Unable to find field mapping for {k}", err)
_document["threatintel.opencti.updated_at"] = update_time
# This scrubs the Cut object and returns a dict
_document = remove_nones(_document)
try:
# Don't render timestamped index since this is an update
# Submit to Elastic index
logger.debug(
f"Updating doc to {_write_idx}:\n {_document}")
self.es_client.index(
index=_write_idx,
id=data["x_opencti_id"],
body=_document,
)
except RequestError as err:
logger.error("Unexpected error:", err, _document)
except Exception as err:
logger.error("Something else happened", err, _document)
return _document
creation_time: str = (datetime.now(
tz=timezone.utc).isoformat().replace("+00:00", "Z"))
_document: dict = {
"@timestamp": timestamp,
"event": {
"created": creation_time,
"kind": "enrichment",
"category": "threat",
"type": "indicator",
"dataset": "threatintel.opencti",
},
"threatintel": {},
}
if len(entity.get("externalReferences", [])) > 0:
_document["event"]["reference"] = [
item.get("url", None) for item in entity["externalReferences"]
]
if self.config.get("opencti.platform_url") is not None:
_document["event"]["url"] = urllib.parse.urljoin(
f"{self.config.get('opencti.platform_url')}",
f"/dashboard/observations/indicators/{entity['id']}",
)
_document["event"]["risk_score"] = entity.get("x_opencti_score", None)
_document["event"]["risk_score_norm"] = entity.get(
"x_opencti_score", None)
_document["threatintel"]["confidence"] = entity.get("confidence", None)
_document["threatintel"]["confidence_norm"] = entity.get(
"confidence", None)
_document["threatintel"]["opencti"] = {
"internal_id": entity.get("id", None),
"valid_from": entity.get("valid_from", None),
"valid_until": entity.get("valid_until", None),
"enable_detection": entity.get("x_opencti_detection", None),
"original_pattern": entity.get("pattern", None),
"pattern_type": entity.get("pattern_type", None),
"created_at": entity.get("created_at", None),
"updated_at": entity.get("created_at", None),
"revoked": entity.get("revoked", None),
}
if entity.get("killChainPhases", None):
phases = []
for phase in sorted(
entity["killChainPhases"],
key=lambda i: (i["kill_chain_name"], i["x_opencti_order"]),
):
phases.append({
"killchain_name": phase["kill_chain_name"],
"phase_name": phase["phase_name"],
"opencti_phase_order": phase["x_opencti_order"],
})
_document["threatintel"]["opencti"]["killchain_phases"] = phases
if entity.get("x_mitre_platforms", None):
_document["threatintel"]["opencti"]["mitre"] = {
"platforms": entity.get("x_mitre_platforms", None)
}
if entity["pattern_type"] == "stix":
_indicator: dict = self._create_ecs_indicator_stix(entity)
if _indicator == {}:
return {}
_document["threatintel"]["stix"] = {
"id": entity.get("standard_id")
}
_document["threatintel"]["indicator"] = _indicator
else:
logger.warning(
f"Unsupported indicator pattern type: {entity['pattern_type']}. Skipping."
)
return {}
_document = remove_nones(_document)
try:
# Render date-specific index, if we're doing logstash style indices
_write_idx = self.write_idx
m = self.pattern.search(_write_idx)
if m is not None:
m = m.groupdict()
if m.get("modulo", None) is not None:
_fmt = m.get("format") or DM_DEFAULT_FMT
logger.debug(f"{m['modulo']} -> {_fmt}")
_val = dm(m.get("modulo"), now=timestamp).format(_fmt)
_write_idx = self.pattern.sub(_val, _write_idx)
# Submit to Elastic index
logger.debug(f"Indexing doc to {_write_idx}:\n {_document}")
self.es_client.index(
index=_write_idx,
id=data["x_opencti_id"],
body=_document,
)
except RequestError as err:
logger.error("Unexpected error:", err, _document)
except Exception as err:
logger.error("Something else happened", err, _document)
return _document
def testParse(self):
# Baisc dates
self.assertEqual(dm('2016.01.02').format(iso8601), '2016-01-02T00:00:00-00:00')
self.assertEqual(dm('2016-01-02').format(iso8601), '2016-01-02T00:00:00-00:00')
self.assertEqual(dm('2016-01-02 01:00:00').format(iso8601), '2016-01-02T01:00:00-00:00')
# Rounding Tests
self.assertEqual(dm('2016-01-01||/d').format('YYYY-MM-DDTHH:mm:ssZZ'), '2016-01-01T00:00:00-00:00')
self.assertEqual(dm('2014-11-18||/y').format('YYYY-MM-DDTHH:mm:ssZZ'), '2014-01-01T00:00:00-00:00')
self.assertEqual(dm('2016-01-01 14:00:00||/w').format('YYYY-MM-DDTHH:mm:ssZZ'), '2015-12-28T00:00:00-00:00')
self.assertEqual(dm('2014-11||/M').format('YYYY-MM-DDTHH:mm:ssZZ'), '2014-11-01T00:00:00-00:00')
self.assertEqual(dm('2016-01-02||/M+1h+1m').format(iso8601), '2016-01-01T01:01:00-00:00')
self.assertEqual(dm('2016-01-02||/d+1h').format(iso8601), '2016-01-02T01:00:00-00:00')
self.assertEqual(dm('2016-01-02T14:02:00||/h').format(iso8601), '2016-01-02T14:00:00-00:00')
self.assertEqual(dm('2016-01-02T14:02:00||/H').format(iso8601), '2016-01-02T14:00:00-00:00')
# relitive formats
# addition
self.assertEqual(dm('+1s').format(iso8601), arrow.utcnow().replace(seconds=+1).format(iso8601))
self.assertEqual(dm('+1m').format(iso8601), arrow.utcnow().replace(minutes=+1).format(iso8601))
self.assertEqual(dm('+1h').format(iso8601), arrow.utcnow().replace(hours=+1).format(iso8601))
self.assertEqual(dm('+1d').format(iso8601), arrow.utcnow().replace(days=+1).format(iso8601))
self.assertEqual(dm('+1w').format(iso8601), arrow.utcnow().replace(weeks=+1).format(iso8601))
self.assertEqual(dm('+1M').format(iso8601), arrow.utcnow().replace(months=+1).format(iso8601))
self.assertEqual(dm('+1Y').format(iso8601), arrow.utcnow().replace(years=+1).format(iso8601))
self.assertEqual(dm('+1y').format(iso8601), arrow.utcnow().replace(years=+1).format(iso8601))
# subtraction
self.assertEqual(dm('-1s').format(iso8601), arrow.utcnow().replace(seconds=-1).format(iso8601))
self.assertEqual(dm('-1m').format(iso8601), arrow.utcnow().replace(minutes=-1).format(iso8601))
self.assertEqual(dm('-1h').format(iso8601), arrow.utcnow().replace(hours=-1).format(iso8601))
self.assertEqual(dm('-1d').format(iso8601), arrow.utcnow().replace(days=-1).format(iso8601))
self.assertEqual(dm('-1w').format(iso8601), arrow.utcnow().replace(weeks=-1).format(iso8601))
self.assertEqual(dm('-1M').format(iso8601), arrow.utcnow().replace(months=-1).format(iso8601))
self.assertEqual(dm('-1Y').format(iso8601), arrow.utcnow().replace(years=-1).format(iso8601))
self.assertEqual(dm('-1y').format(iso8601), arrow.utcnow().replace(years=-1).format(iso8601))
# rounding
self.assertEqual(dm('/s').format(iso8601), arrow.utcnow().floor('second').format(iso8601))
self.assertEqual(dm('/m').format(iso8601), arrow.utcnow().floor('minute').format(iso8601))
self.assertEqual(dm('/h').format(iso8601), arrow.utcnow().floor('hour').format(iso8601))
self.assertEqual(dm('/d').format(iso8601), arrow.utcnow().floor('day').format(iso8601))
self.assertEqual(dm('/w').format(iso8601), arrow.utcnow().floor('week').format(iso8601))
self.assertEqual(dm('/M').format(iso8601), arrow.utcnow().floor('month').format(iso8601))
self.assertEqual(dm('/Y').format(iso8601), arrow.utcnow().floor('year').format(iso8601))
self.assertEqual(dm('/y').format(iso8601), arrow.utcnow().floor('year').format(iso8601))
# complicated date math
self.assertEqual(dm('now/d-1h').format(iso8601), arrow.utcnow().floor('day').replace(hours=-1).format(iso8601))
self.assertEqual(dm('+1h').format(iso8601), arrow.utcnow().replace(hours=+1).format(iso8601))
self.assertEqual(dm('/M+2d').format(iso8601), arrow.utcnow().floor('month').replace(days=+2).format(iso8601))
self.assertEqual(dm('now/w+2d-2h').format(iso8601), arrow.utcnow().floor('week').replace(days=+2, hours=-2).format(iso8601))
self.assertEqual(dm('now/M+1w-2h+10s').format(iso8601), arrow.utcnow().floor('month').replace(weeks=+1, hours=-2, seconds=+10).format(iso8601))
# future
self.assertEqual(dm('+1s').format(iso8601), arrow.utcnow().replace(seconds=+1).format(iso8601))
self.assertEqual(dm('+1s+2m+3h').format(iso8601), arrow.utcnow().replace(seconds=+1, minutes=+2, hours=+3).format(iso8601))
self.assertEqual(dm('+1m').format(iso8601), arrow.utcnow().replace(minutes=+1).format(iso8601))
self.assertEqual(dm('+1m+5h').format(iso8601), arrow.utcnow().replace(minutes=+1, hours=+5).format(iso8601))
self.assertEqual(dm('/d+1m+5h').format(iso8601), arrow.utcnow().floor('day').replace(minutes=+1, hours=+5).format(iso8601))
self.assertEqual(dm('+1h').format(iso8601), arrow.utcnow().replace(hours=+1).format(iso8601))
self.assertEqual(dm('+1w').format(iso8601), arrow.utcnow().replace(weeks=+1).format(iso8601))
self.assertEqual(dm('+1w+12d').format(iso8601), arrow.utcnow().replace(weeks=+1, days=+12).format(iso8601))
self.assertEqual(dm('+2y').format(iso8601), arrow.utcnow().replace(years=+2).format(iso8601))
self.assertEqual(dm('+2y+22d+4h').format(iso8601), arrow.utcnow().replace(years=+2, days=+22, hours=+4).format(iso8601))
# past
self.assertEqual(dm('-3w').format(iso8601), arrow.utcnow().replace(weeks=-3).format(iso8601))
self.assertEqual(dm('-3W').format(iso8601), arrow.utcnow().replace(weeks=-3).format(iso8601))
self.assertEqual(dm('-3w-2d-6h').format(iso8601), arrow.utcnow().replace(weeks=-3, days=-2, hours=-6).format(iso8601))
self.assertEqual(dm('-3w-2d-22h-36s').format(iso8601), arrow.utcnow().replace(weeks=-3, days=-2, hours=-22, seconds=-36).format(iso8601))
self.assertEqual(dm('-6y-3w-2d-22h-36s').format(iso8601), arrow.utcnow().replace(years=-6, weeks=-3, days=-2, hours=-22, seconds=-36).format(iso8601))
import datetime
delta = datetime.timedelta(seconds=1)
# datetime objects
self.assertAlmostEqual(dm('now').datetime, arrow.utcnow().datetime, delta=delta)
self.assertAlmostEqual(dm('now+1d').datetime, arrow.utcnow().replace(days=+1).datetime, delta=delta)
self.assertAlmostEqual(dm('/w').datetime, arrow.utcnow().floor('week').datetime, delta=delta)
# Floats
self.assertEqual(dm('now-2.5h').format(iso8601), arrow.utcnow().replace(hours=-2.5).format(iso8601))
self.assertEqual(dm('now-2.5d').format(iso8601), arrow.utcnow().replace(days=-2.5).format(iso8601))
def run_expiry_once(self):
now = now_as_iso()
reached_max = False
# Expire data
for collection in self.expirable_collections:
# Call heartbeat pre-dated by 5 minutes. If a collection takes more than
# 5 minutes to expire, this container could be seen as unhealthy. The down
# side is if it is stuck on something it will be more than 5 minutes before
# the container is restarted.
self.heartbeat(int(time.time() + 5 * 60))
# Start of expiry transaction
if self.apm_client:
self.apm_client.begin_transaction("Delete expired documents")
if self.config.core.expiry.batch_delete:
computed_date = epoch_to_iso(
dm(f"{now}||-{self.config.core.expiry.delay}h/d").
float_timestamp)
else:
computed_date = epoch_to_iso(
dm(f"{now}||-{self.config.core.expiry.delay}h").
float_timestamp)
delete_query = f"expiry_ts:[* TO {computed_date}]"
if self.config.core.expiry.delete_storage and collection.name in self.fs_hashmap:
file_delete = True
sort = ["expiry_ts asc", "id asc"]
else:
file_delete = False
sort = None
number_to_delete = collection.search(
delete_query,
rows=0,
as_obj=False,
use_archive=True,
sort=sort,
track_total_hits=EXPIRY_SIZE)['total']
if self.apm_client:
elasticapm.label(query=delete_query)
elasticapm.label(number_to_delete=number_to_delete)
self.log.info(f"Processing collection: {collection.name}")
if number_to_delete != 0:
if file_delete:
with elasticapm.capture_span(
name='FILESTORE [ThreadPoolExecutor] :: delete()',
labels={
"num_files": number_to_delete,
"query": delete_query
}):
# Delete associated files
with concurrent.futures.ThreadPoolExecutor(
self.config.core.expiry.workers,
thread_name_prefix="file_delete") as executor:
for item in collection.search(
delete_query,
fl='id',
rows=number_to_delete,
sort=sort,
use_archive=True,
as_obj=False)['items']:
executor.submit(
self.fs_hashmap[collection.name],
item['id'], computed_date)
self.log.info(
f' Deleted associated files from the '
f'{"cachestore" if "cache" in collection.name else "filestore"}...'
)
# Proceed with deletion
collection.delete_by_query(
delete_query,
workers=self.config.core.expiry.workers,
sort=sort,
max_docs=number_to_delete)
else:
# Proceed with deletion
collection.delete_by_query(
delete_query, workers=self.config.core.expiry.workers)
if number_to_delete == EXPIRY_SIZE:
reached_max = True
self.counter.increment(f'{collection.name}',
increment_by=number_to_delete)
self.log.info(
f" Deleted {number_to_delete} items from the datastore..."
)
else:
self.log.debug(" Nothing to delete in this collection.")
# End of expiry transaction
if self.apm_client:
self.apm_client.end_transaction(collection.name, 'deleted')
return reached_max
def testParse(self):
# Baisc dates
self.assertEqual(
dm('2016.01.02').format(iso8601), '2016-01-02T00:00:00+00:00')
self.assertEqual(
dm('2016-01-02').format(iso8601), '2016-01-02T00:00:00+00:00')
self.assertEqual(
dm('2016-01-02 01:00:00').format(iso8601),
'2016-01-02T01:00:00+00:00')
# Rounding Tests
self.assertEqual(
dm('2016-01-01||/d').format('YYYY-MM-DDTHH:mm:ssZZ'),
'2016-01-01T00:00:00+00:00')
self.assertEqual(
dm('2014-11-18||/y').format('YYYY-MM-DDTHH:mm:ssZZ'),
'2014-01-01T00:00:00+00:00')
self.assertEqual(
dm('2016-01-01 14:00:00||/w').format('YYYY-MM-DDTHH:mm:ssZZ'),
'2015-12-28T00:00:00+00:00')
self.assertEqual(
dm('2014-11||/M').format('YYYY-MM-DDTHH:mm:ssZZ'),
'2014-11-01T00:00:00+00:00')
self.assertEqual(
dm('2016-01-02||/M+1h+1m').format(iso8601),
'2016-01-01T01:01:00+00:00')
self.assertEqual(
dm('2016-01-02||/d+1h').format(iso8601),
'2016-01-02T01:00:00+00:00')
self.assertEqual(
dm('2016-01-02T14:02:00||/h').format(iso8601),
'2016-01-02T14:00:00+00:00')
self.assertEqual(
dm('2016-01-02T14:02:00||/H').format(iso8601),
'2016-01-02T14:00:00+00:00')
# Rounding Up Tests
self.assertEqual(
dm('2016-01-01||/d',
roundDown=False).format('YYYY-MM-DDTHH:mm:ssZZ'),
'2016-01-01T23:59:59+00:00')
self.assertEqual(
dm('2014-11-18||/y',
roundDown=False).format('YYYY-MM-DDTHH:mm:ssZZ'),
'2014-12-31T23:59:59+00:00')
# Timezone Tests
self.assertEqual(
dm('now', tz='US/Pacific').format(iso8601),
arrow.utcnow().to('US/Pacific').format(iso8601))
self.assertEqual(
dm('2017-09-22 10:20:00', tz='US/Pacific').datetime,
pydatetime(2017, 9, 22, 10, 20, 00, tzinfo=tz.gettz('US/Pacific')))
self.assertEqual(dm('2016-01-01', tz='UTC'),
arrow.get('2016-01-01').to('UTC'))
self.assertEqual(dm('2016-01-01', tz='US/Eastern'),
pydatetime(2016, 1, 1, tzinfo=tz.gettz('US/Eastern')))
self.assertEqual(
datemath('2016-01-01T01:00:00', tz='US/Central'),
pydatetime(2016, 1, 1, 1, 0, 0, tzinfo=tz.gettz('US/Central')))
self.assertEqual(
datemath('2016-01-01T02:00:00', tz='US/Eastern'),
pydatetime(2016, 1, 1, 2, tzinfo=tz.gettz('US/Eastern')))
# relitive formats
# addition
self.assertEqual(
dm('+1s').format(iso8601),
arrow.utcnow().shift(seconds=+1).format(iso8601))
self.assertEqual(
dm('+1m').format(iso8601),
arrow.utcnow().shift(minutes=+1).format(iso8601))
self.assertEqual(
dm('+1h').format(iso8601),
arrow.utcnow().shift(hours=+1).format(iso8601))
self.assertEqual(
dm('+1d').format(iso8601),
arrow.utcnow().shift(days=+1).format(iso8601))
self.assertEqual(
dm('+1w').format(iso8601),
arrow.utcnow().shift(weeks=+1).format(iso8601))
self.assertEqual(
dm('+1M').format(iso8601),
arrow.utcnow().shift(months=+1).format(iso8601))
self.assertEqual(
dm('+1Y').format(iso8601),
arrow.utcnow().shift(years=+1).format(iso8601))
self.assertEqual(
dm('+1y').format(iso8601),
arrow.utcnow().shift(years=+1).format(iso8601))
# subtraction
self.assertEqual(
dm('-1s').format(iso8601),
arrow.utcnow().shift(seconds=-1).format(iso8601))
self.assertEqual(
dm('-1m').format(iso8601),
arrow.utcnow().shift(minutes=-1).format(iso8601))
self.assertEqual(
dm('-1h').format(iso8601),
arrow.utcnow().shift(hours=-1).format(iso8601))
self.assertEqual(
dm('-1d').format(iso8601),
arrow.utcnow().shift(days=-1).format(iso8601))
self.assertEqual(
dm('-1w').format(iso8601),
arrow.utcnow().shift(weeks=-1).format(iso8601))
self.assertEqual(
dm('-1M').format(iso8601),
arrow.utcnow().shift(months=-1).format(iso8601))
self.assertEqual(
dm('-1Y').format(iso8601),
arrow.utcnow().shift(years=-1).format(iso8601))
self.assertEqual(
dm('-1y').format(iso8601),
arrow.utcnow().shift(years=-1).format(iso8601))
# rounding
self.assertEqual(
dm('/s').format(iso8601),
arrow.utcnow().floor('second').format(iso8601))
self.assertEqual(
dm('/m').format(iso8601),
arrow.utcnow().floor('minute').format(iso8601))
self.assertEqual(
dm('/h').format(iso8601),
arrow.utcnow().floor('hour').format(iso8601))
self.assertEqual(
dm('/d').format(iso8601),
arrow.utcnow().floor('day').format(iso8601))
self.assertEqual(
dm('/w').format(iso8601),
arrow.utcnow().floor('week').format(iso8601))
self.assertEqual(
dm('/M').format(iso8601),
arrow.utcnow().floor('month').format(iso8601))
self.assertEqual(
dm('/Y').format(iso8601),
arrow.utcnow().floor('year').format(iso8601))
self.assertEqual(
dm('/y').format(iso8601),
arrow.utcnow().floor('year').format(iso8601))
# rounding up
self.assertEqual(
dm('/s', roundDown=False).format(iso8601),
arrow.utcnow().ceil('second').format(iso8601))
self.assertEqual(
dm('/m', roundDown=False).format(iso8601),
arrow.utcnow().ceil('minute').format(iso8601))
self.assertEqual(
dm('/h', roundDown=False).format(iso8601),
arrow.utcnow().ceil('hour').format(iso8601))
self.assertEqual(
dm('/d', roundDown=False).format(iso8601),
arrow.utcnow().ceil('day').format(iso8601))
self.assertEqual(
dm('/w', roundDown=False).format(iso8601),
arrow.utcnow().ceil('week').format(iso8601))
self.assertEqual(
dm('/M', roundDown=False).format(iso8601),
arrow.utcnow().ceil('month').format(iso8601))
self.assertEqual(
dm('/Y', roundDown=False).format(iso8601),
arrow.utcnow().ceil('year').format(iso8601))
self.assertEqual(
dm('/y', roundDown=False).format(iso8601),
arrow.utcnow().ceil('year').format(iso8601))
# roundDown option tests
self.assertEqual(
dm('2016-01-01T14:00:00||/d').format(iso8601),
'2016-01-01T00:00:00+00:00')
self.assertEqual(
dm('2016-01-01T14:00:00||/d', roundDown=False).format(iso8601),
'2016-01-01T23:59:59+00:00')
# complicated date math
self.assertEqual(
dm('now/d-1h').format(iso8601),
arrow.utcnow().floor('day').shift(hours=-1).format(iso8601))
self.assertEqual(
dm('+1h').format(iso8601),
arrow.utcnow().shift(hours=+1).format(iso8601))
self.assertEqual(
dm('/M+2d').format(iso8601),
arrow.utcnow().floor('month').shift(days=+2).format(iso8601))
self.assertEqual(
dm('now/w+2d-2h').format(iso8601),
arrow.utcnow().floor('week').shift(days=+2,
hours=-2).format(iso8601))
self.assertEqual(
dm('now/M+1w-2h+10s').format(iso8601),
arrow.utcnow().floor('month').shift(weeks=+1,
hours=-2,
seconds=+10).format(iso8601))
self.assertEqual(
dm('now-1d/d').format(iso8601),
arrow.utcnow().shift(days=-1).floor('day').format(iso8601))
self.assertEqual(
dm('now+1d/d').format(iso8601),
arrow.utcnow().shift(days=1).floor('day').format(iso8601))
self.assertEqual(
dm('now-10d/d').format(iso8601),
arrow.utcnow().shift(days=-10).floor('day').format(iso8601))
self.assertEqual(
dm('now+10d/d').format(iso8601),
arrow.utcnow().shift(days=10).floor('day').format(iso8601))
self.assertEqual(
dm('now-29d/d').format(iso8601),
arrow.utcnow().shift(days=-29).floor('day').format(iso8601))
# future
self.assertEqual(
dm('+1s').format(iso8601),
arrow.utcnow().shift(seconds=+1).format(iso8601))
self.assertEqual(
dm('+1s+2m+3h').format(iso8601),
arrow.utcnow().shift(seconds=+1, minutes=+2,
hours=+3).format(iso8601))
self.assertEqual(
dm('+1m').format(iso8601),
arrow.utcnow().shift(minutes=+1).format(iso8601))
self.assertEqual(
dm('+1m+5h').format(iso8601),
arrow.utcnow().shift(minutes=+1, hours=+5).format(iso8601))
self.assertEqual(
dm('/d+1m+5h').format(iso8601),
arrow.utcnow().floor('day').shift(minutes=+1,
hours=+5).format(iso8601))
self.assertEqual(
dm('+1h').format(iso8601),
arrow.utcnow().shift(hours=+1).format(iso8601))
self.assertEqual(
dm('+1w').format(iso8601),
arrow.utcnow().shift(weeks=+1).format(iso8601))
self.assertEqual(
dm('+1w+12d').format(iso8601),
arrow.utcnow().shift(weeks=+1, days=+12).format(iso8601))
self.assertEqual(
dm('+2y').format(iso8601),
arrow.utcnow().shift(years=+2).format(iso8601))
self.assertEqual(
dm('+2y+22d+4h').format(iso8601),
arrow.utcnow().shift(years=+2, days=+22, hours=+4).format(iso8601))
# past
self.assertEqual(
dm('-3w').format(iso8601),
arrow.utcnow().shift(weeks=-3).format(iso8601))
self.assertEqual(
dm('-3W').format(iso8601),
arrow.utcnow().shift(weeks=-3).format(iso8601))
self.assertEqual(
dm('-3w-2d-6h').format(iso8601),
arrow.utcnow().shift(weeks=-3, days=-2, hours=-6).format(iso8601))
self.assertEqual(
dm('-3w-2d-22h-36s').format(iso8601),
arrow.utcnow().shift(weeks=-3, days=-2, hours=-22,
seconds=-36).format(iso8601))
self.assertEqual(
dm('-6y-3w-2d-22h-36s').format(iso8601),
arrow.utcnow().shift(years=-6,
weeks=-3,
days=-2,
hours=-22,
seconds=-36).format(iso8601))
import datetime
delta = datetime.timedelta(seconds=1)
# datetime objects
self.assertAlmostEqual(dm('now').datetime,
arrow.utcnow().datetime,
delta=delta)
self.assertAlmostEqual(dm('now+1d').datetime,
arrow.utcnow().shift(days=+1).datetime,
delta=delta)
self.assertAlmostEqual(dm('/w').datetime,
arrow.utcnow().floor('week').datetime,
delta=delta)
# Floats
self.assertEqual(
dm('now-2.5h').format(iso8601),
arrow.utcnow().shift(hours=-2.5).format(iso8601))
self.assertEqual(
dm('now-2.5d').format(iso8601),
arrow.utcnow().shift(days=-2.5).format(iso8601))
# Test epoch timestamps
self.assertEqual(
dm('1451610061').format(iso8601), '2016-01-01T01:01:01+00:00')
self.assertEqual(
dm(1367900664).format(iso8601), '2013-05-07T04:24:24+00:00')
self.assertRaises(DateMathException, dm, '1451610061000')
try:
dm(1451610061000)
except DateMathException as e:
self.assertTrue(
'Unable to parse epoch timestamps in millis' in str(e))
# Catch invalid timeunits
self.assertRaises(DateMathException, dm, '+1,')
self.assertRaises(DateMathException, dm, '+1.')
self.assertRaises(DateMathException, dm, '+1ö')
self.assertRaises(DateMathException, dm, '+1ä')
self.assertRaises(DateMathException, dm, '+1ü')
self.assertRaises(DateMathException, dm, '+1ß')
try:
dm('+1,')
except DateMathException as e:
self.assertTrue('is not a valid timeunit' in str(e))
def dm_convert(timestr):
from datemath import dm
return dm(timestr).timestamp