def __init__(self, query, **kwargs): super(TQL, self).__init__(**kwargs) if self._start is None: raise TQLException("Searches require at least a start time with either a timestamp or datemath expression: start='now-10m', end='now' | start='20160101', end='now'") try: self._query_raw = query self._query = query self._start_time = dm(self._start) self._end_time = dm(self._end) self._index = tattle.get_indexes(self._index_name_pattern, self._start_time, self._end_time, pattern=self._index_ts_pattern) except TQLException as e: raise TQLArgsException("Unable to set arguments for TQL, I am missing: %s" % (e)) self._start_time_iso_str = self._start_time.format(self._ISO_TS) self._end_time_iso_str = self._end_time.format(self._ISO_TS) self._start_time_pretty = self._start_time.format(self._PRETTY_TS) self._end_time_pretty = self._end_time.format(self._PRETTY_TS) self._start_time_epoch = int(self._start_time.format('X')) self._start_time_epoch_millis = self._start_time_epoch * 1000 self._end_time_epoch = int(self._end_time.format('X')) self._end_time_epoch_millis = self._end_time_epoch * 1000 self._start_time_unix = self._start_time_epoch self._end_time_unix = self._end_time_epoch self._qd = self.get_intentions(self._query_raw) self.build_es_query()
def __init__(self, **kwargs): self._ISO_TS = 'YYYY-MM-DDTHH:mm:ssZZ' self._PRETTY_TS = 'MMM D YYYY, HH:mm:ss ZZ' self.agg_size_from = 0 self.agg_size = 0 self.hit_size_from = 0 self.hit_size = 10000 self._query_raw = '' self._query = '' self._start = None self._start_time = dm('now-1m') self._end_time = dm('now') self.exclude = '' self._ts_field = '@timestamp' self._index_ts_pattern = 'YYYY.MM.DD' self._index_name_pattern = 'logstash-*' for key, value in kwargs.items(): if key in ('start', 'from', '_from'): self._start = value elif key in ('end', 'to', '_to'): self._end = value setattr(self, key, value) self._index = tattle.get_indexes(self.index, self._start_time, self._end_time, pattern=self._index_ts_pattern)
def __init__(self, **kwargs): self._ISO_TS = 'YYYY-MM-DDTHH:mm:ssZZ' self._PRETTY_TS = 'MMM D YYYY, HH:mm:ss ZZ' self.agg_size_from = 0 self.agg_size = 0 self.hit_size_from = 0 self.hit_size = 10000 self._query_raw = '' self._query = '' self._start = None self._start_time = dm('now-1m') self._end_time = dm('now') self.exclude = '' self._ts_field = '@timestamp' self._index_ts_pattern = 'YYYY.MM.DD' self._index_name_pattern = 'logstash-*' for key, value in kwargs.items(): if key in ('start', 'from', '_from'): self._start = value elif key in ('end', 'to', '_to'): self._end = value setattr(self, key, value) self._index = tattle.get_indexes(self.index, self._start_time, self._end_time, pattern=self._index_ts_pattern)
def __init__(self, query, **kwargs): super(TQL, self).__init__(**kwargs) if self._start is None: raise TQLException( "Searches require at least a start time with either a timestamp or datemath expression: start='now-10m', end='now' | start='20160101', end='now'" ) try: self._query_raw = query self._query = query self._start_time = dm(self._start) self._end_time = dm(self._end) self._index = tattle.get_indexes(self._index_name_pattern, self._start_time, self._end_time, pattern=self._index_ts_pattern) except TQLException as e: raise TQLArgsException( "Unable to set arguments for TQL, I am missing: %s" % (e)) self._start_time_iso_str = self._start_time.format(self._ISO_TS) self._end_time_iso_str = self._end_time.format(self._ISO_TS) self._start_time_pretty = self._start_time.format(self._PRETTY_TS) self._end_time_pretty = self._end_time.format(self._PRETTY_TS) self._start_time_epoch = int(self._start_time.format('X')) self._start_time_epoch_millis = self._start_time_epoch * 1000 self._end_time_epoch = int(self._end_time.format('X')) self._end_time_epoch_millis = self._end_time_epoch * 1000 self._start_time_unix = self._start_time_epoch self._end_time_unix = self._end_time_epoch self._qd = self.get_intentions(self._query_raw) self.build_es_query()
def import_cti_event(self, timestamp: datetime, data: dict, is_update: bool = False) -> dict: try: # Submit to Elastic index _document = data _document["@timestamp"] = timestamp _write_idx = self.idx # Render date-specific index if we're using logstash style indices m = self.pattern.search(_write_idx) if m is not None: m = m.groupdict() if m.get("modulo", None) is not None: _fmt = m.get("format") or DM_DEFAULT_FMT logger.debug(f"{m['modulo']} -> {_fmt}") _val = dm(m.get("modulo"), now=timestamp).format(_fmt) _write_idx = self.pattern.sub(_val, _write_idx) # Submit to Elastic index logger.debug(f"Indexing doc to {_write_idx}:\n {_document}") self.es_client.index( index=_write_idx, id=data["x_opencti_id"], body=_document, ) except RequestError as err: logger.error("Unexpected error:", err, data) except Exception as err: logger.error("Something else happened", err, data) return data
def _validate_steps_count(self, start, end, gap): with warnings.catch_warnings(): warnings.simplefilter("ignore") gaps_count = None ret_type = None try: start = int(start) end = int(end) gap = int(gap) gaps_count = int((end - start) / gap) ret_type = int except ValueError: pass if not gaps_count: if not (gap.startswith("-") or gap.startswith("+")): raise SearchException( "Gap must be preceded with either + or - sign.") try: parsed_start = dm( self.datastore.to_pydatemath(start)).int_timestamp parsed_end = dm( self.datastore.to_pydatemath(end)).int_timestamp parsed_gap = dm(self.datastore.to_pydatemath( gap)).int_timestamp - dm('now').int_timestamp gaps_count = int((parsed_end - parsed_start) / parsed_gap) ret_type = str except (DateMathException, AttributeError): pass if not gaps_count: raise SearchException( "Could not parse date ranges. (start='%s', end='%s', gap='%s')" % (start, end, gap)) if gaps_count > self.MAX_FACET_LIMIT: raise SearchException( f'Histograms are limited to a maximum of {self.MAX_FACET_LIMIT} steps. ' f'Current settings would generate {gaps_count} steps') return ret_type
import random import string import time import warnings from datemath import dm from retrying import retry import pytest from assemblyline.datastore import Collection with warnings.catch_warnings(): warnings.simplefilter("ignore") test_map = { 'test1': {'expiry_dt': dm('now-2d/m').isoformat().replace('+00:00', '.001Z'), 'lvl_i': 400, 'test1_s': 'hello', 'tags_ss': ['a', 'b', 'c']}, 'test2': {'expiry_dt': dm('now-1d/m').isoformat().replace('+00:00', '.001Z'), 'lvl_i': 100, 'test2_s': 'hello', 'tags_ss': ['a', 'b', 'f']}, 'test3': {'expiry_dt': dm('now/m').isoformat().replace('+00:00', '.001Z'), 'lvl_i': 200, 'test3_s': 'hello', 'tags_ss': ['a', 'b', 'e']}, 'test4': {'expiry_dt': dm('now-2d/m').isoformat().replace('+00:00', '.001Z'), 'lvl_i': 400, 'test4_s': 'hello', 'tags_ss': ['a', 'b', 'd']}, 'dict1': {'expiry_dt': dm('now-2d/m').isoformat().replace('+00:00', '.001Z'), 'lvl_i': 400, 'classification_s': 'U', 'test1_s': 'hello', 'tags_ss': []}, 'dict2': {'expiry_dt': dm('now/m').isoformat().replace('+00:00', '.001Z'), 'lvl_i': 100, 'classification_s': 'U', 'test2_s': 'hello', 'tags_ss': []}, 'dict3': {'expiry_dt': dm('now-3d/m').isoformat().replace('+00:00', '.001Z'), 'lvl_i': 200, 'classification_s': 'C', 'test3_s': 'hello', 'tags_ss': []}, 'dict4': {'expiry_dt': dm('now-1d/m').isoformat().replace('+00:00', '.001Z'), 'lvl_i': 400, 'classification_s': 'TS', 'test4_s': 'hello', 'tags_ss': []},
def testParse(self): # Baisc dates self.assertEqual( dm('2016.01.02').format(iso8601), '2016-01-02T00:00:00-00:00') self.assertEqual( dm('2016-01-02').format(iso8601), '2016-01-02T00:00:00-00:00') self.assertEqual( dm('2016-01-02 01:00:00').format(iso8601), '2016-01-02T01:00:00-00:00') # Rounding Tests self.assertEqual( dm('2016-01-01||/d').format('YYYY-MM-DDTHH:mm:ssZZ'), '2016-01-01T00:00:00-00:00') self.assertEqual( dm('2014-11-18||/y').format('YYYY-MM-DDTHH:mm:ssZZ'), '2014-01-01T00:00:00-00:00') self.assertEqual( dm('2016-01-01 14:00:00||/w').format('YYYY-MM-DDTHH:mm:ssZZ'), '2015-12-28T00:00:00-00:00') self.assertEqual( dm('2014-11||/M').format('YYYY-MM-DDTHH:mm:ssZZ'), '2014-11-01T00:00:00-00:00') self.assertEqual( dm('2016-01-02||/M+1h+1m').format(iso8601), '2016-01-01T01:01:00-00:00') self.assertEqual( dm('2016-01-02||/d+1h').format(iso8601), '2016-01-02T01:00:00-00:00') self.assertEqual( dm('2016-01-02T14:02:00||/h').format(iso8601), '2016-01-02T14:00:00-00:00') self.assertEqual( dm('2016-01-02T14:02:00||/H').format(iso8601), '2016-01-02T14:00:00-00:00') # Rounding Up Tests self.assertEqual( dm('2016-01-01||/d', roundDown=False).format('YYYY-MM-DDTHH:mm:ssZZ'), '2016-01-01T23:59:59-00:00') self.assertEqual( dm('2014-11-18||/y', roundDown=False).format('YYYY-MM-DDTHH:mm:ssZZ'), '2014-12-31T23:59:59-00:00') # Timezone Tests self.assertEqual( dm('now', tz='US/Pacific').format(iso8601), arrow.utcnow().to('US/Pacific').format(iso8601)) self.assertEqual( dm('2017-09-22 10:20:00', tz='US/Pacific').datetime, pydatetime(2017, 9, 22, 10, 20, 00, tzinfo=tz.gettz('US/Pacific'))) self.assertEqual(dm('2016-01-01', tz='UTC'), arrow.get('2016-01-01').to('UTC')) self.assertEqual(dm('2016-01-01', tz='US/Eastern'), pydatetime(2016, 1, 1, tzinfo=tz.gettz('US/Eastern'))) self.assertEqual( datemath('2016-01-01T01:00:00', tz='US/Central'), pydatetime(2016, 1, 1, 1, 0, 0, tzinfo=tz.gettz('US/Central'))) # relitive formats # addition self.assertEqual( dm('+1s').format(iso8601), arrow.utcnow().replace(seconds=+1).format(iso8601)) self.assertEqual( dm('+1m').format(iso8601), arrow.utcnow().replace(minutes=+1).format(iso8601)) self.assertEqual( dm('+1h').format(iso8601), arrow.utcnow().replace(hours=+1).format(iso8601)) self.assertEqual( dm('+1d').format(iso8601), arrow.utcnow().replace(days=+1).format(iso8601)) self.assertEqual( dm('+1w').format(iso8601), arrow.utcnow().replace(weeks=+1).format(iso8601)) self.assertEqual( dm('+1M').format(iso8601), arrow.utcnow().replace(months=+1).format(iso8601)) self.assertEqual( dm('+1Y').format(iso8601), arrow.utcnow().replace(years=+1).format(iso8601)) self.assertEqual( dm('+1y').format(iso8601), arrow.utcnow().replace(years=+1).format(iso8601)) # subtraction self.assertEqual( dm('-1s').format(iso8601), arrow.utcnow().replace(seconds=-1).format(iso8601)) self.assertEqual( dm('-1m').format(iso8601), arrow.utcnow().replace(minutes=-1).format(iso8601)) self.assertEqual( dm('-1h').format(iso8601), arrow.utcnow().replace(hours=-1).format(iso8601)) self.assertEqual( dm('-1d').format(iso8601), arrow.utcnow().replace(days=-1).format(iso8601)) self.assertEqual( dm('-1w').format(iso8601), arrow.utcnow().replace(weeks=-1).format(iso8601)) self.assertEqual( dm('-1M').format(iso8601), arrow.utcnow().replace(months=-1).format(iso8601)) self.assertEqual( dm('-1Y').format(iso8601), arrow.utcnow().replace(years=-1).format(iso8601)) self.assertEqual( dm('-1y').format(iso8601), arrow.utcnow().replace(years=-1).format(iso8601)) # rounding self.assertEqual( dm('/s').format(iso8601), arrow.utcnow().floor('second').format(iso8601)) self.assertEqual( dm('/m').format(iso8601), arrow.utcnow().floor('minute').format(iso8601)) self.assertEqual( dm('/h').format(iso8601), arrow.utcnow().floor('hour').format(iso8601)) self.assertEqual( dm('/d').format(iso8601), arrow.utcnow().floor('day').format(iso8601)) self.assertEqual( dm('/w').format(iso8601), arrow.utcnow().floor('week').format(iso8601)) self.assertEqual( dm('/M').format(iso8601), arrow.utcnow().floor('month').format(iso8601)) self.assertEqual( dm('/Y').format(iso8601), arrow.utcnow().floor('year').format(iso8601)) self.assertEqual( dm('/y').format(iso8601), arrow.utcnow().floor('year').format(iso8601)) # rounding up self.assertEqual( dm('/s', roundDown=False).format(iso8601), arrow.utcnow().ceil('second').format(iso8601)) self.assertEqual( dm('/m', roundDown=False).format(iso8601), arrow.utcnow().ceil('minute').format(iso8601)) self.assertEqual( dm('/h', roundDown=False).format(iso8601), arrow.utcnow().ceil('hour').format(iso8601)) self.assertEqual( dm('/d', roundDown=False).format(iso8601), arrow.utcnow().ceil('day').format(iso8601)) self.assertEqual( dm('/w', roundDown=False).format(iso8601), arrow.utcnow().ceil('week').format(iso8601)) self.assertEqual( dm('/M', roundDown=False).format(iso8601), arrow.utcnow().ceil('month').format(iso8601)) self.assertEqual( dm('/Y', roundDown=False).format(iso8601), arrow.utcnow().ceil('year').format(iso8601)) self.assertEqual( dm('/y', roundDown=False).format(iso8601), arrow.utcnow().ceil('year').format(iso8601)) # roundDown option tests self.assertEqual( dm('2016-01-01T14:00:00||/d').format(iso8601), '2016-01-01T00:00:00-00:00') self.assertEqual( dm('2016-01-01T14:00:00||/d', roundDown=False).format(iso8601), '2016-01-01T23:59:59-00:00') # complicated date math self.assertEqual( dm('now/d-1h').format(iso8601), arrow.utcnow().floor('day').replace(hours=-1).format(iso8601)) self.assertEqual( dm('+1h').format(iso8601), arrow.utcnow().replace(hours=+1).format(iso8601)) self.assertEqual( dm('/M+2d').format(iso8601), arrow.utcnow().floor('month').replace(days=+2).format(iso8601)) self.assertEqual( dm('now/w+2d-2h').format(iso8601), arrow.utcnow().floor('week').replace(days=+2, hours=-2).format(iso8601)) self.assertEqual( dm('now/M+1w-2h+10s').format(iso8601), arrow.utcnow().floor('month').replace(weeks=+1, hours=-2, seconds=+10).format(iso8601)) self.assertEqual( dm('now-1d/d').format(iso8601), arrow.utcnow().replace(days=-1).floor('day').format(iso8601)) self.assertEqual( dm('now+1d/d').format(iso8601), arrow.utcnow().replace(days=1).floor('day').format(iso8601)) self.assertEqual( dm('now-10d/d').format(iso8601), arrow.utcnow().replace(days=-10).floor('day').format(iso8601)) self.assertEqual( dm('now+10d/d').format(iso8601), arrow.utcnow().replace(days=10).floor('day').format(iso8601)) # future self.assertEqual( dm('+1s').format(iso8601), arrow.utcnow().replace(seconds=+1).format(iso8601)) self.assertEqual( dm('+1s+2m+3h').format(iso8601), arrow.utcnow().replace(seconds=+1, minutes=+2, hours=+3).format(iso8601)) self.assertEqual( dm('+1m').format(iso8601), arrow.utcnow().replace(minutes=+1).format(iso8601)) self.assertEqual( dm('+1m+5h').format(iso8601), arrow.utcnow().replace(minutes=+1, hours=+5).format(iso8601)) self.assertEqual( dm('/d+1m+5h').format(iso8601), arrow.utcnow().floor('day').replace(minutes=+1, hours=+5).format(iso8601)) self.assertEqual( dm('+1h').format(iso8601), arrow.utcnow().replace(hours=+1).format(iso8601)) self.assertEqual( dm('+1w').format(iso8601), arrow.utcnow().replace(weeks=+1).format(iso8601)) self.assertEqual( dm('+1w+12d').format(iso8601), arrow.utcnow().replace(weeks=+1, days=+12).format(iso8601)) self.assertEqual( dm('+2y').format(iso8601), arrow.utcnow().replace(years=+2).format(iso8601)) self.assertEqual( dm('+2y+22d+4h').format(iso8601), arrow.utcnow().replace(years=+2, days=+22, hours=+4).format(iso8601)) # past self.assertEqual( dm('-3w').format(iso8601), arrow.utcnow().replace(weeks=-3).format(iso8601)) self.assertEqual( dm('-3W').format(iso8601), arrow.utcnow().replace(weeks=-3).format(iso8601)) self.assertEqual( dm('-3w-2d-6h').format(iso8601), arrow.utcnow().replace(weeks=-3, days=-2, hours=-6).format(iso8601)) self.assertEqual( dm('-3w-2d-22h-36s').format(iso8601), arrow.utcnow().replace(weeks=-3, days=-2, hours=-22, seconds=-36).format(iso8601)) self.assertEqual( dm('-6y-3w-2d-22h-36s').format(iso8601), arrow.utcnow().replace(years=-6, weeks=-3, days=-2, hours=-22, seconds=-36).format(iso8601)) import datetime delta = datetime.timedelta(seconds=1) # datetime objects self.assertAlmostEqual(dm('now').datetime, arrow.utcnow().datetime, delta=delta) self.assertAlmostEqual(dm('now+1d').datetime, arrow.utcnow().replace(days=+1).datetime, delta=delta) self.assertAlmostEqual(dm('/w').datetime, arrow.utcnow().floor('week').datetime, delta=delta) # Floats self.assertEqual( dm('now-2.5h').format(iso8601), arrow.utcnow().replace(hours=-2.5).format(iso8601)) self.assertEqual( dm('now-2.5d').format(iso8601), arrow.utcnow().replace(days=-2.5).format(iso8601))
def safe_date(pattern): return dm(f'{pattern}/m').isoformat().replace('+00:00', '.001Z')
def testParse(self): # Baisc dates self.assertEqual( dm('2016.01.02').format(iso8601), '2016-01-02T00:00:00-00:00') self.assertEqual( dm('2016-01-02').format(iso8601), '2016-01-02T00:00:00-00:00') self.assertEqual( dm('2016-01-02 01:00:00').format(iso8601), '2016-01-02T01:00:00-00:00') # Rounding Tests self.assertEqual( dm('2016-01-01||/d').format('YYYY-MM-DDTHH:mm:ssZZ'), '2016-01-01T00:00:00-00:00') self.assertEqual( dm('2014-11-18||/y').format('YYYY-MM-DDTHH:mm:ssZZ'), '2014-01-01T00:00:00-00:00') self.assertEqual( dm('2016-01-01 14:00:00||/w').format('YYYY-MM-DDTHH:mm:ssZZ'), '2015-12-28T00:00:00-00:00') self.assertEqual( dm('2014-11||/M').format('YYYY-MM-DDTHH:mm:ssZZ'), '2014-11-01T00:00:00-00:00') self.assertEqual( dm('2016-01-02||/M+1h+1m').format(iso8601), '2016-01-01T01:01:00-00:00') self.assertEqual( dm('2016-01-02||/d+1h').format(iso8601), '2016-01-02T01:00:00-00:00') self.assertEqual( dm('2016-01-02T14:02:00||/h').format(iso8601), '2016-01-02T14:00:00-00:00') self.assertEqual( dm('2016-01-02T14:02:00||/H').format(iso8601), '2016-01-02T14:00:00-00:00') # relitive formats # addition self.assertEqual( dm('+1s').format(iso8601), arrow.utcnow().replace(seconds=+1).format(iso8601)) self.assertEqual( dm('+1m').format(iso8601), arrow.utcnow().replace(minutes=+1).format(iso8601)) self.assertEqual( dm('+1h').format(iso8601), arrow.utcnow().replace(hours=+1).format(iso8601)) self.assertEqual( dm('+1d').format(iso8601), arrow.utcnow().replace(days=+1).format(iso8601)) self.assertEqual( dm('+1w').format(iso8601), arrow.utcnow().replace(weeks=+1).format(iso8601)) self.assertEqual( dm('+1M').format(iso8601), arrow.utcnow().replace(months=+1).format(iso8601)) self.assertEqual( dm('+1Y').format(iso8601), arrow.utcnow().replace(years=+1).format(iso8601)) self.assertEqual( dm('+1y').format(iso8601), arrow.utcnow().replace(years=+1).format(iso8601)) # subtraction self.assertEqual( dm('-1s').format(iso8601), arrow.utcnow().replace(seconds=-1).format(iso8601)) self.assertEqual( dm('-1m').format(iso8601), arrow.utcnow().replace(minutes=-1).format(iso8601)) self.assertEqual( dm('-1h').format(iso8601), arrow.utcnow().replace(hours=-1).format(iso8601)) self.assertEqual( dm('-1d').format(iso8601), arrow.utcnow().replace(days=-1).format(iso8601)) self.assertEqual( dm('-1w').format(iso8601), arrow.utcnow().replace(weeks=-1).format(iso8601)) self.assertEqual( dm('-1M').format(iso8601), arrow.utcnow().replace(months=-1).format(iso8601)) self.assertEqual( dm('-1Y').format(iso8601), arrow.utcnow().replace(years=-1).format(iso8601)) self.assertEqual( dm('-1y').format(iso8601), arrow.utcnow().replace(years=-1).format(iso8601)) # rounding self.assertEqual( dm('/s').format(iso8601), arrow.utcnow().floor('second').format(iso8601)) self.assertEqual( dm('/m').format(iso8601), arrow.utcnow().floor('minute').format(iso8601)) self.assertEqual( dm('/h').format(iso8601), arrow.utcnow().floor('hour').format(iso8601)) self.assertEqual( dm('/d').format(iso8601), arrow.utcnow().floor('day').format(iso8601)) self.assertEqual( dm('/w').format(iso8601), arrow.utcnow().floor('week').format(iso8601)) self.assertEqual( dm('/M').format(iso8601), arrow.utcnow().floor('month').format(iso8601)) self.assertEqual( dm('/Y').format(iso8601), arrow.utcnow().floor('year').format(iso8601)) self.assertEqual( dm('/y').format(iso8601), arrow.utcnow().floor('year').format(iso8601)) # complicated date math self.assertEqual( dm('now/d-1h').format(iso8601), arrow.utcnow().floor('day').replace(hours=-1).format(iso8601)) self.assertEqual( dm('+1h').format(iso8601), arrow.utcnow().replace(hours=+1).format(iso8601)) self.assertEqual( dm('/M+2d').format(iso8601), arrow.utcnow().floor('month').replace(days=+2).format(iso8601)) self.assertEqual( dm('now/w+2d-2h').format(iso8601), arrow.utcnow().floor('week').replace(days=+2, hours=-2).format(iso8601)) self.assertEqual( dm('now/M+1w-2h+10s').format(iso8601), arrow.utcnow().floor('month').replace(weeks=+1, hours=-2, seconds=+10).format(iso8601)) self.assertEqual( dm('now-1d/d').format(iso8601), arrow.utcnow().replace(days=-1).floor('day').format(iso8601)) self.assertEqual( dm('now+1d/d').format(iso8601), arrow.utcnow().replace(days=1).floor('day').format(iso8601)) self.assertEqual( dm('now-10d/d').format(iso8601), arrow.utcnow().replace(days=-10).floor('day').format(iso8601)) self.assertEqual( dm('now+10d/d').format(iso8601), arrow.utcnow().replace(days=10).floor('day').format(iso8601)) # future self.assertEqual( dm('+1s').format(iso8601), arrow.utcnow().replace(seconds=+1).format(iso8601)) self.assertEqual( dm('+1s+2m+3h').format(iso8601), arrow.utcnow().replace(seconds=+1, minutes=+2, hours=+3).format(iso8601)) self.assertEqual( dm('+1m').format(iso8601), arrow.utcnow().replace(minutes=+1).format(iso8601)) self.assertEqual( dm('+1m+5h').format(iso8601), arrow.utcnow().replace(minutes=+1, hours=+5).format(iso8601)) self.assertEqual( dm('/d+1m+5h').format(iso8601), arrow.utcnow().floor('day').replace(minutes=+1, hours=+5).format(iso8601)) self.assertEqual( dm('+1h').format(iso8601), arrow.utcnow().replace(hours=+1).format(iso8601)) self.assertEqual( dm('+1w').format(iso8601), arrow.utcnow().replace(weeks=+1).format(iso8601)) self.assertEqual( dm('+1w+12d').format(iso8601), arrow.utcnow().replace(weeks=+1, days=+12).format(iso8601)) self.assertEqual( dm('+2y').format(iso8601), arrow.utcnow().replace(years=+2).format(iso8601)) self.assertEqual( dm('+2y+22d+4h').format(iso8601), arrow.utcnow().replace(years=+2, days=+22, hours=+4).format(iso8601)) # past self.assertEqual( dm('-3w').format(iso8601), arrow.utcnow().replace(weeks=-3).format(iso8601)) self.assertEqual( dm('-3W').format(iso8601), arrow.utcnow().replace(weeks=-3).format(iso8601)) self.assertEqual( dm('-3w-2d-6h').format(iso8601), arrow.utcnow().replace(weeks=-3, days=-2, hours=-6).format(iso8601)) self.assertEqual( dm('-3w-2d-22h-36s').format(iso8601), arrow.utcnow().replace(weeks=-3, days=-2, hours=-22, seconds=-36).format(iso8601)) self.assertEqual( dm('-6y-3w-2d-22h-36s').format(iso8601), arrow.utcnow().replace(years=-6, weeks=-3, days=-2, hours=-22, seconds=-36).format(iso8601)) import datetime delta = datetime.timedelta(seconds=1) # datetime objects self.assertAlmostEqual(dm('now').datetime, arrow.utcnow().datetime, delta=delta) self.assertAlmostEqual(dm('now+1d').datetime, arrow.utcnow().replace(days=+1).datetime, delta=delta) self.assertAlmostEqual(dm('/w').datetime, arrow.utcnow().floor('week').datetime, delta=delta) # Floats self.assertEqual( dm('now-2.5h').format(iso8601), arrow.utcnow().replace(hours=-2.5).format(iso8601)) self.assertEqual( dm('now-2.5d').format(iso8601), arrow.utcnow().replace(days=-2.5).format(iso8601))
def dm_convert(timestr): from datemath import dm return dm(timestr).timestamp
import random import string import time import warnings from datemath import dm from retrying import retry import pytest from assemblyline.datastore import Collection with warnings.catch_warnings(): warnings.simplefilter("ignore") test_map = { 'test1': { 'expiry_dt': dm('now-2d/m').isoformat().replace('+00:00', '.001Z'), 'lvl_i': 400, 'test1_s': 'hello', 'tags_ss': ['a', 'b', 'c'] }, 'test2': { 'expiry_dt': dm('now-1d/m').isoformat().replace('+00:00', '.001Z'), 'lvl_i': 100, 'test2_s': 'hello', 'tags_ss': ['a', 'b', 'f'] }, 'test3': { 'expiry_dt': dm('now/m').isoformat().replace('+00:00', '.001Z'), 'lvl_i': 200, 'test3_s': 'hello', 'tags_ss': ['a', 'b', 'e']
def import_cti_event(self, timestamp: datetime, data: dict, is_update: bool = False) -> dict: logger.debug(f"Querying indicator: { data['x_opencti_id']}") entity = self.helper.api.indicator.read(id=data["x_opencti_id"]) logger.debug(entity) _result: dict = {} _document: Cut = {} if data["type"] != "indicator": logger.error( f"Data type unsupported: {data['type']}. Only 'indicators are currently supported." ) return None if is_update is True: update_time: str = (datetime.now( tz=timezone.utc).isoformat().replace("+00:00", "Z")) try: # Attempt to retreive existing document logger.debug(f"Retrieving document id: {data['x_opencti_id']}") _result = self.es_client.get(index=self.idx_pattern, id=data["x_opencti_id"], doc_type="_doc") except NotFoundError: logger.warn( f"Document not found to update at /{self.idx}/_doc/{data['x_opencti_id']}" ) logger.warn("Skipping") return {} except RequestError as err: logger.error( f"Unexpected error retreiving document at /{self.idx}/_doc/{data['x_opencti_id']}:", err.__dict__, ) if _result["found"] is True: _document = Cut(_result["_source"]) _write_idx = _result["_index"] if data.get("x_data_update", None): if data["x_data_update"].get("replace", None): if entity["pattern_type"] == "stix": # Pull in any indicator updates _indicator: dict = self._create_ecs_indicator_stix( entity) if _indicator == {}: return {} _document["threatintel.indicator"] = _indicator if entity.get("killChainPhases", None): phases = [] for phase in sorted( entity["killChainPhases"], key=lambda i: ( i["kill_chain_name"], i["x_opencti_order"], ), ): phases.append({ "killchain_name": phase["kill_chain_name"], "phase_name": phase["phase_name"], "opencti_phase_order": phase["x_opencti_order"], }) _document.setdefault( "threatintel.opencti.killchain_phases", phases) else: logger.warning( f"Unsupported indicator pattern type: {entity['pattern_type']}. Skipping." ) return _document for k, v in data["x_data_update"].get("replace", {}).items(): logger.debug( f"Updating field {k} -> {entity_field_mapping.get(k)} to {v}" ) try: _field = entity_field_mapping.get(k) _document.setdefault(_field, v) _document[_field] = v except KeyError as err: logger.error( f"Unable to find field mapping for {k}", err) _document["threatintel.opencti.updated_at"] = update_time # This scrubs the Cut object and returns a dict _document = remove_nones(_document) try: # Don't render timestamped index since this is an update # Submit to Elastic index logger.debug( f"Updating doc to {_write_idx}:\n {_document}") self.es_client.index( index=_write_idx, id=data["x_opencti_id"], body=_document, ) except RequestError as err: logger.error("Unexpected error:", err, _document) except Exception as err: logger.error("Something else happened", err, _document) return _document creation_time: str = (datetime.now( tz=timezone.utc).isoformat().replace("+00:00", "Z")) _document: dict = { "@timestamp": timestamp, "event": { "created": creation_time, "kind": "enrichment", "category": "threat", "type": "indicator", "dataset": "threatintel.opencti", }, "threatintel": {}, } if len(entity.get("externalReferences", [])) > 0: _document["event"]["reference"] = [ item.get("url", None) for item in entity["externalReferences"] ] if self.config.get("opencti.platform_url") is not None: _document["event"]["url"] = urllib.parse.urljoin( f"{self.config.get('opencti.platform_url')}", f"/dashboard/observations/indicators/{entity['id']}", ) _document["event"]["risk_score"] = entity.get("x_opencti_score", None) _document["event"]["risk_score_norm"] = entity.get( "x_opencti_score", None) _document["threatintel"]["confidence"] = entity.get("confidence", None) _document["threatintel"]["confidence_norm"] = entity.get( "confidence", None) _document["threatintel"]["opencti"] = { "internal_id": entity.get("id", None), "valid_from": entity.get("valid_from", None), "valid_until": entity.get("valid_until", None), "enable_detection": entity.get("x_opencti_detection", None), "original_pattern": entity.get("pattern", None), "pattern_type": entity.get("pattern_type", None), "created_at": entity.get("created_at", None), "updated_at": entity.get("created_at", None), "revoked": entity.get("revoked", None), } if entity.get("killChainPhases", None): phases = [] for phase in sorted( entity["killChainPhases"], key=lambda i: (i["kill_chain_name"], i["x_opencti_order"]), ): phases.append({ "killchain_name": phase["kill_chain_name"], "phase_name": phase["phase_name"], "opencti_phase_order": phase["x_opencti_order"], }) _document["threatintel"]["opencti"]["killchain_phases"] = phases if entity.get("x_mitre_platforms", None): _document["threatintel"]["opencti"]["mitre"] = { "platforms": entity.get("x_mitre_platforms", None) } if entity["pattern_type"] == "stix": _indicator: dict = self._create_ecs_indicator_stix(entity) if _indicator == {}: return {} _document["threatintel"]["stix"] = { "id": entity.get("standard_id") } _document["threatintel"]["indicator"] = _indicator else: logger.warning( f"Unsupported indicator pattern type: {entity['pattern_type']}. Skipping." ) return {} _document = remove_nones(_document) try: # Render date-specific index, if we're doing logstash style indices _write_idx = self.write_idx m = self.pattern.search(_write_idx) if m is not None: m = m.groupdict() if m.get("modulo", None) is not None: _fmt = m.get("format") or DM_DEFAULT_FMT logger.debug(f"{m['modulo']} -> {_fmt}") _val = dm(m.get("modulo"), now=timestamp).format(_fmt) _write_idx = self.pattern.sub(_val, _write_idx) # Submit to Elastic index logger.debug(f"Indexing doc to {_write_idx}:\n {_document}") self.es_client.index( index=_write_idx, id=data["x_opencti_id"], body=_document, ) except RequestError as err: logger.error("Unexpected error:", err, _document) except Exception as err: logger.error("Something else happened", err, _document) return _document
def testParse(self): # Baisc dates self.assertEqual(dm('2016.01.02').format(iso8601), '2016-01-02T00:00:00-00:00') self.assertEqual(dm('2016-01-02').format(iso8601), '2016-01-02T00:00:00-00:00') self.assertEqual(dm('2016-01-02 01:00:00').format(iso8601), '2016-01-02T01:00:00-00:00') # Rounding Tests self.assertEqual(dm('2016-01-01||/d').format('YYYY-MM-DDTHH:mm:ssZZ'), '2016-01-01T00:00:00-00:00') self.assertEqual(dm('2014-11-18||/y').format('YYYY-MM-DDTHH:mm:ssZZ'), '2014-01-01T00:00:00-00:00') self.assertEqual(dm('2016-01-01 14:00:00||/w').format('YYYY-MM-DDTHH:mm:ssZZ'), '2015-12-28T00:00:00-00:00') self.assertEqual(dm('2014-11||/M').format('YYYY-MM-DDTHH:mm:ssZZ'), '2014-11-01T00:00:00-00:00') self.assertEqual(dm('2016-01-02||/M+1h+1m').format(iso8601), '2016-01-01T01:01:00-00:00') self.assertEqual(dm('2016-01-02||/d+1h').format(iso8601), '2016-01-02T01:00:00-00:00') self.assertEqual(dm('2016-01-02T14:02:00||/h').format(iso8601), '2016-01-02T14:00:00-00:00') self.assertEqual(dm('2016-01-02T14:02:00||/H').format(iso8601), '2016-01-02T14:00:00-00:00') # relitive formats # addition self.assertEqual(dm('+1s').format(iso8601), arrow.utcnow().replace(seconds=+1).format(iso8601)) self.assertEqual(dm('+1m').format(iso8601), arrow.utcnow().replace(minutes=+1).format(iso8601)) self.assertEqual(dm('+1h').format(iso8601), arrow.utcnow().replace(hours=+1).format(iso8601)) self.assertEqual(dm('+1d').format(iso8601), arrow.utcnow().replace(days=+1).format(iso8601)) self.assertEqual(dm('+1w').format(iso8601), arrow.utcnow().replace(weeks=+1).format(iso8601)) self.assertEqual(dm('+1M').format(iso8601), arrow.utcnow().replace(months=+1).format(iso8601)) self.assertEqual(dm('+1Y').format(iso8601), arrow.utcnow().replace(years=+1).format(iso8601)) self.assertEqual(dm('+1y').format(iso8601), arrow.utcnow().replace(years=+1).format(iso8601)) # subtraction self.assertEqual(dm('-1s').format(iso8601), arrow.utcnow().replace(seconds=-1).format(iso8601)) self.assertEqual(dm('-1m').format(iso8601), arrow.utcnow().replace(minutes=-1).format(iso8601)) self.assertEqual(dm('-1h').format(iso8601), arrow.utcnow().replace(hours=-1).format(iso8601)) self.assertEqual(dm('-1d').format(iso8601), arrow.utcnow().replace(days=-1).format(iso8601)) self.assertEqual(dm('-1w').format(iso8601), arrow.utcnow().replace(weeks=-1).format(iso8601)) self.assertEqual(dm('-1M').format(iso8601), arrow.utcnow().replace(months=-1).format(iso8601)) self.assertEqual(dm('-1Y').format(iso8601), arrow.utcnow().replace(years=-1).format(iso8601)) self.assertEqual(dm('-1y').format(iso8601), arrow.utcnow().replace(years=-1).format(iso8601)) # rounding self.assertEqual(dm('/s').format(iso8601), arrow.utcnow().floor('second').format(iso8601)) self.assertEqual(dm('/m').format(iso8601), arrow.utcnow().floor('minute').format(iso8601)) self.assertEqual(dm('/h').format(iso8601), arrow.utcnow().floor('hour').format(iso8601)) self.assertEqual(dm('/d').format(iso8601), arrow.utcnow().floor('day').format(iso8601)) self.assertEqual(dm('/w').format(iso8601), arrow.utcnow().floor('week').format(iso8601)) self.assertEqual(dm('/M').format(iso8601), arrow.utcnow().floor('month').format(iso8601)) self.assertEqual(dm('/Y').format(iso8601), arrow.utcnow().floor('year').format(iso8601)) self.assertEqual(dm('/y').format(iso8601), arrow.utcnow().floor('year').format(iso8601)) # complicated date math self.assertEqual(dm('now/d-1h').format(iso8601), arrow.utcnow().floor('day').replace(hours=-1).format(iso8601)) self.assertEqual(dm('+1h').format(iso8601), arrow.utcnow().replace(hours=+1).format(iso8601)) self.assertEqual(dm('/M+2d').format(iso8601), arrow.utcnow().floor('month').replace(days=+2).format(iso8601)) self.assertEqual(dm('now/w+2d-2h').format(iso8601), arrow.utcnow().floor('week').replace(days=+2, hours=-2).format(iso8601)) self.assertEqual(dm('now/M+1w-2h+10s').format(iso8601), arrow.utcnow().floor('month').replace(weeks=+1, hours=-2, seconds=+10).format(iso8601)) # future self.assertEqual(dm('+1s').format(iso8601), arrow.utcnow().replace(seconds=+1).format(iso8601)) self.assertEqual(dm('+1s+2m+3h').format(iso8601), arrow.utcnow().replace(seconds=+1, minutes=+2, hours=+3).format(iso8601)) self.assertEqual(dm('+1m').format(iso8601), arrow.utcnow().replace(minutes=+1).format(iso8601)) self.assertEqual(dm('+1m+5h').format(iso8601), arrow.utcnow().replace(minutes=+1, hours=+5).format(iso8601)) self.assertEqual(dm('/d+1m+5h').format(iso8601), arrow.utcnow().floor('day').replace(minutes=+1, hours=+5).format(iso8601)) self.assertEqual(dm('+1h').format(iso8601), arrow.utcnow().replace(hours=+1).format(iso8601)) self.assertEqual(dm('+1w').format(iso8601), arrow.utcnow().replace(weeks=+1).format(iso8601)) self.assertEqual(dm('+1w+12d').format(iso8601), arrow.utcnow().replace(weeks=+1, days=+12).format(iso8601)) self.assertEqual(dm('+2y').format(iso8601), arrow.utcnow().replace(years=+2).format(iso8601)) self.assertEqual(dm('+2y+22d+4h').format(iso8601), arrow.utcnow().replace(years=+2, days=+22, hours=+4).format(iso8601)) # past self.assertEqual(dm('-3w').format(iso8601), arrow.utcnow().replace(weeks=-3).format(iso8601)) self.assertEqual(dm('-3W').format(iso8601), arrow.utcnow().replace(weeks=-3).format(iso8601)) self.assertEqual(dm('-3w-2d-6h').format(iso8601), arrow.utcnow().replace(weeks=-3, days=-2, hours=-6).format(iso8601)) self.assertEqual(dm('-3w-2d-22h-36s').format(iso8601), arrow.utcnow().replace(weeks=-3, days=-2, hours=-22, seconds=-36).format(iso8601)) self.assertEqual(dm('-6y-3w-2d-22h-36s').format(iso8601), arrow.utcnow().replace(years=-6, weeks=-3, days=-2, hours=-22, seconds=-36).format(iso8601)) import datetime delta = datetime.timedelta(seconds=1) # datetime objects self.assertAlmostEqual(dm('now').datetime, arrow.utcnow().datetime, delta=delta) self.assertAlmostEqual(dm('now+1d').datetime, arrow.utcnow().replace(days=+1).datetime, delta=delta) self.assertAlmostEqual(dm('/w').datetime, arrow.utcnow().floor('week').datetime, delta=delta) # Floats self.assertEqual(dm('now-2.5h').format(iso8601), arrow.utcnow().replace(hours=-2.5).format(iso8601)) self.assertEqual(dm('now-2.5d').format(iso8601), arrow.utcnow().replace(days=-2.5).format(iso8601))
def run_expiry_once(self): now = now_as_iso() reached_max = False # Expire data for collection in self.expirable_collections: # Call heartbeat pre-dated by 5 minutes. If a collection takes more than # 5 minutes to expire, this container could be seen as unhealthy. The down # side is if it is stuck on something it will be more than 5 minutes before # the container is restarted. self.heartbeat(int(time.time() + 5 * 60)) # Start of expiry transaction if self.apm_client: self.apm_client.begin_transaction("Delete expired documents") if self.config.core.expiry.batch_delete: computed_date = epoch_to_iso( dm(f"{now}||-{self.config.core.expiry.delay}h/d"). float_timestamp) else: computed_date = epoch_to_iso( dm(f"{now}||-{self.config.core.expiry.delay}h"). float_timestamp) delete_query = f"expiry_ts:[* TO {computed_date}]" if self.config.core.expiry.delete_storage and collection.name in self.fs_hashmap: file_delete = True sort = ["expiry_ts asc", "id asc"] else: file_delete = False sort = None number_to_delete = collection.search( delete_query, rows=0, as_obj=False, use_archive=True, sort=sort, track_total_hits=EXPIRY_SIZE)['total'] if self.apm_client: elasticapm.label(query=delete_query) elasticapm.label(number_to_delete=number_to_delete) self.log.info(f"Processing collection: {collection.name}") if number_to_delete != 0: if file_delete: with elasticapm.capture_span( name='FILESTORE [ThreadPoolExecutor] :: delete()', labels={ "num_files": number_to_delete, "query": delete_query }): # Delete associated files with concurrent.futures.ThreadPoolExecutor( self.config.core.expiry.workers, thread_name_prefix="file_delete") as executor: for item in collection.search( delete_query, fl='id', rows=number_to_delete, sort=sort, use_archive=True, as_obj=False)['items']: executor.submit( self.fs_hashmap[collection.name], item['id'], computed_date) self.log.info( f' Deleted associated files from the ' f'{"cachestore" if "cache" in collection.name else "filestore"}...' ) # Proceed with deletion collection.delete_by_query( delete_query, workers=self.config.core.expiry.workers, sort=sort, max_docs=number_to_delete) else: # Proceed with deletion collection.delete_by_query( delete_query, workers=self.config.core.expiry.workers) if number_to_delete == EXPIRY_SIZE: reached_max = True self.counter.increment(f'{collection.name}', increment_by=number_to_delete) self.log.info( f" Deleted {number_to_delete} items from the datastore..." ) else: self.log.debug(" Nothing to delete in this collection.") # End of expiry transaction if self.apm_client: self.apm_client.end_transaction(collection.name, 'deleted') return reached_max
def testParse(self): # Baisc dates self.assertEqual( dm('2016.01.02').format(iso8601), '2016-01-02T00:00:00+00:00') self.assertEqual( dm('2016-01-02').format(iso8601), '2016-01-02T00:00:00+00:00') self.assertEqual( dm('2016-01-02 01:00:00').format(iso8601), '2016-01-02T01:00:00+00:00') # Rounding Tests self.assertEqual( dm('2016-01-01||/d').format('YYYY-MM-DDTHH:mm:ssZZ'), '2016-01-01T00:00:00+00:00') self.assertEqual( dm('2014-11-18||/y').format('YYYY-MM-DDTHH:mm:ssZZ'), '2014-01-01T00:00:00+00:00') self.assertEqual( dm('2016-01-01 14:00:00||/w').format('YYYY-MM-DDTHH:mm:ssZZ'), '2015-12-28T00:00:00+00:00') self.assertEqual( dm('2014-11||/M').format('YYYY-MM-DDTHH:mm:ssZZ'), '2014-11-01T00:00:00+00:00') self.assertEqual( dm('2016-01-02||/M+1h+1m').format(iso8601), '2016-01-01T01:01:00+00:00') self.assertEqual( dm('2016-01-02||/d+1h').format(iso8601), '2016-01-02T01:00:00+00:00') self.assertEqual( dm('2016-01-02T14:02:00||/h').format(iso8601), '2016-01-02T14:00:00+00:00') self.assertEqual( dm('2016-01-02T14:02:00||/H').format(iso8601), '2016-01-02T14:00:00+00:00') # Rounding Up Tests self.assertEqual( dm('2016-01-01||/d', roundDown=False).format('YYYY-MM-DDTHH:mm:ssZZ'), '2016-01-01T23:59:59+00:00') self.assertEqual( dm('2014-11-18||/y', roundDown=False).format('YYYY-MM-DDTHH:mm:ssZZ'), '2014-12-31T23:59:59+00:00') # Timezone Tests self.assertEqual( dm('now', tz='US/Pacific').format(iso8601), arrow.utcnow().to('US/Pacific').format(iso8601)) self.assertEqual( dm('2017-09-22 10:20:00', tz='US/Pacific').datetime, pydatetime(2017, 9, 22, 10, 20, 00, tzinfo=tz.gettz('US/Pacific'))) self.assertEqual(dm('2016-01-01', tz='UTC'), arrow.get('2016-01-01').to('UTC')) self.assertEqual(dm('2016-01-01', tz='US/Eastern'), pydatetime(2016, 1, 1, tzinfo=tz.gettz('US/Eastern'))) self.assertEqual( datemath('2016-01-01T01:00:00', tz='US/Central'), pydatetime(2016, 1, 1, 1, 0, 0, tzinfo=tz.gettz('US/Central'))) self.assertEqual( datemath('2016-01-01T02:00:00', tz='US/Eastern'), pydatetime(2016, 1, 1, 2, tzinfo=tz.gettz('US/Eastern'))) # relitive formats # addition self.assertEqual( dm('+1s').format(iso8601), arrow.utcnow().shift(seconds=+1).format(iso8601)) self.assertEqual( dm('+1m').format(iso8601), arrow.utcnow().shift(minutes=+1).format(iso8601)) self.assertEqual( dm('+1h').format(iso8601), arrow.utcnow().shift(hours=+1).format(iso8601)) self.assertEqual( dm('+1d').format(iso8601), arrow.utcnow().shift(days=+1).format(iso8601)) self.assertEqual( dm('+1w').format(iso8601), arrow.utcnow().shift(weeks=+1).format(iso8601)) self.assertEqual( dm('+1M').format(iso8601), arrow.utcnow().shift(months=+1).format(iso8601)) self.assertEqual( dm('+1Y').format(iso8601), arrow.utcnow().shift(years=+1).format(iso8601)) self.assertEqual( dm('+1y').format(iso8601), arrow.utcnow().shift(years=+1).format(iso8601)) # subtraction self.assertEqual( dm('-1s').format(iso8601), arrow.utcnow().shift(seconds=-1).format(iso8601)) self.assertEqual( dm('-1m').format(iso8601), arrow.utcnow().shift(minutes=-1).format(iso8601)) self.assertEqual( dm('-1h').format(iso8601), arrow.utcnow().shift(hours=-1).format(iso8601)) self.assertEqual( dm('-1d').format(iso8601), arrow.utcnow().shift(days=-1).format(iso8601)) self.assertEqual( dm('-1w').format(iso8601), arrow.utcnow().shift(weeks=-1).format(iso8601)) self.assertEqual( dm('-1M').format(iso8601), arrow.utcnow().shift(months=-1).format(iso8601)) self.assertEqual( dm('-1Y').format(iso8601), arrow.utcnow().shift(years=-1).format(iso8601)) self.assertEqual( dm('-1y').format(iso8601), arrow.utcnow().shift(years=-1).format(iso8601)) # rounding self.assertEqual( dm('/s').format(iso8601), arrow.utcnow().floor('second').format(iso8601)) self.assertEqual( dm('/m').format(iso8601), arrow.utcnow().floor('minute').format(iso8601)) self.assertEqual( dm('/h').format(iso8601), arrow.utcnow().floor('hour').format(iso8601)) self.assertEqual( dm('/d').format(iso8601), arrow.utcnow().floor('day').format(iso8601)) self.assertEqual( dm('/w').format(iso8601), arrow.utcnow().floor('week').format(iso8601)) self.assertEqual( dm('/M').format(iso8601), arrow.utcnow().floor('month').format(iso8601)) self.assertEqual( dm('/Y').format(iso8601), arrow.utcnow().floor('year').format(iso8601)) self.assertEqual( dm('/y').format(iso8601), arrow.utcnow().floor('year').format(iso8601)) # rounding up self.assertEqual( dm('/s', roundDown=False).format(iso8601), arrow.utcnow().ceil('second').format(iso8601)) self.assertEqual( dm('/m', roundDown=False).format(iso8601), arrow.utcnow().ceil('minute').format(iso8601)) self.assertEqual( dm('/h', roundDown=False).format(iso8601), arrow.utcnow().ceil('hour').format(iso8601)) self.assertEqual( dm('/d', roundDown=False).format(iso8601), arrow.utcnow().ceil('day').format(iso8601)) self.assertEqual( dm('/w', roundDown=False).format(iso8601), arrow.utcnow().ceil('week').format(iso8601)) self.assertEqual( dm('/M', roundDown=False).format(iso8601), arrow.utcnow().ceil('month').format(iso8601)) self.assertEqual( dm('/Y', roundDown=False).format(iso8601), arrow.utcnow().ceil('year').format(iso8601)) self.assertEqual( dm('/y', roundDown=False).format(iso8601), arrow.utcnow().ceil('year').format(iso8601)) # roundDown option tests self.assertEqual( dm('2016-01-01T14:00:00||/d').format(iso8601), '2016-01-01T00:00:00+00:00') self.assertEqual( dm('2016-01-01T14:00:00||/d', roundDown=False).format(iso8601), '2016-01-01T23:59:59+00:00') # complicated date math self.assertEqual( dm('now/d-1h').format(iso8601), arrow.utcnow().floor('day').shift(hours=-1).format(iso8601)) self.assertEqual( dm('+1h').format(iso8601), arrow.utcnow().shift(hours=+1).format(iso8601)) self.assertEqual( dm('/M+2d').format(iso8601), arrow.utcnow().floor('month').shift(days=+2).format(iso8601)) self.assertEqual( dm('now/w+2d-2h').format(iso8601), arrow.utcnow().floor('week').shift(days=+2, hours=-2).format(iso8601)) self.assertEqual( dm('now/M+1w-2h+10s').format(iso8601), arrow.utcnow().floor('month').shift(weeks=+1, hours=-2, seconds=+10).format(iso8601)) self.assertEqual( dm('now-1d/d').format(iso8601), arrow.utcnow().shift(days=-1).floor('day').format(iso8601)) self.assertEqual( dm('now+1d/d').format(iso8601), arrow.utcnow().shift(days=1).floor('day').format(iso8601)) self.assertEqual( dm('now-10d/d').format(iso8601), arrow.utcnow().shift(days=-10).floor('day').format(iso8601)) self.assertEqual( dm('now+10d/d').format(iso8601), arrow.utcnow().shift(days=10).floor('day').format(iso8601)) self.assertEqual( dm('now-29d/d').format(iso8601), arrow.utcnow().shift(days=-29).floor('day').format(iso8601)) # future self.assertEqual( dm('+1s').format(iso8601), arrow.utcnow().shift(seconds=+1).format(iso8601)) self.assertEqual( dm('+1s+2m+3h').format(iso8601), arrow.utcnow().shift(seconds=+1, minutes=+2, hours=+3).format(iso8601)) self.assertEqual( dm('+1m').format(iso8601), arrow.utcnow().shift(minutes=+1).format(iso8601)) self.assertEqual( dm('+1m+5h').format(iso8601), arrow.utcnow().shift(minutes=+1, hours=+5).format(iso8601)) self.assertEqual( dm('/d+1m+5h').format(iso8601), arrow.utcnow().floor('day').shift(minutes=+1, hours=+5).format(iso8601)) self.assertEqual( dm('+1h').format(iso8601), arrow.utcnow().shift(hours=+1).format(iso8601)) self.assertEqual( dm('+1w').format(iso8601), arrow.utcnow().shift(weeks=+1).format(iso8601)) self.assertEqual( dm('+1w+12d').format(iso8601), arrow.utcnow().shift(weeks=+1, days=+12).format(iso8601)) self.assertEqual( dm('+2y').format(iso8601), arrow.utcnow().shift(years=+2).format(iso8601)) self.assertEqual( dm('+2y+22d+4h').format(iso8601), arrow.utcnow().shift(years=+2, days=+22, hours=+4).format(iso8601)) # past self.assertEqual( dm('-3w').format(iso8601), arrow.utcnow().shift(weeks=-3).format(iso8601)) self.assertEqual( dm('-3W').format(iso8601), arrow.utcnow().shift(weeks=-3).format(iso8601)) self.assertEqual( dm('-3w-2d-6h').format(iso8601), arrow.utcnow().shift(weeks=-3, days=-2, hours=-6).format(iso8601)) self.assertEqual( dm('-3w-2d-22h-36s').format(iso8601), arrow.utcnow().shift(weeks=-3, days=-2, hours=-22, seconds=-36).format(iso8601)) self.assertEqual( dm('-6y-3w-2d-22h-36s').format(iso8601), arrow.utcnow().shift(years=-6, weeks=-3, days=-2, hours=-22, seconds=-36).format(iso8601)) import datetime delta = datetime.timedelta(seconds=1) # datetime objects self.assertAlmostEqual(dm('now').datetime, arrow.utcnow().datetime, delta=delta) self.assertAlmostEqual(dm('now+1d').datetime, arrow.utcnow().shift(days=+1).datetime, delta=delta) self.assertAlmostEqual(dm('/w').datetime, arrow.utcnow().floor('week').datetime, delta=delta) # Floats self.assertEqual( dm('now-2.5h').format(iso8601), arrow.utcnow().shift(hours=-2.5).format(iso8601)) self.assertEqual( dm('now-2.5d').format(iso8601), arrow.utcnow().shift(days=-2.5).format(iso8601)) # Test epoch timestamps self.assertEqual( dm('1451610061').format(iso8601), '2016-01-01T01:01:01+00:00') self.assertEqual( dm(1367900664).format(iso8601), '2013-05-07T04:24:24+00:00') self.assertRaises(DateMathException, dm, '1451610061000') try: dm(1451610061000) except DateMathException as e: self.assertTrue( 'Unable to parse epoch timestamps in millis' in str(e)) # Catch invalid timeunits self.assertRaises(DateMathException, dm, '+1,') self.assertRaises(DateMathException, dm, '+1.') self.assertRaises(DateMathException, dm, '+1ö') self.assertRaises(DateMathException, dm, '+1ä') self.assertRaises(DateMathException, dm, '+1ü') self.assertRaises(DateMathException, dm, '+1ß') try: dm('+1,') except DateMathException as e: self.assertTrue('is not a valid timeunit' in str(e))
def dm_convert(timestr): from datemath import dm return dm(timestr).timestamp