def __init__(self,
params,
agent=Agent,
client_ip=ClientIp,
referrer=Referrer,
request=Request):
self._agent = agent()
self._clientip = client_ip()
self._referrer = referrer()
self._request = request()
# We will reuse the event dictionary. This assumes that each field will be present (and thus overwritten) in each event.
# This reduces object churn and improves peak indexing throughput.
self._event = {}
if "index" in params:
index = re.sub(r"<\s*yyyy\s*>",
"{ts[yyyy]}",
params["index"],
flags=re.IGNORECASE)
index = re.sub(r"<\s*yy\s*>",
"{ts[yy]}",
index,
flags=re.IGNORECASE)
index = re.sub(r"<\s*mm\s*>",
"{ts[mm]}",
index,
flags=re.IGNORECASE)
index = re.sub(r"<\s*dd\s*>",
"{ts[dd]}",
index,
flags=re.IGNORECASE)
index = re.sub(r"<\s*hh\s*>",
"{ts[hh]}",
index,
flags=re.IGNORECASE)
self._index = index
self._index_pattern = True
else:
self._index = "elasticlogs"
self._index_pattern = False
self._type = "doc"
self._timestamp_generator = TimestampStructGenerator(
params.get("starting_point", "now"),
params.get("offset"),
float(params.get("acceleration_factor", "1.0")),
# this is only expected to be used in tests
params.get("__utc_now"))
if "daily_logging_volume" in params and "client_count" in params:
# in bytes
self.daily_logging_volume = convert_to_bytes(
params["daily_logging_volume"]) // int(params["client_count"])
else:
self.daily_logging_volume = None
self.current_logging_volume = 0
self.total_days = params.get("number_of_days")
self.remaining_days = self.total_days
self.record_raw_event_size = params.get("record_raw_event_size", False)
def test_generate_interval_and_skip():
clock = ReproducibleClock(start=datetime.datetime(year=2019,
month=1,
day=5,
hour=15),
delta=datetime.timedelta(seconds=1))
g = TimestampStructGenerator(starting_point="2018-05-01:00:59:56",
acceleration_factor=3.0,
utcnow=clock)
assert g.next_timestamp() == {
"iso": "2018-05-01T00:59:59.000Z",
"iso_prefix": "2018-05-01T00:59:59",
"yyyy": "2018",
"yy": "18",
"mm": "05",
"dd": "01",
"hh": "00"
}
assert g.next_timestamp() == {
"iso": "2018-05-01T01:00:02.000Z",
"iso_prefix": "2018-05-01T01:00:02",
"yyyy": "2018",
"yy": "18",
"mm": "05",
"dd": "01",
"hh": "01"
}
g.skip(datetime.timedelta(days=1))
assert g.next_timestamp() == {
"iso": "2018-05-02T00:59:59.000Z",
"iso_prefix": "2018-05-02T00:59:59",
"yyyy": "2018",
"yy": "18",
"mm": "05",
"dd": "02",
"hh": "00"
}
assert g.next_timestamp() == {
"iso": "2018-05-02T01:00:02.000Z",
"iso_prefix": "2018-05-02T01:00:02",
"yyyy": "2018",
"yy": "18",
"mm": "05",
"dd": "02",
"hh": "01"
}
def __init__(self, params):
self._agent = Agent()
self._clientip = ClientIp()
self._referrer = Referrer()
self._request = Request()
# We will reuse the event dictionary. This assumes that each field will be present (and thus overwritten) in each event.
# This reduces object churn and improves peak indexing throughput.
self._event = {}
self._index = 'elasticlogs'
self._index_pattern = False
if 'index' in params.keys():
index = re.sub(r'<\s*yyyy\s*>', '{ts[yyyy]}', params['index'], flags=re.IGNORECASE)
index = re.sub(r'<\s*yy\s*>', '{ts[yy]}', index, flags=re.IGNORECASE)
index = re.sub(r'<\s*mm\s*>', '{ts[mm]}', index, flags=re.IGNORECASE)
index = re.sub(r'<\s*dd\s*>', '{ts[dd]}', index, flags=re.IGNORECASE)
index = re.sub(r'<\s*hh\s*>', '{ts[hh]}', index, flags=re.IGNORECASE)
self._index = index
self._index_pattern = True
self._type = 'doc'
if 'starting_point' in params.keys():
sp = params['starting_point']
else:
sp = "now"
if 'end_point' in params.keys():
ep = params['end_point']
self._timestamp_generator = TimestampStructGenerator.Interval(sp, ep)
else:
if 'acceleration_factor' in params.keys():
af = float(params['acceleration_factor'])
self._timestamp_generator = TimestampStructGenerator.StartingPoint(sp, af)
else:
self._timestamp_generator = TimestampStructGenerator.StartingPoint(sp)
self._delete_fields = []
if 'delete_fields' in params.keys():
for d in params['delete_fields']:
self._delete_fields.append(d.split('.'))
def test_generate_interval_from_now():
clock = ReproducibleClock(start=datetime.datetime(year=2019,
month=1,
day=5,
hour=15),
delta=datetime.timedelta(seconds=5))
g = TimestampStructGenerator(starting_point="now", utcnow=clock)
# first generated timestamp will be one (clock) invocation after the original start
assert g.next_timestamp() == {
"iso": "2019-01-05T15:00:05.000Z",
"iso_prefix": "2019-01-05T15:00:05",
"yyyy": "2019",
"yy": "19",
"mm": "01",
"dd": "05",
"hh": "15"
}
assert g.next_timestamp() == {
"iso": "2019-01-05T15:00:10.000Z",
"iso_prefix": "2019-01-05T15:00:10",
"yyyy": "2019",
"yy": "19",
"mm": "01",
"dd": "05",
"hh": "15"
}
assert g.next_timestamp() == {
"iso": "2019-01-05T15:00:15.000Z",
"iso_prefix": "2019-01-05T15:00:15",
"yyyy": "2019",
"yy": "19",
"mm": "01",
"dd": "05",
"hh": "15"
}
def test_generate_interval_from_fixed_starting_point_with_offset():
clock = ReproducibleClock(start=datetime.datetime(year=2019,
month=1,
day=5,
hour=15),
delta=datetime.timedelta(seconds=1))
g = TimestampStructGenerator(starting_point="2018-05-01:00:59:56",
offset="+10d",
acceleration_factor=3.0,
utcnow=clock)
assert g.next_timestamp() == {
"iso": "2018-05-11T00:59:59.000Z",
"yyyy": "2018",
"yy": "18",
"mm": "05",
"dd": "11",
"hh": "00"
}
assert g.next_timestamp() == {
"iso": "2018-05-11T01:00:02.000Z",
"yyyy": "2018",
"yy": "18",
"mm": "05",
"dd": "11",
"hh": "01"
}
assert g.next_timestamp() == {
"iso": "2018-05-11T01:00:05.000Z",
"yyyy": "2018",
"yy": "18",
"mm": "05",
"dd": "11",
"hh": "01"
}
def test_simulate_ticks():
clock = ReproducibleClock(start=datetime.datetime(year=2019,
month=1,
day=5,
hour=15),
delta=datetime.timedelta(seconds=1))
g = TimestampStructGenerator(starting_point="2018-05-01:00:59:56",
acceleration_factor=3.0,
utcnow=clock)
assert g.next_timestamp() == {
"iso": "2018-05-01T00:59:59.000Z",
"iso_prefix": "2018-05-01T00:59:59",
"yyyy": "2018",
"yy": "18",
"mm": "05",
"dd": "01",
"hh": "00"
}
assert g.simulate_tick(micros=1.0) == {
"iso": "2018-05-01T00:59:59.001Z",
"iso_prefix": "2018-05-01T00:59:59",
"yyyy": "2018",
"yy": "18",
"mm": "05",
"dd": "01",
"hh": "00"
}
assert g.simulate_tick(micros=0.1) == {
"iso": "2018-05-01T00:59:59.001Z",
"iso_prefix": "2018-05-01T00:59:59",
"yyyy": "2018",
"yy": "18",
"mm": "05",
"dd": "01",
"hh": "00"
}
assert g.simulate_tick(micros=10.0) == {
"iso": "2018-05-01T00:59:59.011Z",
"iso_prefix": "2018-05-01T00:59:59",
"yyyy": "2018",
"yy": "18",
"mm": "05",
"dd": "01",
"hh": "00"
}
def test_generate_invalid_time_interval():
# "w" is unsupported
with pytest.raises(TimeParsingError) as ex:
TimestampStructGenerator(starting_point="now+1w")
assert "Invalid time format: now+1w" == str(ex.value)
class RandomEvent:
def __init__(self,
params,
agent=Agent,
client_ip=ClientIp,
referrer=Referrer,
request=Request):
self._agent = agent()
self._clientip = client_ip()
self._referrer = referrer()
self._request = request()
# We will reuse the event dictionary. This assumes that each field will be present (and thus overwritten) in each event.
# This reduces object churn and improves peak indexing throughput.
self._event = {}
if "index" in params:
index = re.sub(r"<\s*yyyy\s*>",
"{ts[yyyy]}",
params["index"],
flags=re.IGNORECASE)
index = re.sub(r"<\s*yy\s*>",
"{ts[yy]}",
index,
flags=re.IGNORECASE)
index = re.sub(r"<\s*mm\s*>",
"{ts[mm]}",
index,
flags=re.IGNORECASE)
index = re.sub(r"<\s*dd\s*>",
"{ts[dd]}",
index,
flags=re.IGNORECASE)
index = re.sub(r"<\s*hh\s*>",
"{ts[hh]}",
index,
flags=re.IGNORECASE)
self._index = index
self._index_pattern = True
else:
self._index = "elasticlogs"
self._index_pattern = False
self._type = "doc"
self._timestamp_generator = TimestampStructGenerator(
params.get("starting_point", "now"),
params.get("offset"),
float(params.get("acceleration_factor", "1.0")),
# this is only expected to be used in tests
params.get("__utc_now"))
if "daily_logging_volume" in params and "client_count" in params:
# in bytes
self.daily_logging_volume = convert_to_bytes(
params["daily_logging_volume"]) // int(params["client_count"])
else:
self.daily_logging_volume = None
self.current_logging_volume = 0
self.total_days = params.get("number_of_days")
self.remaining_days = self.total_days
self.record_raw_event_size = params.get("record_raw_event_size", False)
self._offset = 0
self._web_host = itertools.cycle([1, 2, 3])
self._timestruct = None
self._index_name = None
self._time_interval_current_bulk = 0
@property
def percent_completed(self):
if self.daily_logging_volume is None or self.total_days is None:
return None
else:
full_days = self.total_days - self.remaining_days
already_generated = self.daily_logging_volume * full_days + self.current_logging_volume
total = self.total_days * self.daily_logging_volume
return already_generated / total
def start_bulk(self, bulk_size):
self._time_interval_current_bulk = 1 / bulk_size
self._timestruct = self._timestamp_generator.next_timestamp()
self._index_name = self.__generate_index_pattern(self._timestruct)
def generate_event(self):
if self.remaining_days == 0:
raise StopIteration()
# advance time by a few micros
self._timestruct = self._timestamp_generator.simulate_tick(
self._time_interval_current_bulk)
# index for the current line - we may cross a date boundary later if we're above the daily logging volume
index = self._index_name
event = self._event
event["@timestamp"] = self._timestruct["iso"]
# assume a typical event size of 263 bytes but limit the file size to 4GB
event["offset"] = (self._offset + 263) % (4 * 1024 * 1024 * 1024)
self._agent.add_fields(event)
self._clientip.add_fields(event)
self._referrer.add_fields(event)
self._request.add_fields(event)
event["hostname"] = "web-%s-%s.elastic.co" % (
event["geoip_continent_code"], next(self._web_host))
if self.record_raw_event_size or self.daily_logging_volume:
# determine the raw event size (as if this were contained in nginx log file). We do not bother to
# reformat the timestamp as this is not worth the overhead.
raw_event = '%s - - [%s] "%s %s HTTP/%s" %s %s "%s" "%s"' % (
event["clientip"], event["@timestamp"], event["verb"],
event["request"], event["httpversion"], event["response"],
event["bytes"], event["referrer"], event["agent"])
if self.daily_logging_volume:
self.current_logging_volume += len(raw_event)
if self.current_logging_volume > self.daily_logging_volume:
if self.remaining_days is not None:
self.remaining_days -= 1
self._timestamp_generator.skip(datetime.timedelta(days=1))
# advance time now for real (we usually use #simulate_tick() which will keep everything except for
# microseconds constant.
self._timestruct = self._timestamp_generator.next_timestamp(
)
self._index_name = self.__generate_index_pattern(
self._timestruct)
self.current_logging_volume = 0
if self.record_raw_event_size:
# we are on the hot code path here and thus we want to avoid conditionally creating strings so we duplicate
# the event.
line = '{"@timestamp": "%s", ' \
'"_raw_event_size":%d, ' \
'"offset":%s, ' \
'"source":"/usr/local/var/log/nginx/access.log","fileset":{"module":"nginx","name":"access"},"input":{"type":"log"},' \
'"beat":{"version":"6.3.0","hostname":"%s","name":"%s"},' \
'"prospector":{"type":"log"},' \
'"nginx":{"access":{"user_name": "-",' \
'"agent":"%s","user_agent": {"major": "%s","os": "%s","os_major": "%s","name": "%s","os_name": "%s","device": "%s"},' \
'"remote_ip": "%s","remote_ip_list":["%s"],' \
'"geoip":{"continent_name": "%s","city_name": "%s","country_name": "%s","country_iso_code": "%s","location":{"lat": %s,"lon": %s} },' \
'"referrer":"%s",' \
'"url": "%s","body_sent":{"bytes": %s},"method":"%s","response_code":%s,"http_version":"%s"} } }' % \
(event["@timestamp"],
len(raw_event),
event["offset"],
event["hostname"],event["hostname"],
event["agent"], event["useragent_major"], event["useragent_os"], event["useragent_os_major"], event["useragent_name"], event["useragent_os_name"], event["useragent_device"],
event["clientip"], event["clientip"],
event["geoip_continent_name"], event["geoip_city_name"], event["geoip_country_name"], event["geoip_country_iso_code"], event["geoip_location_lat"], event["geoip_location_lon"],
event["referrer"],
event["request"], event["bytes"], event["verb"], event["response"], event["httpversion"])
else:
line = '{"@timestamp": "%s", ' \
'"offset":%s, ' \
'"source":"/usr/local/var/log/nginx/access.log","fileset":{"module":"nginx","name":"access"},"input":{"type":"log"},' \
'"beat":{"version":"6.3.0","hostname":"%s","name":"%s"},' \
'"prospector":{"type":"log"},' \
'"nginx":{"access":{"user_name": "-",' \
'"agent":"%s","user_agent": {"major": "%s","os": "%s","os_major": "%s","name": "%s","os_name": "%s","device": "%s"},' \
'"remote_ip": "%s","remote_ip_list":["%s"],' \
'"geoip":{"continent_name": "%s","city_name": "%s","country_name": "%s","country_iso_code": "%s","location":{"lat": %s,"lon": %s} },' \
'"referrer":"%s",' \
'"url": "%s","body_sent":{"bytes": %s},"method":"%s","response_code":%s,"http_version":"%s"} } }' % \
(event["@timestamp"],
event["offset"],
event["hostname"],event["hostname"],
event["agent"], event["useragent_major"], event["useragent_os"], event["useragent_os_major"], event["useragent_name"], event["useragent_os_name"], event["useragent_device"],
event["clientip"], event["clientip"],
event["geoip_continent_name"], event["geoip_city_name"], event["geoip_country_name"], event["geoip_country_iso_code"], event["geoip_location_lat"], event["geoip_location_lon"],
event["referrer"],
event["request"], event["bytes"], event["verb"], event["response"], event["httpversion"])
return line, index, self._type
def __generate_index_pattern(self, timestruct):
if self._index_pattern:
return self._index.format(ts=timestruct)
else:
return self._index
def __init__(self, track, params, **kwargs):
self._indices = track.indices
self._params = params
self._samples = []
self._next_index = 0
self._bulk_size = 1000
if 'bulk-size' in params.keys():
self._bulk_size = params['bulk-size']
self._default_index = False
if 'index' not in params.keys():
if len(self._indices) > 1:
logger.debug(
"[bulk] More than one index specified in track configuration. Will use the first one ({})"
.format(self._indices[0].name))
else:
logger.debug(
"[bulk] Using index specified in track configuration ({})".
format(self._indices[0].name))
self._params['index'] = self._indices[0].name
self._default_index = True
else:
logger.debug(
"[bulk] Index pattern specified in parameters ({}) will be used"
.format(params['index']))
if 'type' not in params.keys():
t = self._indices[0].types[0]
self._params['type'] = t if isinstance(t, str) else t.name
if 'timestamp_field' not in params.keys():
self._params['timestamp_field'] = []
else:
if isinstance(params['timestamp_field'], list):
self._timestamp_field = params['timestamp_field']
else:
self._timestamp_field = [params['timestamp_field']]
if 'sample_file' not in params.keys():
raise ConfigurationError(
'Sample file(s) not supplied through the sample_file configuration parameter.'
)
else:
if isinstance(params['sample_file'], list):
self._params['sample_file'] = params['sample_file']
else:
self._params['sample_file'] = [params['sample_file']]
records = load_json_file.load_data_files(self._params['sample_file'])
logger.info("[sample_based_bulk] {} samples loaded.".format(
len(records)))
for rec in records:
sample = {}
if '_type' in rec.keys():
sample['type'] = rec['_type']
rec.pop('_type', None)
if '_index' in rec.keys():
sample['index'] = rec['_index']
rec.pop('_index', None)
sample['record'] = rec
self._samples.append(sample)
self._index = 'logs'
self._index_pattern = False
if 'index' in params.keys():
index = re.sub(r'{{\s*yyyy\s*}}',
'{ts[yyyy]}',
params['index'],
flags=re.IGNORECASE)
index = re.sub(r'{{\s*yy\s*}}',
'{ts[yy]}',
index,
flags=re.IGNORECASE)
index = re.sub(r'{{\s*mm\s*}}',
'{ts[mm]}',
index,
flags=re.IGNORECASE)
index = re.sub(r'{{\s*dd\s*}}',
'{ts[dd]}',
index,
flags=re.IGNORECASE)
index = re.sub(r'{{\s*hh\s*}}',
'{ts[hh]}',
index,
flags=re.IGNORECASE)
self._index = index
self._index_pattern = True
self._type = 'logs'
if 'type' in params.keys():
self._type = params['type']
if 'starting_point' in params.keys():
sp = params['starting_point']
else:
sp = "now"
if 'end_point' in params.keys():
ep = params['end_point']
self._timestamp_generator = TimestampStructGenerator.Interval(
sp, ep)
else:
if 'acceleration_factor' in params.keys():
af = float(params['acceleration_factor'])
self._timestamp_generator = TimestampStructGenerator.StartingPoint(
sp, af)
else:
self._timestamp_generator = TimestampStructGenerator.StartingPoint(
sp)