def __init__(self):
if '_agents' in ebs.global_lookups.keys():
self._agents = ebs.global_lookups['_agents']
else:
self._agents = WeightedArray('%s/data/agents.json.gz' % cwd)
ebs.global_lookups['_agents'] = self._agents
if '_agents_name_lookup' in ebs.global_lookups.keys():
self._agents_name_lookup = ebs.global_lookups['_agents_name_lookup']
else:
with gzip.open('%s/data/agents_name_lookup.json.gz' % cwd, 'rt') as data_file:
self._agents_name_lookup = json.load(data_file)
ebs.global_lookups['_agents_name_lookup'] = self._agents_name_lookup
if '_agents_os_lookup' in ebs.global_lookups.keys():
self._agents_os_lookup = ebs.global_lookups['_agents_os_lookup']
else:
with gzip.open('%s/data/agents_os_lookup.json.gz' % cwd, 'rt') as data_file:
self._agents_os_lookup = json.load(data_file)
ebs.global_lookups['_agents_os_lookup'] = self._agents_os_lookup
if '_agents_os_name_lookup' in ebs.global_lookups.keys():
self._agents_os_name_lookup = ebs.global_lookups['_agents_os_name_lookup']
else:
with gzip.open('%s/data/agents_os_name_lookup.json.gz' % cwd, 'rt') as data_file:
self._agents_os_name_lookup = json.load(data_file)
ebs.global_lookups['_agents_os_name_lookup'] = self._agents_os_name_lookup
if '_agents_os_major_lookup' in ebs.global_lookups.keys():
self._agents_os_major_lookup = ebs.global_lookups['_agents_os_major_lookup']
else:
with gzip.open('%s/data/agents_os_major_lookup.json.gz' % cwd, 'rt') as data_file:
self._agents_os_major_lookup = json.load(data_file)
ebs.global_lookups['_agents_os_major_lookup'] = self._agents_os_major_lookup
if '_agents_major_lookup' in ebs.global_lookups.keys():
self._agents_major_lookup = ebs.global_lookups['_agents_major_lookup']
else:
with gzip.open('%s/data/agents_major_lookup.json.gz' % cwd, 'rt') as data_file:
self._agents_major_lookup = json.load(data_file)
ebs.global_lookups['_agents_major_lookup'] = self._agents_major_lookup
if '_agents_device_lookup' in ebs.global_lookups.keys():
self._agents_device_lookup = ebs.global_lookups['_agents_device_lookup']
else:
with gzip.open('%s/data/agents_device_lookup.json.gz' % cwd, 'rt') as data_file:
self._agents_device_lookup = json.load(data_file)
ebs.global_lookups['_agents_device_lookup'] = self._agents_device_lookup
if '_agent_lookup' in ebs.global_lookups.keys():
self._agent_lookup = ebs.global_lookups['_agent_lookup']
else:
with gzip.open('%s/data/agent_lookup.json.gz' % cwd, 'rt') as data_file:
self._agent_lookup = json.load(data_file)
ebs.global_lookups['_agent_lookup'] = self._agent_lookup
def __init__(self):
if '_requests' in ebs.global_lookups.keys():
self._requests = ebs.global_lookups['_requests']
else:
self._requests = WeightedArray('%s/data/requests.json.gz' % cwd)
ebs.global_lookups['_requests'] = self._requests
if '_requests_url_base_lookup' in ebs.global_lookups.keys():
self._requests_url_base_lookup = ebs.global_lookups['_requests_url_base_lookup']
else:
with gzip.open('%s/data/requests_url_base_lookup.json.gz' % cwd, 'rt') as data_file:
self._requests_url_base_lookup = json.load(data_file)
ebs.global_lookups['_requests_url_base_lookup'] = self._requests_url_base_lookup
def __init__(self):
self._rare_clientip_probability = 0.269736965199
self._clientips = WeightedArray('%s/data/clientips.json.gz' % cwd)
self._rare_clientips = WeightedArray('%s/data/rare_clientips.json.gz' %
cwd)
with gzip.open('%s/data/clientips_country_lookup.json.gz' % cwd,
'rt') as data_file:
self._clientips_country_lookup = json.load(data_file)
with gzip.open('%s/data/clientips_location_lookup.json.gz' % cwd,
'rt') as data_file:
self._clientips_location_lookup = json.load(data_file)
def __init__(self):
self._agents = WeightedArray('%s/data/agents.json.gz' % cwd)
with gzip.open('%s/data/agents_name_lookup.json.gz' % cwd,
'rt') as data_file:
self._agents_name_lookup = json.load(data_file)
with gzip.open('%s/data/agents_os_lookup.json.gz' % cwd,
'rt') as data_file:
self._agents_os_lookup = json.load(data_file)
with gzip.open('%s/data/agents_os_name_lookup.json.gz' % cwd,
'rt') as data_file:
self._agents_os_name_lookup = json.load(data_file)
def __init__(self):
self._rare_clientip_probability = 0.269736965199
if '_clientips' in ebs.global_lookups.keys():
self._clientips = ebs.global_lookups['_clientips']
else:
self._clientips = WeightedArray('%s/data/clientips.json.gz' % cwd)
ebs.global_lookups['_clientips'] = self._clientips
if '_rare_clientips' in ebs.global_lookups.keys():
self._rare_clientips = ebs.global_lookups['_rare_clientips']
else:
self._rare_clientips = WeightedArray('%s/data/rare_clientips.json.gz' % cwd)
ebs.global_lookups['_rare_clientips'] = self._rare_clientips
if '_clientips_country_name_lookup' in ebs.global_lookups.keys():
self._clientips_country_name_lookup = ebs.global_lookups['_clientips_country_name_lookup']
else:
with gzip.open('%s/data/clientips_country_name_lookup.json.gz' % cwd, 'rt') as data_file:
self._clientips_country_name_lookup = json.load(data_file)
ebs.global_lookups['_clientips_country_name_lookup'] = self._clientips_country_name_lookup
if '_clientips_country_iso_code_lookup' in ebs.global_lookups.keys():
self._clientips_country_iso_code_lookup = ebs.global_lookups['_clientips_country_iso_code_lookup']
else:
with gzip.open('%s/data/clientips_country_iso_code_lookup.json.gz' % cwd, 'rt') as data_file:
self._clientips_country_iso_code_lookup = json.load(data_file)
ebs.global_lookups['_clientips_country_iso_code_lookup'] = self._clientips_country_iso_code_lookup
if '_clientips_continent_name_lookup' in ebs.global_lookups.keys():
self._clientips_continent_name_lookup = ebs.global_lookups['_clientips_continent_name_lookup']
else:
with gzip.open('%s/data/clientips_continent_name_lookup.json.gz' % cwd, 'rt') as data_file:
self._clientips_continent_name_lookup = json.load(data_file)
ebs.global_lookups['_clientips_continent_name_lookup'] = self._clientips_continent_name_lookup
if '_clientips_continent_code_lookup' in ebs.global_lookups.keys():
self._clientips_continent_code_lookup = ebs.global_lookups['_clientips_continent_code_lookup']
else:
with gzip.open('%s/data/clientips_continent_code_lookup.json.gz' % cwd, 'rt') as data_file:
self._clientips_continent_code_lookup = json.load(data_file)
ebs.global_lookups['_clientips_continent_code_lookup'] = self._clientips_continent_code_lookup
if '_clientips_city_name_lookup' in ebs.global_lookups.keys():
self._clientips_city_name_lookup = ebs.global_lookups['_clientips_city_name_lookup']
else:
with gzip.open('%s/data/clientips_city_name_lookup.json.gz' % cwd, 'rt') as data_file:
self._clientips_city_name_lookup = json.load(data_file)
ebs.global_lookups['_clientips_city_name_lookup'] = self._clientips_city_name_lookup
class Request:
def __init__(self):
if '_requests' in ebs.global_lookups.keys():
self._requests = ebs.global_lookups['_requests']
else:
self._requests = WeightedArray('%s/data/requests.json.gz' % cwd)
ebs.global_lookups['_requests'] = self._requests
if '_requests_url_base_lookup' in ebs.global_lookups.keys():
self._requests_url_base_lookup = ebs.global_lookups[
'_requests_url_base_lookup']
else:
with gzip.open('%s/data/requests_url_base_lookup.json.gz' % cwd,
'rt') as data_file:
self._requests_url_base_lookup = json.load(data_file)
ebs.global_lookups[
'_requests_url_base_lookup'] = self._requests_url_base_lookup
def add_fields(self, event):
data = self._requests.get_random()
event['request'] = "{}{}".format(
self._requests_url_base_lookup[data[0]], data[1])
event['bytes'] = data[2]
event['verb'] = data[3]
event['response'] = data[4]
event['httpversion'] = data[5]
class Referrer:
def __init__(self):
self._referrers = WeightedArray('%s/data/referrers.json.gz' % cwd)
with gzip.open('%s/data/referrers_url_base_lookup.json.gz' % cwd,
'rt') as data_file:
self._referrers_url_base_lookup = json.load(data_file)
def add_fields(self, event):
data = self._referrers.get_random()
event['referrer'] = "%s%s" % (self._referrers_url_base_lookup[data[0]],
data[1])
class ClientIp:
def __init__(self):
self._rare_clientip_probability = 0.269736965199
self._clientips = WeightedArray('%s/data/clientips.json.gz' % cwd)
self._rare_clientips = WeightedArray('%s/data/rare_clientips.json.gz' %
cwd)
with gzip.open('%s/data/clientips_country_lookup.json.gz' % cwd,
'rt') as data_file:
self._clientips_country_lookup = json.load(data_file)
with gzip.open('%s/data/clientips_location_lookup.json.gz' % cwd,
'rt') as data_file:
self._clientips_location_lookup = json.load(data_file)
def add_fields(self, event):
p = random.random()
if p < self._rare_clientip_probability:
data = self._rare_clientips.get_random()
event['clientip'] = self.__fill_out_ip_prefix(data[0])
else:
data = self._clientips.get_random()
event['clientip'] = data[0]
event['country_name'] = self._clientips_country_lookup[data[1]]
event['location'] = self._clientips_location_lookup[data[2]]
def __fill_out_ip_prefix(self, ip_prefix):
rnd1 = random.random()
v1 = rnd1 * (1 - rnd1) * 255 * 4
k1 = (int)(v1)
rnd2 = random.random()
v2 = rnd2 * (1 - rnd2) * 255 * 4
k2 = (int)(v2)
return "{}.{}.{}".format(ip_prefix, k1, k2)
class Referrer:
def __init__(self):
if '_referrers' in ebs.global_lookups.keys():
self._referrers = ebs.global_lookups['_referrers']
else:
self._referrers = WeightedArray('%s/data/referrers.json.gz' % cwd)
ebs.global_lookups['_referrers'] = self._referrers
if '_referrers_url_base_lookup' in ebs.global_lookups.keys():
self._referrers_url_base_lookup = ebs.global_lookups['_referrers_url_base_lookup']
else:
with gzip.open('%s/data/referrers_url_base_lookup.json.gz' % cwd, 'rt') as data_file:
self._referrers_url_base_lookup = json.load(data_file)
ebs.global_lookups['_referrers_url_base_lookup'] = self._referrers_url_base_lookup
def add_fields(self, event):
data = self._referrers.get_random()
event['referrer'] = "%s%s" % (self._referrers_url_base_lookup[data[0]], data[1])
class Agent:
def __init__(self):
self._agents = WeightedArray('%s/data/agents.json.gz' % cwd)
with gzip.open('%s/data/agents_name_lookup.json.gz' % cwd,
'rt') as data_file:
self._agents_name_lookup = json.load(data_file)
with gzip.open('%s/data/agents_os_lookup.json.gz' % cwd,
'rt') as data_file:
self._agents_os_lookup = json.load(data_file)
with gzip.open('%s/data/agents_os_name_lookup.json.gz' % cwd,
'rt') as data_file:
self._agents_os_name_lookup = json.load(data_file)
def add_fields(self, event):
agent = self._agents.get_random()
event['agent'] = agent[0]
event['useragent_os'] = self._agents_os_lookup[agent[1]]
event['useragent_os_name'] = self._agents_os_name_lookup[agent[2]]
event['useragent_name'] = self._agents_name_lookup[agent[3]]
class Agent:
def __init__(self):
if '_agents' in ebs.global_lookups.keys():
self._agents = ebs.global_lookups['_agents']
else:
self._agents = WeightedArray('%s/data/agents.json.gz' % cwd)
ebs.global_lookups['_agents'] = self._agents
if '_agents_name_lookup' in ebs.global_lookups.keys():
self._agents_name_lookup = ebs.global_lookups[
'_agents_name_lookup']
else:
with gzip.open('%s/data/agents_name_lookup.json.gz' % cwd,
'rt') as data_file:
self._agents_name_lookup = json.load(data_file)
ebs.global_lookups[
'_agents_name_lookup'] = self._agents_name_lookup
if '_agents_os_lookup' in ebs.global_lookups.keys():
self._agents_os_lookup = ebs.global_lookups['_agents_os_lookup']
else:
with gzip.open('%s/data/agents_os_lookup.json.gz' % cwd,
'rt') as data_file:
self._agents_os_lookup = json.load(data_file)
ebs.global_lookups['_agents_os_lookup'] = self._agents_os_lookup
if '_agents_os_name_lookup' in ebs.global_lookups.keys():
self._agents_os_name_lookup = ebs.global_lookups[
'_agents_os_name_lookup']
else:
with gzip.open('%s/data/agents_os_name_lookup.json.gz' % cwd,
'rt') as data_file:
self._agents_os_name_lookup = json.load(data_file)
ebs.global_lookups[
'_agents_os_name_lookup'] = self._agents_os_name_lookup
if '_agents_os_major_lookup' in ebs.global_lookups.keys():
self._agents_os_major_lookup = ebs.global_lookups[
'_agents_os_major_lookup']
else:
with gzip.open('%s/data/agents_os_major_lookup.json.gz' % cwd,
'rt') as data_file:
self._agents_os_major_lookup = json.load(data_file)
ebs.global_lookups[
'_agents_os_major_lookup'] = self._agents_os_major_lookup
if '_agents_major_lookup' in ebs.global_lookups.keys():
self._agents_major_lookup = ebs.global_lookups[
'_agents_major_lookup']
else:
with gzip.open('%s/data/agents_major_lookup.json.gz' % cwd,
'rt') as data_file:
self._agents_major_lookup = json.load(data_file)
ebs.global_lookups[
'_agents_major_lookup'] = self._agents_major_lookup
if '_agents_device_lookup' in ebs.global_lookups.keys():
self._agents_device_lookup = ebs.global_lookups[
'_agents_device_lookup']
else:
with gzip.open('%s/data/agents_device_lookup.json.gz' % cwd,
'rt') as data_file:
self._agents_device_lookup = json.load(data_file)
ebs.global_lookups[
'_agents_device_lookup'] = self._agents_device_lookup
if '_agent_lookup' in ebs.global_lookups.keys():
self._agent_lookup = ebs.global_lookups['_agent_lookup']
else:
with gzip.open('%s/data/agent_lookup.json.gz' % cwd,
'rt') as data_file:
self._agent_lookup = json.load(data_file)
ebs.global_lookups['_agent_lookup'] = self._agent_lookup
def add_fields(self, event):
agent = self._agents.get_random()
event['useragent_name'] = self.__get_lookup_value(
self._agents_name_lookup, agent[0])
event['useragent_os'] = self.__get_lookup_value(
self._agents_os_lookup, agent[1])
event['useragent_os_name'] = self.__get_lookup_value(
self._agents_os_name_lookup, agent[2])
event['useragent_device'] = self.__get_lookup_value(
self._agents_device_lookup, agent[3])
event['useragent_os_major'] = self.__get_lookup_value(
self._agents_os_major_lookup, agent[4])
event['useragent_major'] = self.__get_lookup_value(
self._agents_major_lookup, agent[5])
event['agent'] = self.__get_lookup_value(self._agent_lookup, agent[6])
def __get_lookup_value(self, lookup, key):
if key == "":
return key
else:
return lookup[key]
class ClientIp:
def __init__(self):
self._rare_clientip_probability = 0.269736965199
if '_clientips' in ebs.global_lookups.keys():
self._clientips = ebs.global_lookups['_clientips']
else:
self._clientips = WeightedArray('%s/data/clientips.json.gz' % cwd)
ebs.global_lookups['_clientips'] = self._clientips
if '_rare_clientips' in ebs.global_lookups.keys():
self._rare_clientips = ebs.global_lookups['_rare_clientips']
else:
self._rare_clientips = WeightedArray(
'%s/data/rare_clientips.json.gz' % cwd)
ebs.global_lookups['_rare_clientips'] = self._rare_clientips
if '_clientips_country_name_lookup' in ebs.global_lookups.keys():
self._clientips_country_name_lookup = ebs.global_lookups[
'_clientips_country_name_lookup']
else:
with gzip.open(
'%s/data/clientips_country_name_lookup.json.gz' % cwd,
'rt') as data_file:
self._clientips_country_name_lookup = json.load(data_file)
ebs.global_lookups[
'_clientips_country_name_lookup'] = self._clientips_country_name_lookup
if '_clientips_country_iso_code_lookup' in ebs.global_lookups.keys():
self._clientips_country_iso_code_lookup = ebs.global_lookups[
'_clientips_country_iso_code_lookup']
else:
with gzip.open(
'%s/data/clientips_country_iso_code_lookup.json.gz' % cwd,
'rt') as data_file:
self._clientips_country_iso_code_lookup = json.load(data_file)
ebs.global_lookups[
'_clientips_country_iso_code_lookup'] = self._clientips_country_iso_code_lookup
if '_clientips_continent_name_lookup' in ebs.global_lookups.keys():
self._clientips_continent_name_lookup = ebs.global_lookups[
'_clientips_continent_name_lookup']
else:
with gzip.open(
'%s/data/clientips_continent_name_lookup.json.gz' % cwd,
'rt') as data_file:
self._clientips_continent_name_lookup = json.load(data_file)
ebs.global_lookups[
'_clientips_continent_name_lookup'] = self._clientips_continent_name_lookup
if '_clientips_continent_code_lookup' in ebs.global_lookups.keys():
self._clientips_continent_code_lookup = ebs.global_lookups[
'_clientips_continent_code_lookup']
else:
with gzip.open(
'%s/data/clientips_continent_code_lookup.json.gz' % cwd,
'rt') as data_file:
self._clientips_continent_code_lookup = json.load(data_file)
ebs.global_lookups[
'_clientips_continent_code_lookup'] = self._clientips_continent_code_lookup
if '_clientips_city_name_lookup' in ebs.global_lookups.keys():
self._clientips_city_name_lookup = ebs.global_lookups[
'_clientips_city_name_lookup']
else:
with gzip.open('%s/data/clientips_city_name_lookup.json.gz' % cwd,
'rt') as data_file:
self._clientips_city_name_lookup = json.load(data_file)
ebs.global_lookups[
'_clientips_city_name_lookup'] = self._clientips_city_name_lookup
def add_fields(self, event):
p = random.random()
if p < self._rare_clientip_probability:
data = self._rare_clientips.get_random()
event['clientip'] = self.__fill_out_ip_prefix(data[0])
else:
data = self._clientips.get_random()
event['clientip'] = data[0]
event['geoip_location_lat'] = data[1][0]
event['geoip_location_lon'] = data[1][1]
event['geoip_city_name'] = self.__get_lookup_value(
self._clientips_city_name_lookup, data[2])
event['geoip_country_name'] = self.__get_lookup_value(
self._clientips_country_name_lookup, data[3])
event['geoip_country_iso_code'] = self.__get_lookup_value(
self._clientips_country_iso_code_lookup, data[4])
event['geoip_continent_name'] = self.__get_lookup_value(
self._clientips_continent_name_lookup, data[5])
event['geoip_continent_code'] = self.__get_lookup_value(
self._clientips_continent_code_lookup, data[5])
def __fill_out_ip_prefix(self, ip_prefix):
rnd1 = random.random()
v1 = rnd1 * (1 - rnd1) * 255 * 4
k1 = (int)(v1)
rnd2 = random.random()
v2 = rnd2 * (1 - rnd2) * 255 * 4
k2 = (int)(v2)
return "{}.{}.{}".format(ip_prefix, k1, k2)
def __get_lookup_value(self, lookup, key):
if key == "":
return key
else:
return lookup[key]
def __init__(self):
self._requests = WeightedArray('%s/data/requests.json.gz' % cwd)
with gzip.open('%s/data/requests_url_base_lookup.json.gz' % cwd,
'rt') as data_file:
self._requests_url_base_lookup = json.load(data_file)
