def process2d(self, file, name, field):
datasets = []
with open(file, 'r') as f:
for l in f.readlines():
if re.match(r'^#', l):
continue
l = re.sub(r'[\r\n]+$', '', l)
dat = re.split(r'\s+', l)
dataset = Dataset()
dataset.name = name
dataset.start_time = int(dat.pop(0))
dataset.stop_time = dataset.start_time + 60
d1 = Dimension('All')
d1.value = 'ALL'
dataset.dimensions.append(d1)
d2 = Dimension(field)
while dat:
if len(dat) < 2:
raise Exception('DAT %r dataset %r: invalid number of elements for a 2d dataset' % (file, name))
k = dat.pop(0)
v = dat.pop(0)
d2.values[k] = int(v)
d1.dimensions.append(d2)
datasets.append(dataset)
return datasets
def process(self, file):
dom = minidom.parse(file)
datasets = []
for array in dom.getElementsByTagName('array'):
if process_dataset and not array.getAttribute(
'name') in process_dataset:
continue
dataset = Dataset()
dataset.name = array.getAttribute('name')
dataset.start_time = int(array.getAttribute('start_time'))
dataset.stop_time = int(array.getAttribute('stop_time'))
dimensions = [None, None]
for dimension in array.getElementsByTagName('dimension'):
if dimension.getAttribute('number') == '1':
if dimensions[0]:
logging.warning('Overwriting dimension 1 for %s' %
dataset.name)
dimensions[0] = dimension.getAttribute('type')
elif dimension.getAttribute('number') == '2':
if dimensions[1]:
logging.warning('Overwriting dimension 2 for %s' %
dataset.name)
dimensions[1] = dimension.getAttribute('type')
else:
logging.warning(
'Invalid dimension number %r for %s' %
(dimension.getAttribute('number'), dataset.name))
for node1 in array.getElementsByTagName(dimensions[0]):
d1 = Dimension(dimensions[0])
d1.value = node1.getAttribute('val')
try:
if node1.getAttribute('base64'):
d1.value = base64.b64decode(d1.value).decode('utf-8')
except Exception as e:
pass
dataset.dimensions.append(d1)
d2 = Dimension(dimensions[1])
d1.dimensions.append(d2)
for node2 in node1.getElementsByTagName(dimensions[1]):
val = node2.getAttribute('val')
try:
if node2.getAttribute('base64'):
val = base64.b64decode(val).decode('utf-8')
except Exception as e:
pass
d2.values[val] = int(node2.getAttribute('count'))
datasets.append(dataset)
return datasets
def process(self, datasets):
gen_datasets = []
for dataset in datasets:
if dataset.name != 'client_subnet':
continue
subnets = {}
for d1 in dataset.dimensions:
for d2 in d1.dimensions:
for k, v in d2.values.items():
if k == args.skipped_key:
continue
elif k == args.skipped_sum_key:
continue
if k in subnets:
subnets[k] += v
else:
subnets[k] = v
auth = {}
for subnet in subnets:
ip = ipaddress.ip_address(subnet)
if ip.version == 4:
idx = ipaddress.ip_network('%s/8' % ip, strict=False)
ip = ipaddress.ip_network('%s/32' % ip)
else:
idx = ipaddress.ip_network('%s/24' % ip, strict=False)
ip = ipaddress.ip_network('%s/128' % ip)
if not idx.network_address in self.auth:
idx = '??'
else:
for entry in self.auth[idx.network_address]:
if entry['net'].overlaps(ip):
idx = entry['auth']
break
if idx in auth:
auth[idx] += subnets[subnet]
else:
auth[idx] = subnets[subnet]
if auth:
authd = Dataset()
authd.name = 'client_subnet_authority'
authd.start_time = dataset.start_time
authd.stop_time = dataset.stop_time
gen_datasets.append(authd)
authd1 = Dimension('ClientAuthority')
authd1.values = auth
authd.dimensions.append(authd1)
return gen_datasets
def process(self, datasets):
gen_datasets = []
for dataset in datasets:
if dataset.name != 'client_subnet':
continue
subnets = {}
for d1 in dataset.dimensions:
for d2 in d1.dimensions:
for k, v in d2.values.items():
if k == args.skipped_key:
continue
elif k == args.skipped_sum_key:
continue
if k in subnets:
subnets[k] += v
else:
subnets[k] = v
cc = {}
for subnet in subnets:
try:
c = self.reader.get(subnet)
except Exception as e:
if not self.nonstrict:
raise e
continue
if c:
iso_code = c.get('country', {}).get('iso_code', '??')
if iso_code in cc:
cc[iso_code] += subnets[subnet]
else:
cc[iso_code] = subnets[subnet]
if cc:
ccd = Dataset()
ccd.name = 'client_subnet_country'
ccd.start_time = dataset.start_time
ccd.stop_time = dataset.stop_time
gen_datasets.append(ccd)
ccd1 = Dimension('ClientCountry')
ccd1.values = cc
ccd.dimensions.append(ccd1)
return gen_datasets
def process1d(self, file, name):
datasets = []
with open(file, 'r') as f:
for l in f.readlines():
if re.match(r'^#', l):
continue
l = re.sub(r'[\r\n]+$', '', l)
dat = re.split(r'\s+', l)
if len(dat) != 2:
raise Exception('DAT %r dataset %r: invalid number of elements for a 1d dataset' % (file, name))
dataset = Dataset()
dataset.name = name
dataset.start_time = int(dat.pop(0))
dataset.stop_time = dataset.start_time + 60
d1 = Dimension('All')
d1.values = { 'ALL': int(dat[0]) }
dataset.dimensions.append(d1)
datasets.append(dataset)
return datasets
def test_dimension():
o = Dimension('test')
assert '%r' % o == '<Dimension name=\'test\' value=None dimension=[]>'