def get_layers(kernel_width, num_kernels, num_layers):
layers = []
for layer_index in range(num_layers):
layer = copy_copy(LAYER_DICT_TEMPLATE)
if layer_index == num_layers - 1:
# for the last layer, the output needs to be
layer["out_channels"] = 1
else:
layer["out_channels"] = num_kernels
if kernel_width == 5:
layer['kernel_width'] = 5
layer['padding_width'] = 2
layer['stride_width'] = 1
elif kernel_width == 7:
layer['kernel_width'] = 7
layer['padding_width'] = 3
layer['stride_width'] = 1
if layer_index == 0:
layer['in_channels'] = 1
else:
layer['in_channels'] = layers[-1]['out_channels']
layer['name'] = 'conv{}'.format(layer_index + 1)
layers.append(layer)
return layers
def startThreads(numberOfThreads):
global targetsList
global ctime
global TRACE
thread_counter = 0
probesReverseCounter = copy_copy(PROBECOUNT)
while probesReverseCounter != 0:
StartTime = time.time() #cycle start time in 1532321174.2756 format
for target in targetsList:
thread_counter += 1
# continue starting threads until maximum numberOfThreads reached
# and length of targetsList smaller then count of threads already started
if thread_counter != numberOfThreads and len(
targetsList) != thread_counter:
startThread(target)
# wait for threads to complete by joining them
else:
startThread(target)
thread_counter = 0
try:
for th in threads:
if TRACE: #if traceroute command then set timeout to 30 x probetimeout
th.join(timeout=30 * PROBETIMEOUT / 1000)
else:
th.join(timeout=3 * PROBETIMEOUT / 1000)
if SSILENT == False:
reprinter()
except KeyboardInterrupt:
closeFiles()
raise SystemExit('Exit: Interrupted by User')
# join remaining threads and run reprinter
try:
for th in threads:
if TRACE: #if traceroute command then set timeout to 30 x probetimeout
th.join(timeout=30 * PROBETIMEOUT / 1000)
else:
th.join(timeout=3 * PROBETIMEOUT / 1000)
if SSILENT == False:
reprinter()
else:
# write final logs for all probes if end reached
if probesReverseCounter == 1:
reprinter()
except KeyboardInterrupt:
closeFiles()
raise SystemExit('Exit: Interrupted by User')
probesReverseCounter -= 1
#calculate time spent running above threads/probes, if spent les than PROBEINTERVAL than sleep time remaining:
TimeElapsed = round(time.time() - StartTime, 4)
if PROBEINTERVAL / 1000 > TimeElapsed:
TimeToSleep = PROBEINTERVAL / 1000 - TimeElapsed
try:
time.sleep(TimeToSleep)
except KeyboardInterrupt:
closeFiles()
raise SystemExit('Exit: Interrupted by User')
def __init__(self, seperator, limits, default, censor_char='2'):
ConfigElement.__init__(self)
self.marked_pos = 0
self.seperator = seperator
self.limits = limits
self.censor_char = censor_char
self.last_value = self.default = default
self.value = copy_copy(default)
self.endNotifier = None
return
def _broadcast_matrix(a, b, wrt, data):
global _bs_setup_data1, _bs_setup_data2
if len(set((a.shape, b.shape))) == 1:
uid = a.shape
if uid not in _bs_setup_data1:
asz = a.size
IS = np.arange(asz)
_bs_setup_data1[uid] = sp.csc_matrix((np.empty(asz), (IS, IS)),
shape=(asz, asz))
result = copy_copy(_bs_setup_data1[uid])
if isinstance(data, np.ndarray):
result.data = data.ravel()
else: # assumed scalar
result.data = np.empty(result.nnz)
result.data.fill(data)
else:
uid = (a.shape, b.shape, wrt is a, wrt is b)
if uid not in _bs_setup_data2:
input_sz = wrt.size
output_sz = np.broadcast(a.r, b.r).size
a2 = np.arange(a.size).reshape(a.shape) if wrt is a else np.zeros(
a.shape)
b2 = np.arange(b.size).reshape(
b.shape) if (wrt is b and wrt is not a) else np.zeros(b.shape)
IS = np.arange(output_sz)
JS = np.asarray((np.add(a2, b2)).ravel(), np.uint32)
_bs_setup_data2[uid] = sp.csc_matrix(
(np.arange(IS.size), (IS, JS)), shape=(output_sz, input_sz))
result = copy_copy(_bs_setup_data2[uid])
if isinstance(data, np.ndarray):
result.data = data[result.data]
else: # assumed scalar
result.data = np.empty(result.nnz)
result.data.fill(data)
if np.prod(result.shape) == 1:
return np.array(data)
else:
return result
def _broadcast_matrix(a, b, wrt, data):
global _bs_setup_data1, _bs_setup_data2
if len(set((a.shape, b.shape))) == 1:
uid = a.shape
if uid not in _bs_setup_data1:
asz = a.size
IS = np.arange(asz)
_bs_setup_data1[uid] = sp.csc_matrix((np.empty(asz), (IS, IS)), shape=(asz, asz))
result = copy_copy(_bs_setup_data1[uid])
if isinstance(data, np.ndarray):
result.data = data.ravel()
else: # assumed scalar
result.data = np.empty(result.nnz)
result.data.fill(data)
else:
uid = (a.shape, b.shape, wrt is a, wrt is b)
if uid not in _bs_setup_data2:
input_sz = wrt.size
output_sz = np.broadcast(a.r, b.r).size
a2 = np.arange(a.size).reshape(a.shape) if wrt is a else np.zeros(a.shape)
b2 = np.arange(b.size).reshape(b.shape) if (wrt is b and wrt is not a) else np.zeros(b.shape)
IS = np.arange(output_sz)
JS = np.asarray((np.add(a2,b2)).ravel(), np.uint32)
_bs_setup_data2[uid] = sp.csc_matrix((np.arange(IS.size), (IS, JS)), shape=(output_sz, input_sz))
result = copy_copy(_bs_setup_data2[uid])
if isinstance(data, np.ndarray):
result.data = data[result.data]
else: # assumed scalar
result.data = np.empty(result.nnz)
result.data.fill(data)
if np.prod(result.shape) == 1:
return np.array(data)
else:
return result
def __init__(self, seperator, limits, default, censor_char = ""): ConfigElement.__init__(self) assert isinstance(limits, list) and len(limits[0]) == 2, "limits must be [(min, max),...]-tuple-list" assert censor_char == "" or len(censor_char) == 1, "censor char must be a single char (or \"\")" #assert isinstance(default, list), "default must be a list" #assert isinstance(default[0], int), "list must contain numbers" #assert len(default) == len(limits), "length must match" self.marked_pos = 0 self.seperator = seperator self.limits = limits self.censor_char = censor_char self.last_value = self.default = default self.value = copy_copy(default) self.endNotifier = None
def __init__(self, seperator, limits, default, censor_char = ""): ConfigElement.__init__(self) assert isinstance(limits, list) and len(limits[0]) == 2, "limits must be [(min, max),...]-tuple-list" assert censor_char == "" or len(censor_char) == 1, "censor char must be a single char (or \"\")" #assert isinstance(default, list), "default must be a list" #assert isinstance(default[0], int), "list must contain numbers" #assert len(default) == len(limits), "length must match" self.marked_pos = 0 self.seperator = seperator self.limits = limits self.censor_char = censor_char self.last_value = self.default = default self.value = copy_copy(default) self.endNotifier = None
def targets_from_subnet(TD):
#TD - target dictionary based on targetDictglob var
result = []
if ":" in TD['target']: #create IPv6 subnet object
subnetObj = ipaddress_IPv6Network(TD['target'], strict=False)
elif "." in TD['target']: #create IPv4 subnet object:
subnetObj = ipaddress_IPv4Network(TD['target'], strict=False)
subnet_hosts = [str(i) for i in list(subnetObj.hosts())]
#go over subnet hosts and copy targetDict parameters on them:
for subnet_host in subnet_hosts:
result.append({})
result[-1].update(TD)
result[-1]['target'] = subnet_host
result[-1]['Command'] = TD['Command'].replace(TD['target'],
subnet_host)
#have had to do below to prevent subnet_host becoming equal to last hos tin subnet:
TD['commandList'][hostIndex] = subnet_host
result[-1]['commandList'] = copy_copy(TD['commandList'])
return result
def load_medium_file(
filename: str, logger: Logger = getLogger(__name__)) -> pd.DataFrame:
'''Load a file to fit in a dataframe supporting informations linked to a medium
:param filename: Path of the file
:type filename: str
:return: A dataframe formatted
:rtype: pd.DataFrame
'''
df = pd.DataFrame()
try:
if not os_path.isfile(filename):
raise FileNotFoundError('File %s not found' % (filename, ))
# Find delimiter.
with open(filename) as fid:
dialect = csv.Sniffer().sniff(fid.readline())
# Load.
df = pd.read_csv(filename, sep=dialect.delimiter)
# Check header.
header_required = copy_copy(__MEDIUM_HEADER)
for header in __MEDIUM_HEADER_OPTIONAL:
if header in header_required:
header_required.remove(header)
for header in header_required:
if not header in df.columns:
raise HeaderMalformated('Header is malformated')
# Fmt.
df[__MEDIUM_HEADER_BOUND] = df[__MEDIUM_HEADER_BOUND].astype(float)
# Translate to rp_compound
df = create_rp_compound(df, logger)
except Exception as e:
logger.error(str(e))
df = pd.DataFrame()
return df
def onDeselect(self, session): if self.last_value != self._value: self.changedFinal() self.last_value = copy_copy(self._value)
def onDeselect(self, session): if self.last_value != self._value: self.changedFinal() self.last_value = copy_copy(self._value)
def get_model_dict(kernel_width, num_kernels, num_layers):
model_dict = copy_copy(MODEL_DICT_TEMPLATE)
layers = get_layers(kernel_width, num_kernels, num_layers)
model_dict['layers'] = layers
return model_dict
def get_which_model_from_params_fname(model_params_fname, return_params=False):
# load the model
model_params = read_model_params(model_params_fname)
# If the json is a list, it's a FlexNet
if 'type' in model_params and model_params[
'type'] == 'alexnet' and model_params['version'] == '1.0':
from lib.flexnet import FlexNet
model_class = FlexNet
model = FlexNet(model_params_fname)
if return_params is True:
return model, model_params
return model
if 'model' not in model_params:
# print('get_which_model_from_params_fname: using LeNet')
from lib.lenet import LeNet # Circular dependency
model_class = LeNet
elif model_params['model'] == 'AlexNet':
# print('get_which_model_from_params_fname: using AlexNet')
from lib.alexnet import AlexNet
model_class = AlexNet
elif model_params['model'] == 'MLPB5':
from lib.flexnet import FlexNet
model_init_params = copy_copy(model_params)
# Delete training parameters
del model_init_params['batch_size']
del model_init_params['data_is_target']
del model_init_params['data_noise_gaussian']
del model_init_params['data_train']
del model_init_params['data_val']
del model_init_params['k']
del model_init_params['learning_rate']
del model_init_params['loss_function']
del model_init_params['model']
del model_init_params['momentum']
del model_init_params['optimizer']
del model_init_params['version']
del model_init_params['patience']
del model_init_params['weight_decay']
model = FlexNet(model_init_params)
if return_params is True:
return model, model_params
return model
elif model_params['model'] in ['FCN', 'FCNN']:
from lib.flexnet import FlexNet
model_init_params = copy_copy(model_params)
# Delete training parameters
del model_init_params['batch_size']
del model_init_params['data_is_target']
del model_init_params['data_noise_gaussian']
del model_init_params['data_train']
del model_init_params['data_val']
del model_init_params['k']
del model_init_params['learning_rate']
del model_init_params['loss_function']
del model_init_params['model']
del model_init_params['momentum']
del model_init_params['optimizer']
del model_init_params['version']
del model_init_params['patience']
del model_init_params['weight_decay']
model = FlexNet(model_init_params)
if return_params is True:
return model, model_params
return model
if 'input_channel' in model_params:
input_channel = model_params['input_channel']
else:
input_channel = 2 # By default, we used 2-channel (2*65) input
if '2018' in model_params_fname:
from lib.lenet_1d import LeNet_1D # Circular dependency
model_class = LeNet_1D
if 'type' in model_params and model_params['type'] == 'lenet_1d':
from lib.lenet_1d import LeNet_1D # Circular dependency
model_class = LeNet_1D
try:
model = model_class(
input_channel,
# model_params['input'],
model_params['output_size'],
model_params['batch_norm'],
model_params['use_pooling'],
model_params['pooling_method'],
model_params['conv1_kernel_size'],
model_params['conv1_num_kernels'],
model_params['conv1_stride'],
model_params['conv1_dropout'],
model_params['pool1_kernel_size'],
model_params['pool1_stride'],
model_params['conv2_kernel_size'],
model_params['conv2_num_kernels'],
model_params['conv2_stride'],
model_params['conv2_dropout'],
model_params['pool2_kernel_size'],
model_params['pool2_stride'],
model_params['fcs_hidden_size'],
model_params['fcs_num_hidden_layers'],
model_params['fcs_dropout'])
except Exception as e:
raise RuntimeError(
'{}.get_which_model_from_params_fname: unable to instantiate model class {} with model_params = {}\n. Encountered error: {}'
.format(__name__, model_class, model_params, e))
if return_params is True:
return model, model_params
return model
def copy(pObj):
return copy_copy(pObj)
def copy(self):
return copy_copy(self)
def gettargets(targets_data=SrcFile):
"""
Function to form list of targets with their parameters from SrcFile
Variables:
SrcFile - file object with text data in semi-CSV format.
Returns:
Updates global targetsList list with targets dictionaries details.
"""
#get targets list to probe from targets.txt
global DNS
global DNSSRV
global targetsList
global TARGETS
if TARGETS:
targetsSource = [i.replace(' ', '') for i in TARGETS.split(',')]
else:
try:
with open(targets_data, 'r') as f:
#readlines to temp list targetsSource:
targetsSource = f.read().splitlines()
except FileNotFoundError:
print('targets.txt file not found.')
targetsSource = [
'8.8.8.8, Google Public DNS', '8.8.4.4, Google Public DNS'
]
#Iterate over targetsSource lines and extract targets hosts and description:
for target in targetsSource:
targetTempDict = copy_copy(targetDict)
#skip comments:
if target.startswith('#'):
continue
#skip empty lines:
elif target.strip() == '':
continue
#check if splitChar (deafult char - ',' comma) in target, if so, try to extract additional parameters like IP, description and command:
elif splitChar in target:
targetTempDict['target'] = target.split(
splitChar)[0].strip() #get target name/ip
targetTempDict['Description'] = target.split(
splitChar)[1].strip() #get target description
try:
targetTempDict['commandList'] = target.split(
splitChar)[2].strip().strip('"').strip(
"'") #get target command
#FORM Command to run:
if PING == True: #if -P given, has to use ping command and override all commnds given in file
targetTempDict['commandList'] = copy_copy(
defaultCommand
) #assign defaultCommand list to command item
targetTempDict['commandList'][hostIndex] = targetTempDict[
'target'] #set {target} equal to target IP/name
targetTempDict['Command'] = (' ').join(
targetTempDict['commandList']
) #append command to description
elif TRACE == True: #if -T given, has to use traceroute command and override all commnds given in file
targetTempDict['commandList'] = copy_copy(
defaultCommand
) #assign defaultCommand list to command item
targetTempDict['commandList'][hostIndex] = targetTempDict[
'target'] #set {target} equal to target IP/name
targetTempDict['Command'] = (' ').join(
targetTempDict['commandList']
) #append command to description
elif USERCOMMAND != '': #if -C commnd given, have to use it
targetTempDict['commandList'] = copy_copy(
defaultCommand
) #assign defaultCommand list to command item
targetTempDict['commandList'][hostIndex] = targetTempDict[
'target'] #set {target} equal to target IP/name
targetTempDict['Command'] = (' ').join(
targetTempDict['commandList']
) #append command to description
elif targetTempDict[
'commandList'] != '': #check that command is not empty, if not - use it:
targetTempDict[
'commandList'] = targetTempDict['commandList'].replace(
'{target}', targetTempDict['target']
) #replace {target} in command with target IP/name
targetTempDict['Command'] = targetTempDict[
'commandList'] #add Command string to target
targetTempDict['commandList'] = targetTempDict[
'commandList'].split(
) #split command based on spaces to create list to run it with subprocess
elif targetTempDict[
'commandList'] == '': #in case if command is empty - use default command:
targetTempDict['commandList'] = copy_copy(
defaultCommand
) #assign defaultCommand list to command item
targetTempDict['commandList'][hostIndex] = targetTempDict[
'target'] #set {target} equal to target IP/name
targetTempDict['Command'] = (' ').join(
targetTempDict['commandList']
) #append command to description
except: #except occurs when no command give in line, hence target.split(splitChar)[2] will produce an error
targetTempDict['commandList'] = copy_copy(
defaultCommand
) #assign defaultCommand list to command item
targetTempDict['commandList'][hostIndex] = targetTempDict[
'target'] #set {target} equal to target IP/name
targetTempDict['Command'] = (' ').join(
targetTempDict['commandList']
) #append command to description
if targetTempDict['target'].count(
'/') == 1: #if we have / in target - means subnet given
targetsList += targets_from_subnet(
targetTempDict
) #extract and add IPs from subnet to targetsList
else:
targetsList.append(targetTempDict)
#else - no splitChar in target line, get target:
else:
targetTempDict['target'] = target.strip(
) #get target and clean it from spaces
targetTempDict['commandList'] = copy_copy(
defaultCommand) #sign probrparms list to command item
targetTempDict['commandList'][hostIndex] = targetTempDict[
'target'] # {target} equal to target IP/name
targetTempDict['Command'] = (' ').join(
targetTempDict['commandList']) #append command to description
if targetTempDict['target'].count(
'/') == 1: #if we have / in target - means subnet given
targetsList += targets_from_subnet(
targetTempDict
) #extract and add IPs from subnet to targetsList
else:
targetsList.append(targetTempDict)
#perform DNS resolution of targets if -D flag given:
if DNS == True or DNSSRV != False:
DNSthreads = []
#start threads to resolve names:
for target in targetsList:
DNSth = threading_Thread(target=DNSresolve,
kwargs=dict(target=target,
dnsServer=DNSSRV))
try:
DNSth.start()
DNSthreads.append(DNSth)
except KeyboardInterrupt:
closeFiles()
raise SystemExit('Exit: Interrupted by User')
except:
pass
#join threads and wait for them to comlete:
for DNSth in DNSthreads:
DNSth.join(timeout=30 * PROBETIMEOUT / 1000)
#create formatter for output formatting:
getFormatter()
#print all targets for the first time if SILENT mode not True:
if SILENT == False:
#clear screen if Windows, and delte 0 lines if Linux:
delete_last_lines(0)
#print targets to the screen:
printer()
#create log files and fill dictionary of log file name using index to make them unique:
for index, target in enumerate(targetsList):
file_name = target['target'].replace("/", "_").replace(":", "_")
target['logFile'] = open(logMainDir + logSubDirName + str(index + 1) +
'_' + logFileName.format(file_name),
'a',
buffering=1)
