def get_layers(kernel_width, num_kernels, num_layers): layers = [] for layer_index in range(num_layers): layer = copy_copy(LAYER_DICT_TEMPLATE) if layer_index == num_layers - 1: # for the last layer, the output needs to be layer["out_channels"] = 1 else: layer["out_channels"] = num_kernels if kernel_width == 5: layer['kernel_width'] = 5 layer['padding_width'] = 2 layer['stride_width'] = 1 elif kernel_width == 7: layer['kernel_width'] = 7 layer['padding_width'] = 3 layer['stride_width'] = 1 if layer_index == 0: layer['in_channels'] = 1 else: layer['in_channels'] = layers[-1]['out_channels'] layer['name'] = 'conv{}'.format(layer_index + 1) layers.append(layer) return layers
def startThreads(numberOfThreads): global targetsList global ctime global TRACE thread_counter = 0 probesReverseCounter = copy_copy(PROBECOUNT) while probesReverseCounter != 0: StartTime = time.time() #cycle start time in 1532321174.2756 format for target in targetsList: thread_counter += 1 # continue starting threads until maximum numberOfThreads reached # and length of targetsList smaller then count of threads already started if thread_counter != numberOfThreads and len( targetsList) != thread_counter: startThread(target) # wait for threads to complete by joining them else: startThread(target) thread_counter = 0 try: for th in threads: if TRACE: #if traceroute command then set timeout to 30 x probetimeout th.join(timeout=30 * PROBETIMEOUT / 1000) else: th.join(timeout=3 * PROBETIMEOUT / 1000) if SSILENT == False: reprinter() except KeyboardInterrupt: closeFiles() raise SystemExit('Exit: Interrupted by User') # join remaining threads and run reprinter try: for th in threads: if TRACE: #if traceroute command then set timeout to 30 x probetimeout th.join(timeout=30 * PROBETIMEOUT / 1000) else: th.join(timeout=3 * PROBETIMEOUT / 1000) if SSILENT == False: reprinter() else: # write final logs for all probes if end reached if probesReverseCounter == 1: reprinter() except KeyboardInterrupt: closeFiles() raise SystemExit('Exit: Interrupted by User') probesReverseCounter -= 1 #calculate time spent running above threads/probes, if spent les than PROBEINTERVAL than sleep time remaining: TimeElapsed = round(time.time() - StartTime, 4) if PROBEINTERVAL / 1000 > TimeElapsed: TimeToSleep = PROBEINTERVAL / 1000 - TimeElapsed try: time.sleep(TimeToSleep) except KeyboardInterrupt: closeFiles() raise SystemExit('Exit: Interrupted by User')
def __init__(self, seperator, limits, default, censor_char='2'): ConfigElement.__init__(self) self.marked_pos = 0 self.seperator = seperator self.limits = limits self.censor_char = censor_char self.last_value = self.default = default self.value = copy_copy(default) self.endNotifier = None return
def _broadcast_matrix(a, b, wrt, data): global _bs_setup_data1, _bs_setup_data2 if len(set((a.shape, b.shape))) == 1: uid = a.shape if uid not in _bs_setup_data1: asz = a.size IS = np.arange(asz) _bs_setup_data1[uid] = sp.csc_matrix((np.empty(asz), (IS, IS)), shape=(asz, asz)) result = copy_copy(_bs_setup_data1[uid]) if isinstance(data, np.ndarray): result.data = data.ravel() else: # assumed scalar result.data = np.empty(result.nnz) result.data.fill(data) else: uid = (a.shape, b.shape, wrt is a, wrt is b) if uid not in _bs_setup_data2: input_sz = wrt.size output_sz = np.broadcast(a.r, b.r).size a2 = np.arange(a.size).reshape(a.shape) if wrt is a else np.zeros( a.shape) b2 = np.arange(b.size).reshape( b.shape) if (wrt is b and wrt is not a) else np.zeros(b.shape) IS = np.arange(output_sz) JS = np.asarray((np.add(a2, b2)).ravel(), np.uint32) _bs_setup_data2[uid] = sp.csc_matrix( (np.arange(IS.size), (IS, JS)), shape=(output_sz, input_sz)) result = copy_copy(_bs_setup_data2[uid]) if isinstance(data, np.ndarray): result.data = data[result.data] else: # assumed scalar result.data = np.empty(result.nnz) result.data.fill(data) if np.prod(result.shape) == 1: return np.array(data) else: return result
def _broadcast_matrix(a, b, wrt, data): global _bs_setup_data1, _bs_setup_data2 if len(set((a.shape, b.shape))) == 1: uid = a.shape if uid not in _bs_setup_data1: asz = a.size IS = np.arange(asz) _bs_setup_data1[uid] = sp.csc_matrix((np.empty(asz), (IS, IS)), shape=(asz, asz)) result = copy_copy(_bs_setup_data1[uid]) if isinstance(data, np.ndarray): result.data = data.ravel() else: # assumed scalar result.data = np.empty(result.nnz) result.data.fill(data) else: uid = (a.shape, b.shape, wrt is a, wrt is b) if uid not in _bs_setup_data2: input_sz = wrt.size output_sz = np.broadcast(a.r, b.r).size a2 = np.arange(a.size).reshape(a.shape) if wrt is a else np.zeros(a.shape) b2 = np.arange(b.size).reshape(b.shape) if (wrt is b and wrt is not a) else np.zeros(b.shape) IS = np.arange(output_sz) JS = np.asarray((np.add(a2,b2)).ravel(), np.uint32) _bs_setup_data2[uid] = sp.csc_matrix((np.arange(IS.size), (IS, JS)), shape=(output_sz, input_sz)) result = copy_copy(_bs_setup_data2[uid]) if isinstance(data, np.ndarray): result.data = data[result.data] else: # assumed scalar result.data = np.empty(result.nnz) result.data.fill(data) if np.prod(result.shape) == 1: return np.array(data) else: return result
def __init__(self, seperator, limits, default, censor_char = ""): ConfigElement.__init__(self) assert isinstance(limits, list) and len(limits[0]) == 2, "limits must be [(min, max),...]-tuple-list" assert censor_char == "" or len(censor_char) == 1, "censor char must be a single char (or \"\")" #assert isinstance(default, list), "default must be a list" #assert isinstance(default[0], int), "list must contain numbers" #assert len(default) == len(limits), "length must match" self.marked_pos = 0 self.seperator = seperator self.limits = limits self.censor_char = censor_char self.last_value = self.default = default self.value = copy_copy(default) self.endNotifier = None
def targets_from_subnet(TD): #TD - target dictionary based on targetDictglob var result = [] if ":" in TD['target']: #create IPv6 subnet object subnetObj = ipaddress_IPv6Network(TD['target'], strict=False) elif "." in TD['target']: #create IPv4 subnet object: subnetObj = ipaddress_IPv4Network(TD['target'], strict=False) subnet_hosts = [str(i) for i in list(subnetObj.hosts())] #go over subnet hosts and copy targetDict parameters on them: for subnet_host in subnet_hosts: result.append({}) result[-1].update(TD) result[-1]['target'] = subnet_host result[-1]['Command'] = TD['Command'].replace(TD['target'], subnet_host) #have had to do below to prevent subnet_host becoming equal to last hos tin subnet: TD['commandList'][hostIndex] = subnet_host result[-1]['commandList'] = copy_copy(TD['commandList']) return result
def load_medium_file( filename: str, logger: Logger = getLogger(__name__)) -> pd.DataFrame: '''Load a file to fit in a dataframe supporting informations linked to a medium :param filename: Path of the file :type filename: str :return: A dataframe formatted :rtype: pd.DataFrame ''' df = pd.DataFrame() try: if not os_path.isfile(filename): raise FileNotFoundError('File %s not found' % (filename, )) # Find delimiter. with open(filename) as fid: dialect = csv.Sniffer().sniff(fid.readline()) # Load. df = pd.read_csv(filename, sep=dialect.delimiter) # Check header. header_required = copy_copy(__MEDIUM_HEADER) for header in __MEDIUM_HEADER_OPTIONAL: if header in header_required: header_required.remove(header) for header in header_required: if not header in df.columns: raise HeaderMalformated('Header is malformated') # Fmt. df[__MEDIUM_HEADER_BOUND] = df[__MEDIUM_HEADER_BOUND].astype(float) # Translate to rp_compound df = create_rp_compound(df, logger) except Exception as e: logger.error(str(e)) df = pd.DataFrame() return df
def onDeselect(self, session): if self.last_value != self._value: self.changedFinal() self.last_value = copy_copy(self._value)
def get_model_dict(kernel_width, num_kernels, num_layers): model_dict = copy_copy(MODEL_DICT_TEMPLATE) layers = get_layers(kernel_width, num_kernels, num_layers) model_dict['layers'] = layers return model_dict
def get_which_model_from_params_fname(model_params_fname, return_params=False): # load the model model_params = read_model_params(model_params_fname) # If the json is a list, it's a FlexNet if 'type' in model_params and model_params[ 'type'] == 'alexnet' and model_params['version'] == '1.0': from lib.flexnet import FlexNet model_class = FlexNet model = FlexNet(model_params_fname) if return_params is True: return model, model_params return model if 'model' not in model_params: # print('get_which_model_from_params_fname: using LeNet') from lib.lenet import LeNet # Circular dependency model_class = LeNet elif model_params['model'] == 'AlexNet': # print('get_which_model_from_params_fname: using AlexNet') from lib.alexnet import AlexNet model_class = AlexNet elif model_params['model'] == 'MLPB5': from lib.flexnet import FlexNet model_init_params = copy_copy(model_params) # Delete training parameters del model_init_params['batch_size'] del model_init_params['data_is_target'] del model_init_params['data_noise_gaussian'] del model_init_params['data_train'] del model_init_params['data_val'] del model_init_params['k'] del model_init_params['learning_rate'] del model_init_params['loss_function'] del model_init_params['model'] del model_init_params['momentum'] del model_init_params['optimizer'] del model_init_params['version'] del model_init_params['patience'] del model_init_params['weight_decay'] model = FlexNet(model_init_params) if return_params is True: return model, model_params return model elif model_params['model'] in ['FCN', 'FCNN']: from lib.flexnet import FlexNet model_init_params = copy_copy(model_params) # Delete training parameters del model_init_params['batch_size'] del model_init_params['data_is_target'] del model_init_params['data_noise_gaussian'] del model_init_params['data_train'] del model_init_params['data_val'] del model_init_params['k'] del model_init_params['learning_rate'] del model_init_params['loss_function'] del model_init_params['model'] del model_init_params['momentum'] del model_init_params['optimizer'] del model_init_params['version'] del model_init_params['patience'] del model_init_params['weight_decay'] model = FlexNet(model_init_params) if return_params is True: return model, model_params return model if 'input_channel' in model_params: input_channel = model_params['input_channel'] else: input_channel = 2 # By default, we used 2-channel (2*65) input if '2018' in model_params_fname: from lib.lenet_1d import LeNet_1D # Circular dependency model_class = LeNet_1D if 'type' in model_params and model_params['type'] == 'lenet_1d': from lib.lenet_1d import LeNet_1D # Circular dependency model_class = LeNet_1D try: model = model_class( input_channel, # model_params['input'], model_params['output_size'], model_params['batch_norm'], model_params['use_pooling'], model_params['pooling_method'], model_params['conv1_kernel_size'], model_params['conv1_num_kernels'], model_params['conv1_stride'], model_params['conv1_dropout'], model_params['pool1_kernel_size'], model_params['pool1_stride'], model_params['conv2_kernel_size'], model_params['conv2_num_kernels'], model_params['conv2_stride'], model_params['conv2_dropout'], model_params['pool2_kernel_size'], model_params['pool2_stride'], model_params['fcs_hidden_size'], model_params['fcs_num_hidden_layers'], model_params['fcs_dropout']) except Exception as e: raise RuntimeError( '{}.get_which_model_from_params_fname: unable to instantiate model class {} with model_params = {}\n. Encountered error: {}' .format(__name__, model_class, model_params, e)) if return_params is True: return model, model_params return model
def copy(pObj): return copy_copy(pObj)
def copy(self): return copy_copy(self)
def gettargets(targets_data=SrcFile): """ Function to form list of targets with their parameters from SrcFile Variables: SrcFile - file object with text data in semi-CSV format. Returns: Updates global targetsList list with targets dictionaries details. """ #get targets list to probe from targets.txt global DNS global DNSSRV global targetsList global TARGETS if TARGETS: targetsSource = [i.replace(' ', '') for i in TARGETS.split(',')] else: try: with open(targets_data, 'r') as f: #readlines to temp list targetsSource: targetsSource = f.read().splitlines() except FileNotFoundError: print('targets.txt file not found.') targetsSource = [ '8.8.8.8, Google Public DNS', '8.8.4.4, Google Public DNS' ] #Iterate over targetsSource lines and extract targets hosts and description: for target in targetsSource: targetTempDict = copy_copy(targetDict) #skip comments: if target.startswith('#'): continue #skip empty lines: elif target.strip() == '': continue #check if splitChar (deafult char - ',' comma) in target, if so, try to extract additional parameters like IP, description and command: elif splitChar in target: targetTempDict['target'] = target.split( splitChar)[0].strip() #get target name/ip targetTempDict['Description'] = target.split( splitChar)[1].strip() #get target description try: targetTempDict['commandList'] = target.split( splitChar)[2].strip().strip('"').strip( "'") #get target command #FORM Command to run: if PING == True: #if -P given, has to use ping command and override all commnds given in file targetTempDict['commandList'] = copy_copy( defaultCommand ) #assign defaultCommand list to command item targetTempDict['commandList'][hostIndex] = targetTempDict[ 'target'] #set {target} equal to target IP/name targetTempDict['Command'] = (' ').join( targetTempDict['commandList'] ) #append command to description elif TRACE == True: #if -T given, has to use traceroute command and override all commnds given in file targetTempDict['commandList'] = copy_copy( defaultCommand ) #assign defaultCommand list to command item targetTempDict['commandList'][hostIndex] = targetTempDict[ 'target'] #set {target} equal to target IP/name targetTempDict['Command'] = (' ').join( targetTempDict['commandList'] ) #append command to description elif USERCOMMAND != '': #if -C commnd given, have to use it targetTempDict['commandList'] = copy_copy( defaultCommand ) #assign defaultCommand list to command item targetTempDict['commandList'][hostIndex] = targetTempDict[ 'target'] #set {target} equal to target IP/name targetTempDict['Command'] = (' ').join( targetTempDict['commandList'] ) #append command to description elif targetTempDict[ 'commandList'] != '': #check that command is not empty, if not - use it: targetTempDict[ 'commandList'] = targetTempDict['commandList'].replace( '{target}', targetTempDict['target'] ) #replace {target} in command with target IP/name targetTempDict['Command'] = targetTempDict[ 'commandList'] #add Command string to target targetTempDict['commandList'] = targetTempDict[ 'commandList'].split( ) #split command based on spaces to create list to run it with subprocess elif targetTempDict[ 'commandList'] == '': #in case if command is empty - use default command: targetTempDict['commandList'] = copy_copy( defaultCommand ) #assign defaultCommand list to command item targetTempDict['commandList'][hostIndex] = targetTempDict[ 'target'] #set {target} equal to target IP/name targetTempDict['Command'] = (' ').join( targetTempDict['commandList'] ) #append command to description except: #except occurs when no command give in line, hence target.split(splitChar)[2] will produce an error targetTempDict['commandList'] = copy_copy( defaultCommand ) #assign defaultCommand list to command item targetTempDict['commandList'][hostIndex] = targetTempDict[ 'target'] #set {target} equal to target IP/name targetTempDict['Command'] = (' ').join( targetTempDict['commandList'] ) #append command to description if targetTempDict['target'].count( '/') == 1: #if we have / in target - means subnet given targetsList += targets_from_subnet( targetTempDict ) #extract and add IPs from subnet to targetsList else: targetsList.append(targetTempDict) #else - no splitChar in target line, get target: else: targetTempDict['target'] = target.strip( ) #get target and clean it from spaces targetTempDict['commandList'] = copy_copy( defaultCommand) #sign probrparms list to command item targetTempDict['commandList'][hostIndex] = targetTempDict[ 'target'] # {target} equal to target IP/name targetTempDict['Command'] = (' ').join( targetTempDict['commandList']) #append command to description if targetTempDict['target'].count( '/') == 1: #if we have / in target - means subnet given targetsList += targets_from_subnet( targetTempDict ) #extract and add IPs from subnet to targetsList else: targetsList.append(targetTempDict) #perform DNS resolution of targets if -D flag given: if DNS == True or DNSSRV != False: DNSthreads = [] #start threads to resolve names: for target in targetsList: DNSth = threading_Thread(target=DNSresolve, kwargs=dict(target=target, dnsServer=DNSSRV)) try: DNSth.start() DNSthreads.append(DNSth) except KeyboardInterrupt: closeFiles() raise SystemExit('Exit: Interrupted by User') except: pass #join threads and wait for them to comlete: for DNSth in DNSthreads: DNSth.join(timeout=30 * PROBETIMEOUT / 1000) #create formatter for output formatting: getFormatter() #print all targets for the first time if SILENT mode not True: if SILENT == False: #clear screen if Windows, and delte 0 lines if Linux: delete_last_lines(0) #print targets to the screen: printer() #create log files and fill dictionary of log file name using index to make them unique: for index, target in enumerate(targetsList): file_name = target['target'].replace("/", "_").replace(":", "_") target['logFile'] = open(logMainDir + logSubDirName + str(index + 1) + '_' + logFileName.format(file_name), 'a', buffering=1)