def test(): # 开始计时 start = time.clock() test_date = time.time() data_path = './data/processed/prt_mfd_7.csv' save_path = './test/purchase/' col_start = 1 col_end = 4 train_begin = 0 train_end = 397 time_step = 15 rnn_unit = 10 lr = 0.0006 run_times = 100 # --------------------------------------------- if not os.path.exists(save_path): os.makedirs(save_path) # --------------------------------------------- model = Model(save_path, test_date, data_path, col_start, col_end, train_begin, train_end, time_step, rnn_unit, lr, run_times) test_y, test_predict = model.get_test_result() # ------------------------------- end = time.clock() spend = end - start utils = Utils() test_score = utils.get_score(test_y, test_predict) describe = 'purchase(1308_1408,[purchase,redeem,mfd_7])' utils.save_test(test_y, test_predict, save_path, test_date, lr, run_times, time_step, rnn_unit, spend, test_score, describe)
def view_login(): utils = Utils() msg = '' if request.method == 'POST': username = request.form.get('username', None) password = request.form.get('password', None) if rds.is_ip_blocked(request.remote_addr): return render_template('login.html', err='Your IP has been blocked.') if verify_password(username, password): session['session'] = username return redirect('/') else: return render_template('login.html', err='Incorrect username or password. \ After 5 attempts, you will get blocked.' ) if not utils.is_version_latest(): msg = 'New Version is Available' return render_template('login.html', msg=msg)
def __init__(self): self.nmap = nmap.PortScanner() self.nmap_args = { 'unpriv_scan': '-sV -sT -n --max-retries 10 --host-timeout 60m', 'priv_scan': '-sV -O -sT -n --max-retries 10 --host-timeout 60m' } self.utils = Utils()
def __init__(self): self.nmap = nmap.PortScanner() self.nmap_args = { # 'unpriv_scan':'-PS --min-rate 5000 --max-retries 1 --open --host-timeout 1m', 'priv_scan':'-n -PR -PO -PS -PA -PP -PM -PE --min-rate 5000 --max-retries 1 --open --host-timeout 1m' } self.utils = Utils()
def commandprocessor(self, cmd): # Quit CMD if (cmd == "q") or (cmd == "quit") or (cmd == "exit"): return True # Help CMD elif (cmd == "h") or (cmd == "help"): self.helpprint() # Create CMD elif (cmd == "c") or (cmd == "create"): self.createblockchain() # Show cmd elif (cmd == "s") or (cmd == "show"): self.blockchain.showblockchain() # Add block cmd elif (cmd == "ab") or (cmd == "addblock"): self.addblock(self.blockchain.getblocknumber()) # Show block cmd elif (cmd == "sb") or (cmd == "showblock"): self.blockchain.showblock(int(input("Block number: "))) # Invalid CMD else: linestoprint = [ "", '"' + cmd + '" not found.', "", 'type "help" for a list of commands.', "", ] Utils().printlines(linestoprint, False)
def run(self): job_file = os.path.join(JOBS_CONFIG_DIR, str(self.jobfile)) job_conf = Utils.read_file(job_file) path = self.input().path if Box(json.loads(job_conf)).gzip: Utils().gzip(self.input().path) path += '.gzip' self.output().put(path, atomic=False)
def __init__(self): self.utils = Utils() self.r = None try: self.conn_pool = redis.ConnectionPool(host=config.RDS_HOST, port=config.RDS_PORT, db=0) self.r = redis.Redis(connection_pool=self.conn_pool) except TimeoutError: logger.error('Redis Connection Timed Out!') sys.exit(1)
def visualize(self): toprint = [ "Index: " + str(self.index), "Message: " + str(self.message), "Timestamp: " + str(self.timestamp), "Previous Hash: " + str(self.previoushash), "Hash: " + str(self.currenthash), "Nonce: " + str(self.nonce), ] Utils().printlines(toprint, False)
def __init__(self, **kwargs): self.app = None self.pid = None self.handle = None self.kwargs = kwargs self.main_window = None self.logger = kwargs.get("logger") self.utils = Utils(logger=self.logger) self.settings = self.utils.load_json("./settings.json") self.executable_path = self.settings["executable_path"] self.process_name = self.settings["process_name"]
def createblockchain(self): if self.blockchain: linestoprint = [ "", "Blockchain already created", "", ] Utils().printlines(linestoprint, False) else: difficulty = input("Difficulty: ") self.blockchain = Blockchain(difficulty) self.addblock(self.blockchain.getblocknumber())
def get(self, component=None): utils = Utils() if not component: return {'status':'Component is missing'}, 400 if component == 'platform': if not utils.is_version_latest: return {'status':'updates are available'} else: return {'status':'system is up to date'} return {'status':'unsupported action'}, 400
def compile_target(self, compilation_commands) -> bool: """ Generates preprocessed files. :return: True """ for curr_command in compilation_commands: self.logger.debug("[*] Initialising the environment " + curr_command[1]) Utils(curr_command[1]).run_cmd( f"{' '.join(curr_command[0])} > {curr_command[3]}", doexit=True) return True
def __init__(self, data, req): self.data = data self.utils = Utils() self.netutils = Network() self.metadata = { 'unique_id' : self.utils.generate_uuid(), 'timestamp' : self.utils.get_datetime(), 'node': gethostname(), 'issuer':{ 'user_agent':req.headers.get('User-Agent'), 'source_ip':req.remote_addr } } self.data['metadata'] = self.metadata
def helpprint(self): helplist = [ "", "create <or> c - Creates a blockchain", "help <or> h <or> ? - Shows this page", "show <or> s - Shows the blockchain", "showblock <or> sb - Shows a block", "addblock <or> ab - Adds a new block", "minechain <or> mc - Mines the chain", "mine <or> m - Mines a block", "quit <or> q <or> exit - Exits this app", "", ] Utils().printlines(helplist, False)
def __init__(self): disp_no = os.getenv("DISPLAY") if disp_no: logger.debug("I'm running under X display = {0}".format(disp_no)) # Check which frame buffer drivers are available # Start with fbcon since directfb hangs with composite output drivers = ['fbcon', 'directfb', 'svgalib'] found = False pygame.init() pygame.display.init() ''' for driver in drivers: # Make sure that SDL_VIDEODRIVER is set if not os.getenv('SDL_VIDEODRIVER'): os.putenv('SDL_VIDEODRIVER', driver) try: pygame.display.init() except pygame.error: logger.debug('Driver: {0} failed.'.format(driver)) continue found = True break ''' pygame.font.init() pygame.mouse.set_visible(0) #hide mouse self.screen = pygame.display.set_mode((WIDTH, HEIGHT)) #self.screen = pygame.display.set_mode((WIDTH, HEIGHT), pygame.FULLSCREEN) pygame.display.set_caption('Menu') self.clock = pygame.time.Clock() self.running = True self.loadAssets() logger.debug("launch joystics init...") utils = Utils() utils.initJoysticks() #disable mouse pygame.event.set_blocked(pygame.MOUSEMOTION) #movements and keys init self.downPushed = False self.upPushed = False self.leftPushed = False self.rightPushed = False self.joyUp = False self.joyDown = False self.joyLeft = False self.joyRight = False self.zPressed = False self.tPressed = False self.screensaver = False
def urls(conf): url = Box(json.loads(conf)).url html = requests.get(Box(json.loads(conf)).url).text soup = BeautifulSoup(html, 'lxml') container = Box(json.loads(conf)).storage.html.container_tag attrs = Box(json.loads(conf)).storage.html.container_attrs href_regex = Box(json.loads(conf)).storage.html.url_regexp urls = soup.find(container, attrs=attrs).find_all("a", href=re.compile(href_regex)) urls = [url.get('href') for url in urls] return [ u if u.startswith("http") else Utils().base_url(url) + u for u in urls ]
def introprint(self, version): toprint = [ "", "*****************", "* *", "* LEARN BLOCK *", "* *", "*****************", "Version: " + version, "", "Let's make a blockchain!", "", "(Type help to see a list of cmds)", "", ] Utils().printlines(toprint, True)
def view_settings(): utils = Utils() email_settings = rds.get_email_settings() slack_settings = rds.get_slack_settings() if request.method == 'POST': u_settings = request.get_json() if u_settings and isinstance(u_settings, dict): if 'email' in u_settings: msg, code = send_email(u_settings['email']) elif 'slack' in u_settings: hook = u_settings['slack'].get('hook', None) if utils.is_string_url(hook): rds.store('p_settings_slack', hook) code, msg = 200, 'Saved Slack Setting' else: code, msg = 400, 'Slack hook must be a URL' else: code, msg = 400, 'Error Occurred' return {'status': msg}, code elif request.method == 'DELETE': u_settings = request.get_json() settings = u_settings.get('settings', None) if settings == 'email': rds.delete('p_settings_email') code, msg = 200, 'Deleted Email Settings' elif settings == 'slack': rds.delete('p_settings_slack') code, msg = 200, 'Deleted Slack Settings' else: code, msg = 400, 'Error Occurred' return {'status': msg}, code return render_template('settings.html', email=email_settings, slack=slack_settings)
def add_file_to_code_window(self, dialog): self.current_file_loc = dialog.get_filename() if self.utils is None: self.utils = Utils(self.current_file_loc, self) else: self.utils.set_data(self.current_file_loc) if self.utils.extension != Constants.ASM: self.show_error("File format should be .asm") return file = open(self.current_file_loc, "r") if self.file_opened is True: self.code.removeTags(self.tags) self.code.set_text_from_file(file) self.file_opened = True self.init_tags() self.utils.remove_all_breakpoints() self.clean() file.close()
def scanner(): utils = Utils() scanner = Scanner() logger.info('Scanner process started') while True: if not rds.is_session_active(): time.sleep(10) continue conf = rds.get_scan_config() if not conf: time.sleep(10) continue hosts = rds.get_ips_to_scan( limit=conf['config']['scan_opts']['parallel_scan']) if hosts: conf = rds.get_scan_config() scan_data = scanner.scan( hosts, max_ports=conf['config']['scan_opts']['max_ports'], custom_ports=conf['config']['scan_opts']['custom_ports'], interface=conf['config']['scan_opts']['interface']) if scan_data: for host, values in scan_data.items(): if 'ports' in values and values['ports']: logger.info('Discovered Asset: {}'.format(host)) logger.debug('Host: {}, Open Ports: {}'.format( host, values['ports'])) rds.store_topology(host) rds.store_sca(host, values) rds.store_inv(host, values) else: if values['status_reason'] == 'echo-reply': logger.info('Discovered Asset: {}'.format(host)) rds.store_topology(host)
def byeprint(self): byelist = ["", "Bye!", ""] Utils().printlines(byelist, False)
def predict(argv=None): ''' generate image from conditional Gated PixelCNN model Usage: python predict_keras_datasets.py -c sample_predict.cfg : prediction example using configfile python predict_keras_datasets.py --option1 hoge ... : predict with command-line options python predict_keras_datasets.py -c predict.cfg --opt1 hoge... : overwrite config options with command-line options ''' ### parsing arguments from command-line or config-file ### if argv is None: argv = sys.argv conf_parser = argparse.ArgumentParser( description=__doc__, # printed with -h/--help formatter_class=argparse.RawDescriptionHelpFormatter, add_help=False) conf_parser.add_argument("-c", "--conf_file", help="Specify config file", metavar="FILE_PATH") args, remaining_argv = conf_parser.parse_known_args() defaults = {} if args.conf_file: config = configparser.SafeConfigParser() config.read([args.conf_file]) defaults.update(dict(config.items("General"))) parser = argparse.ArgumentParser(parents=[conf_parser]) parser.set_defaults(**defaults) parser.add_argument("--checkpoint_file", help="Checkpoint file [Required]", type=str, metavar="FILE_PATH") parser.add_argument( "--conditional", help="model the conditional distribution p(x|h) (default:False)", type=str, metavar="BOOL") parser.add_argument("--dataset_name", help="{'mnist','cifar10','cifar100'}", type=str, metavar="DATASET_NAME") parser.add_argument("--class_label", help="Class label (default: 0)", type=int, metavar="INT") parser.add_argument("--nb_images", help="Number of images to generate", type=int, metavar="INT") parser.add_argument("--batch_size", help="Batch size at prediction", type=int, metavar="INT") parser.add_argument( "--temperature", help="Temparature value for sampling diverse values (default: 1.0)", type=float, metavar="FLOAT") parser.add_argument( "--nb_pixelcnn_layers", help="Number of PixelCNN Layers (except last two ReLu layers)", type=int, metavar="INT") parser.add_argument("--nb_filters", help="Number of filters for each layer", type=int, metavar="INT") parser.add_argument( "--filter_size_1st", help="Filter size for the first layer. (default: (7,7))", metavar="INT,INT") parser.add_argument( "--filter_size", help="Filter size for the subsequent layers. (default: (3,3))", metavar="INT,INT") parser.add_argument( "--save_path", help= "Root directory which generated images are saved (default: /tmp/pixelcnn/results)", type=str, metavar="DIR_PATH") args = parser.parse_args(remaining_argv) utils = Utils() try: checkpoint_file = args.checkpoint_file dataset_name = args.dataset_name except ValueError: sys.exit("Error: --checkpoint_file must be specified.") conditional = strtobool(args.conditional) if args.conditional else False temperature = args.temperature if args.temperature else 1.0 ### mnist image size ### if dataset_name == 'mnist': input_size = (28, 28) nb_classes = 10 nb_channels = 1 elif dataset_name == 'cifar10': input_size = (32, 32) nb_classes = 10 nb_channels = 3 elif dataset_name == 'cifar100': input_size = (32, 32) nb_classes = 100 nb_channels = 3 ### build PixelCNN model ### model_params = {} model_params['input_size'] = input_size model_params['nb_channels'] = nb_channels model_params['conditional'] = conditional model_params['latent_dim'] = nb_classes if args.nb_pixelcnn_layers: model_params['nb_pixelcnn_layers'] = int(args.nb_pixelcnn_layers) if args.nb_filters: model_params['nb_filters'] = int(args.nb_filters) if args.filter_size_1st: model_params['filter_size_1st'] = tuple( map(int, args.filter_size_1st.split(','))) if args.filter_size: model_params['filter_size'] = tuple( map(int, args.filter_size.split(','))) save_path = args.save_path if args.save_path else '/tmp/pixelcnn/results' if not os.path.exists(save_path): os.makedirs(save_path) pixelcnn = PixelCNN(**model_params) pixelcnn.build_model() pixelcnn.model.load_weights(checkpoint_file) ## prepare zeros array class_label = int(args.class_label) if args.class_label else 0 nb_images = int(args.nb_images) if args.nb_images else 8 batch_size = int(args.batch_size) if args.batch_size else nb_images if dataset_name == 'mnist': X_pred = np.zeros((nb_images, input_size[0], input_size[1], 1)) else: X_pred = np.zeros((nb_images, input_size[0], input_size[1], 3)) if conditional: h_pred = np_utils.to_categorical(class_label, nb_classes) h_pred = np.repeat(h_pred, nb_images, axis=0) ### generate images pixel by pixel for i in range(input_size[0]): for j in range(input_size[1]): for k in range(nb_channels): if conditional: x = [X_pred, h_pred] else: x = X_pred next_X_pred = pixelcnn.model.predict(x, batch_size) if dataset_name == 'mnist': binarizer = lambda x: utils.binarize_val(x) binarized_pred = np.vectorize(binarizer)(next_X_pred[:, i, j, k]) X_pred[:, i, j, k] = binarized_pred else: sampled_pred = next_X_pred[:, i * input_size[1] * nb_channels + j * nb_channels + k, :] sampled_pred = np.array([ utils.sample(sampled_pred[i]) for i in range(len(sampled_pred)) ]) X_pred[:, i, j, k] = sampled_pred X_pred = np.squeeze(X_pred) X_pred = (255 * X_pred).astype(np.uint8) ### save images ### for i in range(nb_images): utils.save_generated_image(X_pred[i], 'generated_' + str(i) + '.jpg', save_path) return (0)
def train(argv=None): ''' train conditional Gated PixelCNN model Usage: python train_mnist.py -c sample_train.cfg : training example using configfile python train_mnist.py --option1 hoge ... : train with command-line options python train_mnist.py -c test.cfg --opt1 hoge... : overwrite config options with command-line options ''' ### parsing arguments from command-line or config-file ### if argv is None: argv = sys.argv conf_parser = argparse.ArgumentParser( description=__doc__, # printed with -h/--help formatter_class=argparse.RawDescriptionHelpFormatter, add_help=False) conf_parser.add_argument("-c", "--conf_file", help="Specify config file", metavar="FILE_PATH") args, remaining_argv = conf_parser.parse_known_args() defaults = {} if args.conf_file: config = configparser.SafeConfigParser() config.read([args.conf_file]) defaults.update(dict(config.items("General"))) parser = argparse.ArgumentParser(parents=[conf_parser]) parser.set_defaults(**defaults) parser.add_argument("--nb_epoch", help="Number of epochs [Required]", type=int, metavar="INT") parser.add_argument("--batch_size", help="Minibatch size", type=int, metavar="INT") parser.add_argument( "--conditional", help="model the conditional distribution p(x|h) (default:False)", type=str, metavar="BOOL") parser.add_argument( "--nb_pixelcnn_layers", help="Number of PixelCNN Layers (exept last two ReLu layers)", metavar="INT") parser.add_argument("--nb_filters", help="Number of filters for each layer", metavar="INT") parser.add_argument( "--filter_size_1st", help="Filter size for the first layer. (default: (7,7))", metavar="INT,INT") parser.add_argument( "--filter_size", help="Filter size for the subsequent layers. (default: (3,3))", metavar="INT,INT") parser.add_argument("--optimizer", help="SGD optimizer (default: adadelta)", type=str, metavar="OPT_NAME") parser.add_argument("--es_patience", help="Patience parameter for EarlyStopping", type=int, metavar="INT") parser.add_argument( "--save_root", help= "Root directory which trained files are saved (default: /tmp/pixelcnn)", type=str, metavar="DIR_PATH") parser.add_argument( "--timezone", help= "Trained files are saved in save_root/YYYYMMDDHHMMSS/ (default: Asia/Tokyo)", type=str, metavar="REGION_NAME") parser.add_argument( "--save_best_only", help="The latest best model will not be overwritten (default: False)", type=str, metavar="BOOL") parser.add_argument("--plot_model", help="If True, plot a Keras model (using graphviz)", type=str, metavar="BOOL") args = parser.parse_args(remaining_argv) conditional = strtobool(args.conditional) if args.conditional else False ### mnist image size ### input_size = (28, 28) nb_classes = 10 utils = Utils() ### load mnist dataset ### ### if conditional == True, X_train = [X_train, h_train] ### (X_train, Y_train), (X_validation, Y_validation) = utils.load_mnist_dataset(conditional) ### build PixelCNN model ### model_params = {} model_params['input_size'] = input_size model_params['nb_channels'] = 1 model_params['conditional'] = conditional if conditional: model_params['latent_dim'] = nb_classes if args.nb_pixelcnn_layers: model_params['nb_pixelcnn_layers'] = int(args.nb_pixelcnn_layers) if args.nb_filters: model_params['nb_filters'] = int(args.nb_filters) if args.filter_size_1st: model_params['filter_size_1st'] = tuple( map(int, args.filter_size_1st.split(','))) if args.filter_size: model_params['filter_size'] = tuple( map(int, args.filter_size.split(','))) if args.optimizer: model_params['optimizer'] = args.optimizer if args.es_patience: model_params['es_patience'] = int(args.patience) if args.save_best_only: model_params['save_best_only'] = strtobool(args.save_best_only) save_root = args.save_root if args.save_root else '/tmp/pixelcnn_mnist' timezone = args.timezone if args.timezone else 'Asia/Tokyo' current_datetime = datetime.now( pytz.timezone(timezone)).strftime('%Y%m%d_%H%M%S') save_root = os.path.join(save_root, current_datetime) model_params['save_root'] = save_root if not os.path.exists(save_root): os.makedirs(save_root) try: nb_epoch = int(args.nb_epoch) batch_size = int(args.batch_size) except: sys.exit("Error: {--nb_epoch, --batch_size} must be specified.") pixelcnn = PixelCNN(**model_params) pixelcnn.build_model() pixelcnn.model.summary() pixelcnn.print_train_parameters(save_root) pixelcnn.export_train_parameters(save_root) with open(os.path.join(save_root, 'parameters.txt'), 'a') as txt_file: txt_file.write('########## other options ##########\n') txt_file.write('nb_epoch\t: %s\n' % nb_epoch) txt_file.write('batch_size\t: %s\n' % batch_size) txt_file.write('\n') plot_model = strtobool(args.plot_model) if args.plot_model else True if plot_model: plot(pixelcnn.model, to_file=os.path.join(save_root, 'pixelcnn_mnist_model.png')) train_params = {} train_params['x'] = X_train train_params['y'] = Y_train train_params['validation_data'] = (X_validation, Y_validation) train_params['nb_epoch'] = nb_epoch train_params['batch_size'] = batch_size train_params['shuffle'] = True pixelcnn.fit(**train_params) return (0)
def __init__(self, driver, logger, url): self.driver = driver self.logger = logger self.url = url self.utils = Utils(self.driver, self.logger)
def send_email(settings, data=None): utils = Utils() keys = ('host', 'port', 'user', 'pass', 'to_addr', 'from_addr', 'ssl_type', 'action') if not all(elem in settings for elem in keys): return ('Error, missing settings', 400) if not settings['host'] or not settings['port']: return ('SMTP address or SMTP port are empty', 400) if not isinstance(settings['port'], int): return ('SMTP Port must be a number', 400) if not settings['from_addr'] or not settings['to_addr']: return ('FROM or TO Address are empty', 400) if not utils.is_string_email(settings['from_addr']) or \ not utils.is_string_email(settings['to_addr']): return ('FROM or TO addresses are not valid emails', 400) if settings['ssl_type'] not in ('starttls', 'ssl'): return ('Error in security settings (must be starttls or ssl).', 400) if settings['action'] not in ('save', 'test', 'send'): return ('Error, action is not supported', 400) msg = MIMEMultipart('alternative') subject = '' if settings['action'] == 'test': subject = 'Test by NERVE' part = MIMEText('This is a test.', 'plain') msg.attach(part) elif settings['action'] == 'send': subject = 'Assessment Complete' part = MIMEText(str(data), 'plain') part.add_header('Content-Disposition', 'attachment', filename='assessment.json') msg.attach(part) elif settings['action'] == 'save': rds.store_json('p_settings_email', settings) return ('OK, Saved.', 200) context = ssl.create_default_context() msg['From'] = formataddr((str(Header('NERVE Security', 'utf-8')), settings['from_addr'])) msg['To'] = settings['to_addr'] msg['Subject'] = subject try: if settings['ssl_type'] == 'ssl': # ssl server = smtplib.SMTP_SSL(settings['host'], settings['port'], context=context) else: # starttls server = smtplib.SMTP(settings['host'], settings['port']) server.starttls(context=context) server.login(settings['user'], settings['pass']) server.sendmail(settings['from_addr'], settings['to_addr'], msg.as_string()) server.quit() return ('Message was sent successfully', 200) except Exception as e: return ('Message was could not be sent {}'.format(e), 500)
def __init__(self): self.rds = rds self.utils = Utils()
def __init__(self): super(UserService, self).__init__() self.collection = "users" self.blacklist = BlacklistHelper() self.utils = Utils() self.mongo = MongoDB()
def __init__(self, arg): super(UserLogin, self).__init__(arg) self.jwt_service = JWTService() self.utils = Utils() self.user_service = UserService()
generate_txt ) from bin.scanner import scanner from bin.attacker import attacker from bin.scheduler import scheduler from version import VERSION auth = HTTPBasicAuth() app = Flask(__name__) app.config.update( SESSION_COOKIE_SAMESITE='Strict', ) app.secret_key = os.urandom(24) api = Api(app) register = Register() utils = Utils() def session_required(function_to_protect): @wraps(function_to_protect) def wrapper(*args, **kwargs): if not session.get('session'): return redirect('/login', 307) return function_to_protect(*args, **kwargs) return wrapper def run_workers(): thread = threading.Thread(target=scanner) thread.name = "scanner" thread.daemon = True thread.start()
def __init__(self, arg): super(NewBook, self).__init__(arg) self.utils = Utils() self.book_service = BookService()