def main(args): # Parse config file config = parse_config(args.config_file) # verify the config file and get the Carbon Black Cloud Server list output_params, server_list = verify_config(config) # Store Forward. Attempt to send messages that have been saved due to a failure to reach the destination send_stored_data(output_params) logger.info("Found {0} Carbon Black Cloud Servers in config file".format( len(server_list))) # Iterate through our Carbon Black Cloud Server list for server in server_list: logger.info("Handling notifications for {0}".format( server.get('server_url'))) notification_logs = fetch_notification_logs( server, output_params['output_format'], output_params['policy_action_severity']) logger.info("Sending Notifications") send_new_data(output_params, notification_logs) logger.info("Done Sending Notifications") audit_logs = fetch_audit_logs(server, output_params['output_format']) logger.info("Sending Audit Logs") send_new_data(output_params, audit_logs) logger.info("Done Sending Audit Logs")
def main(): # bert 参数初始化 config = parse_config('BERT') # Parse arguments and print them args = parse_args() print("\nMain arguments:") for k, v in args.__dict__.items(): print("{}={}".format(k, v)) # Generate bert model ckt path and warm start path warm_start_path = None if args.load_bert_ckt: warm_start_path = args.buckets + args.load_bert_ckt + "/model.ckpt-{}".format( args.load_bert_step) warm_start_settings = tf.estimator.WarmStartSettings( warm_start_path, vars_to_warm_start='bert*') elif args.load_all_layers_ckt: warm_start_path = args.buckets + args.load_all_layers_ckt + "/model.ckpt-{}".format( args.load_all_step) warm_start_settings = tf.estimator.WarmStartSettings( warm_start_path, vars_to_warm_start=".*") else: raise ValueError("No pretain params for finetune models") # Check if the model has already exisited model_save_dir = args.buckets + args.checkpoint_dir warm_start_dir = None # bert.* if tf.gfile.Exists(model_save_dir + "/checkpoint" ) and args.load_all_layers_ckt != args.checkpoint_dir: raise ValueError( "Model %s has already existed, please delete them and retry" % model_save_dir) helper.dump_args(model_save_dir, args) bert_model = BertFinetune(config) estimator = tf.estimator.Estimator( model_fn=bert_model.model_fn, model_dir=model_save_dir, config=tf.estimator.RunConfig(session_config=tf.ConfigProto( gpu_options=tf.GPUOptions(allow_growth=True)), save_checkpoints_steps=args.snap_shot, keep_checkpoint_max=100), warm_start_from=warm_start_settings) print("Start training......") estimator.train( finetune_loader.OdpsDataLoader(table_name=args.tables, config=config, mode=1).input_fn, steps=config["num_train_steps"], )
def main(): # bert 参数初始化 config = parse_config('BERT') args = parse_args() print("Main arguments:") for k, v in args.__dict__.items(): print("{}={}".format(k, v)) # Setup distributed inference dist_params = { "task_index": args.task_index, "ps_hosts": args.ps_hosts, "worker_hosts": args.worker_hosts, "job_name": args.job_name } slice_count, slice_id = env.set_dist_env(dist_params) bert_model = BertFinetune(config) # Load model arguments model_save_dir = args.buckets + args.checkpoint_dir checkpoint_path = None if args.step > 0: checkpoint_path = model_save_dir + "/model.ckpt-{}".format(args.step) estimator = tf.estimator.Estimator( model_fn=bert_model.model_fn, model_dir=model_save_dir, config=tf.estimator.RunConfig( session_config=tf.ConfigProto(gpu_options=tf.GPUOptions( allow_growth=True)), save_checkpoints_steps=config["num_train_steps"], keep_checkpoint_max=1)) result_iter = estimator.predict(finetune_loader.OdpsDataLoader( table_name=args.tables, config=config, mode=0, slice_id=slice_id, slice_count=slice_count).input_fn, checkpoint_path=checkpoint_path) odps_writer = dumper.get_odps_writer(args.outputs, slice_id=slice_id) _do_prediction(result_iter, odps_writer, args)
def main(): # bert 参数初始化 config = parse_config('MiniBERT') # Parse arguments and print them args = parse_args() print("\nMain arguments:") for k, v in args.__dict__.items(): print("{}={}".format(k, v)) # Check if the model has already exisited model_save_dir = args.buckets + args.checkpoint_dir warm_start_settings = None if tf.gfile.Exists(model_save_dir + "/checkpoint") and not args.warm_start_step: raise ValueError( "Model %s has already existed, please delete them and retry" % model_save_dir) elif args.warm_start_step: warm_start_path = model_save_dir + "/model.ckpt-{}".format( args.warm_start_step) warm_start_settings = tf.estimator.WarmStartSettings(warm_start_path) print("Model init training from %s" % warm_start_path) else: pass helper.dump_args(model_save_dir, args) bert_model = BertPretrain(config) estimator = tf.estimator.Estimator( model_fn=bert_model.model_fn, model_dir=model_save_dir, config=tf.estimator.RunConfig(session_config=tf.ConfigProto( gpu_options=tf.GPUOptions(allow_growth=True)), save_checkpoints_steps=args.snap_shot, keep_checkpoint_max=100), warm_start_from=warm_start_settings) print("Start training......") estimator.train( pretrain_loader.OdpsDataLoader(table_name=args.tables, config=config, mode=1).input_fn, steps=config["num_train_steps"], )
def main(): parser = ArgumentParser("Research Experiment Runner") parser.add_argument("config", metavar="config_json", help="Experiment configuration JSON file") parser.add_argument( "--override", metavar="override_json", default=None, type=str, help= "Serialized JSON object to merge into configuration (overrides config)", ) args = parser.parse_args() config = parse_config(args.config, args.override) agent_query = config.get("agent", None) agent_class = fetch_class(agent_query) agent_instance = agent_class(config) try: agent_instance.run() finally: agent_instance.finalize()
# # t = T() # for _ in range(3): # env.reset(t.init_func) # for _0 in range(25): # _1, _2, over, _4, _5, _6, _7 = env.step(t.train_func) # if over: # break # env.render('video') # # --------------------------------------------------------------------- # Whole Test. --------------------------------------------------------- from task.framework import DeepQNetwork import util.config as conf_util import os config_file = '/FocusDQN/config.ini' config_file = os.path.abspath(os.path.dirname(os.getcwd())) + config_file config = conf_util.parse_config(config_file) data_adapter = BratsAdapter(enable_data_enhance=False) dqn = DeepQNetwork(config=config, name_space='ME', data_adapter=data_adapter) # Train. dqn.train(epochs=10, max_instances=260) # Test. dqn.test(110, is_validate=False)
-t lookbackTime lookbackTime with units -g groupBy group commits by repo or repo followed by date or date followed by repo. """ from datetime import datetime from docopt import docopt from github.activity import summarise_commits from util.config import parse_config from util.constant import * if __name__ == '__main__': arguments = docopt(__doc__) appname = arguments['-p'] config_dict = parse_config(app_name=appname) lookback_time = 604800 if ('-t' in arguments and arguments['-t']): config_dict[LOOKBACK_TIME] = arguments['-t'] if ('-g' in arguments and arguments['-g']): config_dict[GROUPY_BY] = arguments['-g'] if (config_dict[LOOKBACK_TIME]): time_unit_map = { 's': 1, 'm': 60, 'h': 60 * 60, 'd': 60 * 60 * 24, 'w': 60 * 60 * 24 * 7