def set_config(config_file, two_gpu=False): kwargs={} kwargs['GraphParserNetwork']={} if two_gpu: kwargs['GraphParserNetwork']['two_gpu']=True else: kwargs['GraphParserNetwork']['two_gpu']=False config = Config(config_file=config_file, **kwargs) with open(config_file, 'w') as f: config.write(f)
def resolve_network_dependencies(config, network_class, network_list, networks): if network_list in ('None', ''): return set(), networks else: network_list = network_list.split(':') if network_class not in networks: for _network_class in network_list: config_file = os.path.join(config.get('DEFAULT', _network_class + '_dir'), 'config.cfg') _config = Config(config_file=config_file) _network_list = _config.get(_network_class, 'input_network_classes') input_networks, networks = resolve_network_dependencies(_config, _network_class, _network_list, networks) NetworkClass = getattr(parser, _network_class) networks[_network_class] = NetworkClass(input_networks=input_networks, config=config) return set(networks[_network_class] for _network_class in network_list), networks
def run(**kwargs): """""" #pdb.set_trace() # Get the special arguments save_dir = kwargs.pop('save_dir') save_metadir = kwargs.pop('save_metadir') conllu_files = kwargs.pop('conllu_files') output_dir = kwargs.pop('output_dir') output_filename = kwargs.pop('output_filename') testing = kwargs.pop('testing') debug = kwargs.pop('debug') nornn = kwargs.pop('nornn') check_iter = kwargs.pop('check_iter') gen_tree = kwargs.pop('gen_tree') get_argmax = kwargs.pop('get_argmax') # Get the cl-defined options kwargs = {key: value for key, value in six.iteritems(kwargs) if value is not None} for section, values in six.iteritems(kwargs): if section in section_names: values = [value.split('=', 1) for value in values] kwargs[section] = {opt: value for opt, value in values} if 'DEFAULT' not in kwargs: kwargs['DEFAULT'] = {} # Figure out the save_directory if save_metadir is not None: kwargs['DEFAULT']['save_metadir'] = save_metadir if save_dir is None: save_dir = Config(**kwargs).get('DEFAULT', 'save_dir') config_file = os.path.join(save_dir, 'config.cfg') namelist=save_dir.split('GraphParserNetwork') kwargs['DEFAULT']['save_dir'] = save_dir #kwargs['DEFAULT']['save_dir'] = namelist[0]+'GraphParserNetwork' if testing: #kwargs['DEFAULT']['AUTO_dir'] = 'True' #kwargs['DEFAULT']['modelname'] = namelist[-1] kwargs['CoNLLUDataset']={} if debug: kwargs['CoNLLUDataset']['batch_size'] = 1000 kwargs['CoNLLUDataset']['max_buckets'] = 30 else: kwargs['CoNLLUDataset']['batch_size'] = 1000 config = Config(defaults_file='', config_file=config_file, **kwargs) with open('debug.cfg', 'w') as f: config.write(f) network_class = config.get('DEFAULT', 'network_class') network_list = config.get(network_class, 'input_network_classes') input_networks, networks = resolve_network_dependencies(config, network_class, network_list, {}) NetworkClass = getattr(parser, network_class) network = NetworkClass(input_networks=input_networks, config=config) network.parse(conllu_files, output_dir=output_dir, output_filename=output_filename, testing=testing, debug=debug,nornn=nornn, check_iter=check_iter, gen_tree=gen_tree, get_argmax=get_argmax) return
def run(**kwargs): """""" # Get the special arguments save_dir = kwargs.pop('save_dir') save_metadir = kwargs.pop('save_metadir') conllu_files = kwargs.pop('conllu_files') output_dir = kwargs.pop('output_dir') output_filename = kwargs.pop('output_filename') # Get the cl-defined options kwargs = { key: value for key, value in six.iteritems(kwargs) if value is not None } for section, values in six.iteritems(kwargs): if section in section_names: values = [value.split('=', 1) for value in values] kwargs[section] = {opt: value for opt, value in values} if 'DEFAULT' not in kwargs: kwargs['DEFAULT'] = {} # Figure out the save_directory if save_metadir is not None: kwargs['DEFAULT']['save_metadir'] = save_metadir if save_dir is None: save_dir = Config(**kwargs).get('DEFAULT', 'save_dir') config_file = os.path.join(save_dir, 'config.cfg') kwargs['DEFAULT']['save_dir'] = save_dir config = Config(defaults_file='', config_file=config_file, **kwargs) network_class = config.get('DEFAULT', 'network_class') network_list = config.get(network_class, 'input_network_classes') input_networks, networks = resolve_network_dependencies( config, network_class, network_list, {}) NetworkClass = getattr(parser, network_class) network = NetworkClass(input_networks=input_networks, config=config) network.parse(conllu_files, output_dir=output_dir, output_filename=output_filename) return
def train(**kwargs): """""" # Get the special arguments load = kwargs.pop('load') force = kwargs.pop('force') noscreen = kwargs.pop('noscreen') save_dir = kwargs.pop('save_dir') save_metadir = kwargs.pop('save_metadir') network_class = kwargs.pop('network_class') config_file = kwargs.pop('config_file') # Get the cl-defined options kwargs = {key: value for key, value in six.iteritems(kwargs) if value is not None} for section, values in six.iteritems(kwargs): if section in section_names: values = [value.split('=', 1) for value in values] kwargs[section] = {opt: value for opt, value in values} if 'DEFAULT' not in kwargs: kwargs['DEFAULT'] = {} kwargs['DEFAULT']['network_class'] = network_class # Figure out the save_directory if save_metadir is not None: kwargs['DEFAULT']['save_metadir'] = save_metadir if save_dir is not None: kwargs['DEFAULT']['save_dir'] = save_dir config = Config(config_file=config_file, **kwargs) save_dir = config.get('DEFAULT', 'save_dir') # If not loading, ask the user if they want to overwrite the directory if not load and os.path.isdir(save_dir): if not force: input_str = '' while input_str not in ('y', 'n', 'yes', 'no'): input_str = input('{} already exists. It will be deleted if you continue. Do you want to proceed? [Y/n] '.format(save_dir)).lower() if input_str in ('n', 'no'): print() sys.exit(0) elif noscreen: sys.exit(0) shutil.rmtree(save_dir) # If the save_dir wasn't overwritten, load its config_file if os.path.isdir(save_dir): config_file = os.path.join(save_dir, 'config.cfg') else: os.makedirs(save_dir) os.system('git rev-parse HEAD >> {}'.format(os.path.join(save_dir, 'HEAD'))) network_list = config.get(network_class, 'input_network_classes') if not load: with open(os.path.join(save_dir, 'config.cfg'), 'w') as f: config.write(f) input_networks, networks = resolve_network_dependencies(config, network_class, network_list, {}) NetworkClass = getattr(parser, network_class) network = NetworkClass(input_networks=input_networks, config=config) network.train(load=load, noscreen=noscreen) return
def __init__(self, tup_cmd): self._tup_cmdLine = tuple(i for i in tup_cmd if "sameMachine" not in i) self._str_workload = "" self._tup_workload = () self._str_tools = "" self._tup_tools = () self._tup_simulators = () self._str_tasks = "" self._tup_tasks = () self._str_bench = "" self._tup_bench = () self.trials = 0 self.pid = 0 self.pinThreads = 0 self.cores = 0 self.str_output = "" self.verbose = 0 self.normalize = False self.roiOnly = False self.project = "" self.lockstep = False self.siteTracking = False self.attachPid = False self.generateTrace = False self.printOnly = False self.confIndex = -1 # Allow [parallelBenches] benchmarks to run parallelly self.parallelBenches = 1 self.generateEnergyStats = False self.pinTool = "" self.nonMesiSimulators = 0 self._isMesiPresent = False self._isViserPresent = False self._isRCCSIPresent = False self._isPausePresent = False self.output = None self.jassert = False self.xassert = False self.sameMachine = None self.period = None self.config = Config()
default='tag', choices=['tag', 'char', 'bert'], help='choices of additional features') args = parser.parse_args() logging.basicConfig(filename=args.output, filemode='w', format='%(asctime)s %(levelname)-8s %(message)s', level=logging.INFO, datefmt='%Y-%m-%d %H:%M:%S') logging.info(f"Set the max num of threads to {args.threads}") logging.info(f"Set the seed for generating random numbers to {args.seed}") logging.info(f"Set the device with ID {args.device} visible") torch.set_num_threads(args.threads) torch.manual_seed(args.seed) os.environ['CUDA_VISIBLE_DEVICES'] = args.device args.fields = os.path.join(args.file, 'fields') args.model = os.path.join(args.file, 'model') args.device = 'cuda' if torch.cuda.is_available() else 'cpu' logging.info(f"Override the default configs with parsed arguments") args = Config(args.conf).update(vars(args)) logging.info(args) logging.info(f"Run the subcommand in mode {args.mode}") cmd = subcommands[args.mode] cmd(args)
kwargs['GraphParserNetwork'][ 'input_vocab_classes'] = 'FormMultivocab:XPOSTokenVocab' kwargs['FormMultivocab']['use_subtoken_vocab'] = 'True' elif 'no_char' in cfg: kwargs['GraphParserNetwork'][ 'input_vocab_classes'] = 'FormMultivocab:XPOSTokenVocab:LemmaTokenVocab' kwargs['FormMultivocab']['use_subtoken_vocab'] = 'False' else: kwargs['GraphParserNetwork'][ 'input_vocab_classes'] = 'FormMultivocab:XPOSTokenVocab:LemmaTokenVocab' kwargs['FormMultivocab']['use_subtoken_vocab'] = 'True' #pdb.set_trace() if 'decay' not in cfg: kwargs['Optimizer']['decay_rate'] = 1 kwargs['DEFAULT']['modelname'] = cfg config = Config(config_file=config_file, **kwargs) cfg_dir = 'config_gen/' + cfg + '.cfg' with open(cfg_dir, 'w') as f: config.write(f) if args.twogpu: if args.autorun: train_case='CUDA_VISIBLE_DEVICES='+str(index%2*2)+','+str(index%2*2+1)+' nohup python3 -u main.py train GraphParserNetwork --nowarning --config_file '\ +cfg_dir+' --noscreen > log/'+cfg+'\n' else: train_case='CUDA_VISIBLE_DEVICES='+str(index%2*2)+','+str(index%2*2+1)+' nohup python3 -u main.py train GraphParserNetwork --config_file '\ +cfg_dir+' --noscreen > log/'+cfg+'&'+'\n' else: train_case='CUDA_VISIBLE_DEVICES='+str(index%3)+',3'' nohup python3 -u main.py train GraphParserNetwork --config_file '\ +cfg_dir+' --noscreen > log/'+cfg+'&'+'\n' writer.write(train_case) index += 1
subparser.add_argument('--threads', '-t', default=4, type=int, help='max num of threads') args = parser.parse_args() print(f"Set the max num of threads to {args.threads}") if args.seed != -1: print(f"Set the seed for generating random numbers to {args.seed}") torch.manual_seed(args.seed) else: seed = torch.initial_seed() % 10000000 torch.manual_seed(seed) print(f"the seed of random numbers generating {seed}") print(f"Set the device with ID {args.device} visible") torch.set_num_threads(args.threads) os.environ['CUDA_VISIBLE_DEVICES'] = args.device args.device = 'cuda' if torch.cuda.is_available() else 'cpu' print(f"Override the default configs with parsed arguments") args.vocab = os.path.join(args.file, args.vocab) args.model = os.path.join(args.file, args.model) config = Config(args.conf) config.update(vars(args)) print(config) print(f"Run the subcommand in mode {args.mode}") cmd = subcommands[args.mode] cmd(config)
def hpo(**kwargs): """""" # Get the special arguments noscreen = kwargs.pop('noscreen') save_dir = kwargs.pop('save_dir') save_metadir = kwargs.pop('save_metadir') network_class = kwargs.pop('network_class') config_file = kwargs.pop('config_file') rand_file = kwargs.pop('rand_file') eval_metric = kwargs.pop('eval_metric') # Get the cl-defined options kwargs = {key: value for key, value in six.iteritems(kwargs) if value is not None} for section, values in six.iteritems(kwargs): if section in section_names: values = [value.split('=', 1) for value in values] kwargs[section] = {opt: value for opt, value in values} if 'DEFAULT' not in kwargs: kwargs['DEFAULT'] = {} kwargs['DEFAULT']['network_class'] = network_class # Figure out the save_directory if save_metadir is not None: kwargs['DEFAULT']['save_metadir'] = save_metadir if save_dir is None: save_dir = Config(**kwargs).get('DEFAULT', 'save_dir') if not os.path.exists(save_dir): os.makedirs(save_dir) # Get the randomly generated options and possibly add them #------------------------------------------------------------- lang = kwargs['DEFAULT']['LANG'] treebank = kwargs['DEFAULT']['TREEBANK'] lc = kwargs['DEFAULT']['LC'] tb = kwargs['DEFAULT']['TB'] base = 'data/CoNLL18/UD_{}-{}/{}_{}-ud-dev.conllu'.format(lang, treebank, lc, tb) def eval_func(save_dir): return evaluate(base, os.path.join(save_dir, 'parsed', base), eval_metric) #------------------------------------------------------------- rargs = next(MVGHPO(rand_file, save_dir, eval_func=eval_func)) for section in rargs: if section not in kwargs: kwargs[section] = rargs[section] else: for option, value in six.iteritems(rargs[section]): if option not in kwargs[section]: kwargs[section][option] = value save_dir = os.path.join(save_dir, str(int(time.time()*100000))) # If not loading, ask the user if they want to overwrite the directory if os.path.isdir(save_dir): print() sys.exit(0) else: os.mkdir(save_dir) os.system('git rev-parse HEAD >> {}'.format(os.path.join(save_dir, 'HEAD'))) kwargs['DEFAULT']['save_dir'] = save_dir config = Config(config_file=config_file, **kwargs) network_list = config.get(network_class, 'input_network_classes') with open(os.path.join(save_dir, 'config.cfg'), 'w') as f: config.write(f) input_networks, networks = resolve_network_dependencies(config, network_class, network_list, {}) NetworkClass = getattr(parser, network_class) network = NetworkClass(input_networks=input_networks, config=config) network.train(noscreen=noscreen) return
if path_or_name(args.model) == "name": args.model = os.path.join(args.folder, "models", args.model) if not args.path_annotation_schema: args.path_annotation_schema = os.path.join(args.folder, "annotation_schema.json") print(args.path_annotation_schema) # else: # args.model = os.path.join(args.models, args.model) print("args.model", args.model) # print(f"Set the max num of threads to {args.threads}") print(f"Set the seed for generating random numbers to {args.seed}") # print(f"Set the device with ID {args.device} visible") # torch.set_num_threads(args.threads) torch.manual_seed(args.seed) # os.environ['CUDA_VISIBLE_DEVICES'] = args.device args.device, args.train_on_gpu, args.multi_gpu = get_gpus_configuration( args.gpu_ids) print(f"Override the default configs with parsed arguments") path_file_directory = pathlib.Path(__file__).parent.absolute() args = Config(os.path.join(path_file_directory, args.conf)).update(vars(args)) print(args) print(f"Run the subcommand in mode {args.mode}") cmd = subcommands[args.mode] cmd(args)
# subparser.add_argument('--seed', '-s', default=1, type=int, # help='seed for generating random numbers') # subparser.add_argument('--threads', '-t', default=16, type=int, # help='max num of threads') # subparser.add_argument('--tree', action='store_true', # help='whether to ensure well-formedness') # subparser.add_argument('--feat', default='tag', # choices=['tag', 'char', 'bert'], # help='choices of additional features') args = parser.parse_args() # os.environ['CUDA_VISIBLE_DEVICES'] = args.device args.device = 'cuda' if torch.cuda.is_available() else 'cpu' print(f"Override the default configs with parsed arguments") args = Config(args.conf).update(vars(args)) print(f"Set the max num of threads to {args.threads}") print(f"Set the seed for generating random numbers to {args.seed}") # print(f"Set the device with ID {args.device} visible") torch.set_num_threads(args.threads) torch.manual_seed(args.seed) args.fields = os.path.join(args.file, 'fields') args.model = os.path.join(args.file, 'model') print(args) print(f"Run the subcommand in mode {args.mode}") cmd = subcommands[args.mode] cmd(args)