def main(args): args.gpu = torch.cuda.is_available() utils.manual_seed(args.seed) Model = utils.load_module(args.model) cache_file = args.fcache or (os.path.join( 'cache', 'data_{}_{}.debug.pt'.format(args.model, args.dataset) if args.debug else 'data_{}_{}.pt'.format(args.model, args.dataset))) splits, ext = torch.load(cache_file, map_location=torch.device('cpu')) splits = {k: dataset.Dataset(v) for k, v in splits.items()} splits['train'] = Model.prune_train(splits['train'], args) splits['dev'] = Model.prune_dev(splits['dev'], args) if args.model == 'nl2sql': Reranker = utils.load_module(args.beam_rank) ext['reranker'] = Reranker(args, ext) m = Model(args, ext).place_on_device() d = m.get_file('') if not os.path.isdir(d): os.makedirs(d) pprint.pprint(m.get_stats(splits, ext)) if not args.test_only: if not args.skip_upperbound: print('upperbound') pprint.pprint(m.compute_upperbound(splits['train'][:1000])) if args.aug: augs = [] for a in args.aug: augs.extend(torch.load(a)) aug = dataset.Dataset(augs) splits['aug'] = Model.prune_train(aug, args)[:args.aug_lim] print('aug upperbound') pprint.pprint(m.compute_upperbound(aug[:10])) # aug_args = copy.deepcopy(args) # if 'consistent' not in args.aug: # aug_args.epoch = 10 # aug_dev = dataset.Dataset(random.sample(splits['train'], 3000)) # m.run_train(aug, aug_dev, args=aug_args) pprint.pprint(m.get_stats(splits, ext)) m.run_train(dataset.Dataset(splits['train'] + splits.get('aug', [])), splits['dev'], args=args) if args.resume: m.load_save(fname=args.resume) elif args.resumes: m.average_saves(args.resumes) if args.interactive_eval: dev_preds = m.run_interactive_pred(splits['dev'], args, verbose=True) else: dev_preds = m.run_pred(splits['dev'], args, verbose=True) if args.write_test_pred: with open(args.write_test_pred, 'wt') as f: json.dump(dev_preds, f, indent=2) print('saved test preds to {}'.format(args.write_test_pred)) pprint.pprint(m.compute_metrics(splits['dev'], dev_preds))
def load_checks(self, check_ios): """ Load checks from the database. Arguments: check_ios (Dict(int->List(CheckIO))): Mapping of check IDs to a list of CheckIOs to associate checks with Returns: List(Check,int): A list of checks and the ID of their associated systems """ checks = [] check_rows = db.getall('service_check') for check_id, name, system, port, check_string, poller_string in check_rows: # Build check ios = check_ios[check_id] check_function = load_module(check_string) poller_class = load_module(poller_string) poller = poller_class() check = Check(check_id, name, port, check_function, ios, poller) # Update link from check IOs to this check for check_io in ios: check_io.check = check checks.append((check, system)) return checks
def load_inst(cls, fresume, overwrite=None): binary = torch.load(fresume, map_location=torch.device('cpu')) args = binary['args'] ext = binary['ext'] for k, v in (overwrite or {}).items(): setattr(args, k, v) Model = utils.load_module(args.model) if args.model == 'nl2sql': Reranker = utils.load_module(args.beam_rank) ext['reranker'] = Reranker(args, ext) m = Model(args, ext).place_on_device() m.load_save(fname=fresume) return m
def _parse_modules(self, modules, debug): L = [] for mod in modules: spam = load_module(mod) if debug: spam.mtime = self._getmtime(spam.__file__) L.append(spam) return L
def load_temp_module(src): import tempfile import ntpath from utils import load_module temp = tempfile.NamedTemporaryFile(delete=False) temp.file.write(src.encode('utf-8')) temp.file.flush() module = load_module(ntpath.basename(temp.name), temp.name) return module, temp
def _add_module(self, mod): """Add interceptor(s) and route(s) in module 'mod' and return True if there are any, otherwise False.""" added = False m = mod if type(mod) == types.ModuleType else load_module(mod) for name in dir(m): fn = getattr(m, name) if callable(fn): if self._add_interceptor(fn) or self._add_route(fn): added = True return added
def create_engine(**kwargs): """Create database engine from kwargs. create_engine(engine='db.mysql', user='******', password='******') """ global db if db and db.driver: raise DBError("Database engine has already initialized!") kwargs = CaseInsensitiveDict(**kwargs) engine = kwargs.pop('ENGINE') if isinstance(engine, basestring): engine = load_module(engine) db = engine(**kwargs) return db
def load_results(self): """ Update results with any results not yet loaded from the database. """ if self.results is None: last_id = 0 # Setup dict self.results = {} for team in self.teams: self.results[team.id] = {} for check in self.checks: self.results[team.id][check.id] = [] else: # If results exist, we can just load the latest ones and keep the old ones # Here we find the id of the last result we already have last_ids = [] for team_results in self.results.values(): for check_results in team_results.values(): if len(check_results) != 0: last_ids.append(check_results[-1].id) last_id = -1 if len(last_ids) != 0: last_id = max(last_ids) rows = db.get('result', ['*'], where='id > %s', orderby='time ASC', args=(last_id)) # Gather the results for result_id, check_id, check_io_id, team_id, check_round, time, poll_input, poll_result, result in rows: # Construct the result from the database info check = [c for c in self.checks if c.id == check_id][0] check_io = [ cio for cio in self.check_ios if cio.id == check_io_id ][0] team = [t for t in self.teams if t.id == team_id][0] input_class_str, input_args = json.loads(poll_input) input_class = utils.load_module(input_class_str) poll_input = input_class.deserialize(input_class, input_args, self.teams, self.credentials) poll_result = json.loads(poll_result)[1] res = Result(result_id, check, check_io, team, check_round, time, poll_input, poll_result, result) self.results[team_id][check_id].append(res)
def post(self): """ New a Dbot This API need authorization with signature in headers """ # TODO request data valid check profile = request.files['profile'] specification = request.files['specification'] form = request.form dbot_data = json.load(profile) domain = form.get('domain', dbot_data['info'].get('domain')) if domain is None: abort(400, message="DBot domain is required") dbot_data['info'].update({ 'addr': form['address'], 'owner': form['owner'], 'floor_price': form['floor_price'], 'api_host': form['api_host'], 'protocol': form['protocol'], 'domain': domain }) dbot_data['specification']['data'] = specification.read().decode('utf-8') address = to_checksum_address(dbot_data['info'].get('addr')) if db.dbots.get(address) is not None: abort(404, message="Bad Request, can not insert an exsit DBot service") dbot_data['info']['addr'] = address middleware = None try: mw = dbot_data.get('middleware') if mw: mw_path = os.path.join(db.path(), 'middleware/{}'.format(address)) if not os.path.exists(mw_path): os.makedirs(mw_path) request.files.get('middleware').save(os.path.join(mw_path, '{}.py'.format(mw['module']))) middleware = getattr(load_module(mw['module'], mw_path), mw['class']) except Exception as err: abort(400, message='unable to load DBot middleware: {}'.format(err)) try: dbot.new_service(dbot_data, middleware) except Exception as err: abort(400, message=str(err)) db.dbots.put(address, dbot_data) return 'ok', 200
def execute(self): try: import time start = time.time() from utils import load_module method = load_module(self.action) argument = self._args() result = method(self.owner,argument) self.done() end = time.time() print 'Executing the job with jobid:%s took:%s seconds' % (self.id,(end-start)) return result except Exception,e: message= self.action +'\n' message+='Exception raised while executing Job with jobid:%s :%s' % (self.id,e.__str__()+'\n') print message from utils.emailer import mail_admins mail_admins(message,locals())
def put(self, dbot_address): profile = request.files['profile'] specification = request.files['specification'] form = request.form dbot_data = json.load(profile) domain = form.get('domain', dbot_data['info'].get('domain')) if domain is None: abort(400, message="DBot domain is required") dbot_data['info'].update({ 'addr': form['address'], 'owner': form['owner'], 'floor_price': form['floor_price'], 'api_host': form['api_host'], 'protocol': form['protocol'], 'domain': domain }) dbot_data['specification']['data'] = specification.read().decode('utf-8') if not is_same_address(dbot_data['info']['addr'], dbot_address): abort(400, message="Bad Request, wrong address in DBot data") dbot_address = to_checksum_address(dbot_address) dbot_data['info']['addr'] = dbot_address middleware = None try: # TODO update middleware (delete old one) mw = dbot_data.get('middleware') if mw: mw_path = os.path.join(db.path(), 'middleware/{}'.format(dbot_address)) if not os.path.exists(mw_path): os.makedirs(mw_path) request.files.get('middleware').save(os.path.join(mw_path, '{}.py'.format(mw['module']))) middleware = getattr(load_module(mw['module'], mw_path), mw['class']) except Exception as err: abort(400, message='unable to load DBot middleware: {}'.format(err)) try: dbot.update_service(dbot_data, dbot_address, middleware) except Exception as err: abort(400, message=str(err)) db.dbots.put(dbot_address, dbot_data) return 'ok', 200
def init(self, app, private_key, http_provider=None): self.account = Account.privateKeyToAccount(private_key) self.web3 = Web3( HTTPProvider(app.config['WEB3_PROVIDER_DEFAULT'] if http_provider is None else http_provider)) self.web3.middleware_stack.inject(geth_poa_middleware, layer=0) try: NETWORK_CFG.set_defaults(int(self.web3.version.network)) except HTTPError as err: logger.error( 'Can not connect with blockchain node: {}'.format(err)) raise HTTPError self.state_path = os.path.join(app.config['DB_ROOT'], 'channels') if not os.path.exists(self.state_path): os.makedirs(self.state_path) logger.info("load all exist dbot service") dbot_address_list = db.dbots.keys() for address in dbot_address_list: dbot_data = db.dbots.get(address) mw = dbot_data.get('middleware') if mw: mw_path = os.path.join(db.path(), 'middleware/{}'.format(address)) middleware = getattr(load_module(mw['module'], mw_path), mw['class']) self.new_service(dbot_data, middleware) else: self.new_service(dbot_data) logger.info("start metric collector") DBotMetricsCollector().Start( os.path.join(os.path.dirname(os.path.abspath(__file__)), "collector.conf")) self.metric = DBotApiMetric() DBotMetricsCollector().RegisterMetric(self.metric) self.metric.EnableDetailRecord(False) self.rest_server = None self.server_greenlet = None
def read_module(template, metadata_module, identifier, path, replace_value, doc_index=0): module = utils.load_module(path, identifier, doc_index) if module: parameters = module.get("Parameters", []) if isinstance(metadata_module, dict): for parameter in parameters: if default := metadata_module.get( parameter[:-len(identifier)], metadata_module.get( parameter[:-len("Param" + identifier)])): parameters.get(parameter)['Default'] = default if replace_value: k = next(iter(replace_value)) if (k + "Param") in module.get("Parameters", {}): module["Parameters"][k + "Param"]['Default'] = replace_value[k] reduce(utils.merge, [template, module])
def main(cnf, weights_from): config = utils.load_module(cnf).config if weights_from is None: weights_from = config.weights_file else: weights_from = str(weights_from) files = data_orig.get_image_files(config.get('train_dir')) names = data_orig.get_names(files) labels = data_orig.get_labels(names).astype(np.float32) net = create_net(config) try: net.load_params_from(weights_from) print("loaded weights from {}".format(weights_from)) except IOError: print("couldn't load weights starting from scratch") print("fitting ...") net.fit(files, labels)
def load_check_ios(self, credentials): """ Load CheckIOs from the database. Arguments: credentials (List(Credential)): List of credentials to associate CheckIOs with Returns: Dict(int->List(CheckIO)): Mapping of check IDs to a list of CheckIOs """ check_ios = super().load_check_ios(credentials) for check_id, cios in check_ios.items(): for cio in cios: # Rebuild the PollInput poll_input = cio.poll_input input_class_str, input_args = json.loads(poll_input) input_class = load_module(input_class_str) poll_input = input_class.deserialize(input_class, input_args, self.teams, self.credentials) cio.poll_input = poll_input return check_ios
def load(self, module_name): # If it has been loaded already, just return it! if module_name in self._loaded_modules: return self._loaded_modules[module_name] # If it's not in the cache, download it if module_name not in self._cache: module_path = 'modules/{}.py'.format(module_name) self._download(module_path, module_path) # Just downloaded, still have to verify self._cache[module_name] = False module = utils.load_module(module_name) # Shim some stuff into the module # @TODO: There has got to be a better (but still explicit) way... setattr(module, 'ok', okapi) setattr(module, 'utils', utils) # If it's not verified, try to verify it if not self._cache[module_name]: okapi.log( 'Checking whether {} is installed...'.format(module_name), important=False) if not module.check(): okapi.log( '{} not installed, installing now...'.format(module_name)) try: module.install() except Exception as e: raise utils.OkException( 'Unable to install module {}: {}'.format( module_name, e.message)) self._cache[module_name] = True # At this point we know it exists and is loaded, so... self._loaded_modules[module_name] = module return module
def setup(self, all_args): self.game = load_module(os.environ["game"]).Loader() self.game.args.set(all_args, actor_only=True, game_multi=2) self.gpu = all_args.eval_gpu self.tqdm = all_args.tqdm self.runner = SingleProcessRun() self.runner.args.set(all_args) self.GC = self.game.initialize() self.GC.setup_gpu(self.gpu) self.sampler = Sampler() self.sampler.args.set(all_args, greedy=True) self.trainer = Trainer() self.trainer.args.set(all_args) if self.args.stats == "rewards": self.collector = RewardCount() elif self.args.stats == "winrate": self.collector = WinRate() def actor(sel, sel_gpu, reply): self.trainer.actor(sel, sel_gpu, reply) v = sel[0] for batch_idx, (id, last_terminal) in enumerate(zip(v["id"], v["last_terminal"])): self.collector.feed(id, v["last_r"][batch_idx]) if last_terminal: self.collector.terminal(id) self.GC.reg_callback("actor", actor) self.GC.Start()
""" Training Code Adapted from https://github.com/TwentyBN/smth-smth-v2-baseline-with-models """ import os import time import signal import torch import utils import importlib from visualisation import PlotLearning # load configurations args = utils.load_args() config = utils.load_module(args.config).config # set column model file_name = config['conv_model'] cnn_def = importlib.import_module("{}".format(file_name)) # setup device - CPU or GPU device, device_ids = utils.setup_cuda_devices(args) print(" > Using device: {}".format(device.type)) print(" > Active GPU ids: {}".format(device_ids)) best_loss = float('Inf') if config["input_mode"] == "jpg": from data_loader_jpg import ImLoader else: raise ValueError("Please provide a valid input mode")
def __init__(self, fconf, handler): """ Initialize a MasterServer instance @param fconf the path to the configuration file @param handler the handler object in charge of managing HTTP requests """ Logger.__init__(self, "Manager") conf = json.load(open(fconf)) # Jinja2 initialization. tmpl_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'templates') self.env = Environment(loader=FileSystemLoader(tmpl_path)) self.status = ApplicationStatus() # This is a dictionary structure in the form # reduce_dict["group-name"] = [ # [ file list by unique integers, size in byte # ] => Reduce-0 # [ # ] => Reduce-1 # ] self.reduce_mark = set() self.reduce_dict = defaultdict(list) self.dead_reduce_dict = defaultdict(list) # This is a dictionary nick => Handler instance self.masters = {} self.last_id = -1 self.pending_works = defaultdict(list) # nick => [work, ...] self.ping_max = int(conf["ping-max"]) self.ping_interval = int(conf["ping-interval"]) self.num_reducer = int(conf["num-reducer"]) # This will just keep track of the name of the files self.reduce_files = [] self.results_printed = False for _ in range(self.num_reducer): self.reduce_files.append("N/A") # Load the input module and assing the generator to the work_queue module = load_module(conf["input-module"]) cls = getattr(module, "Input", None) # Some code for the DFS generator = cls(fconf).input() self.use_dfs = use_dfs = conf['dfs-enabled'] if use_dfs: dfsconf = conf['dfs-conf'] dfsconf['host'] = dfsconf['master'] self.path = conf['output-prefix'] else: dfsconf = None self.path = os.path.join( os.path.join(conf['datadir'], conf['output-prefix']) ) self.work_queue = WorkQueue(self.logger, generator, use_dfs, dfsconf) # Lock to synchronize access to the timestamps dictionary self.lock = Lock() self.timestamps = {} # nick => (send_ts:enum, ts:float) # Ping thread self.hb_thread = Thread(target=self.hearthbeat) # Event to mark the end of the server self.finished = Event() self.addrinfo = (conf['master-host'], conf['master-port']) Server.__init__(self, self.addrinfo[0], self.addrinfo[1], handler)
import os import sys import json import utils import tempfile import numpy as np import tensorflow as tf import tensorflow.contrib.slim as slim config = utils.load_config() sys.path.append(os.path.expanduser(config['slim_path'])) resnet = utils.load_module('nets.resnet_v1') logger = utils.get_default_logger() class WildCat: def __init__(self, images, labels=None, n_classes=None, training=False, transfer_conv_size=(3, 3), n_maps_per_class=5, alpha=1.0, k=1, reg=None): self.images = images self.labels = labels self.n_classes = n_classes self.training = training self.transfer_conv_size = transfer_conv_size
def get_model(model_path): mod_model = load_module(model_path) return mod_model.get()
def get_entities(name, path=None): models = load_module(name, path=path) return [model for name, model in models.__dict__.iteritems() if is_subclass(model, Entity)]
import sys import utils def run(): pass if __name__ == "__main__": if len(sys.argv) < 2: raise Exception("Command missing.") command = utils.load_module(sys.argv[1]) command.run()
def main(orig_args): # load pretrained model fresume = os.path.abspath(orig_args.resume) # print('resuming from {}'.format(fresume)) assert os.path.isfile(fresume), '{} does not exist'.format(fresume) orig_args.input = os.path.abspath(orig_args.input) orig_args.tables = os.path.abspath(orig_args.tables) orig_args.db = os.path.abspath(orig_args.db) orig_args.dcache = os.path.abspath(orig_args.dcache) binary = torch.load(fresume, map_location=torch.device('cpu')) args = binary['args'] ext = binary['ext'] args.gpu = torch.cuda.is_available() args.tables = orig_args.tables args.db = orig_args.db args.dcache = orig_args.dcache args.batch = orig_args.batch Model = utils.load_module(args.model) if args.model == 'nl2sql': Reranker = utils.load_module(args.beam_rank) ext['reranker'] = Reranker(args, ext) m = Model(args, ext).place_on_device() if orig_args.resumes: m.average_saves(orig_args.resumes) else: m.load_save(fname=fresume) # preprocess data data = dataset.Dataset() if orig_args.dataset == 'spider': import preprocess_nl2sql as preprocess elif orig_args.dataset == 'sparc': import preprocess_nl2sql_sparc as preprocess elif orig_args.dataset == 'cosql': import preprocess_nl2sql_cosql as preprocess proc_errors = set() with open(orig_args.input) as f: C = preprocess.SQLDataset raw = json.load(f) # make contexts and populate vocab for i, ex in enumerate(raw): for k in ['query', 'query_toks', 'query_toks_no_value', 'sql']: if k in ex: del ex[k] ex['id'] = '{}/{}'.format(ex['db_id'], i) new = C.make_example(ex, m.bert_tokenizer, m.sql_vocab, m.conv.kmaps, m.conv, train=False, evaluation=True) new['question'] = ex['question'] if new is not None: new['cands_query'], new['cands_value'] = C.make_cands( new, m.sql_vocab) data.append(new) else: print('proc error') proc_errors.add(ex['id']) # run preds if orig_args.dataset in {'cosql', 'sparc'}: preds = m.run_interactive_pred(data, args, verbose=True) raise NotImplementedError() else: preds = m.run_pred(data, args, verbose=True) assert len(preds) + len(proc_errors) == len( data), 'got {} predictions for {} examples'.format( len(preds), len(data)) # print('writing to {}'.format(orig_args.output)) with open(orig_args.output, 'wt') as f: for ex in data: if ex['id'] in proc_errors: s = 'ERROR' else: p = preds[ex['id']] s = p['query'] f.write(s + '\n') f.flush()
count += 1 return True def max_size(proto_message_type): cb_args = {} ext_refs = {} immutable_objs = {} ext_constraints = None num_objects = 1 mbt_obj_store.set_walk_enum(True) mbt_obj_store.set_skip_none_enum(True) walk_proto(proto_message_type, max_size_cb, cb_args, num_objects, ext_refs, ext_constraints, immutable_objs) module_name = 'fwlog_pb2' proto_message_name = 'FWEvent' module = utils.load_module(module_name) proto_message_type = getattr(module, proto_message_name) max_size(proto_message_type) print("Max size: " + str(max_size_val))
from __future__ import division import time import click import numpy as np import nn import data_orig import tta import utils cnf='configs/c_512_5x5_32.py' config = utils.load_module(cnf).config config.cnf['batch_size_train'] = 128 runs = {} runs['train'] = config.get('train_dir') net = nn.create_net(config) weights_from = 'weights/c_512_5x5_32/weights_final.pkl' net.load_params_from(weights_from) tf, color_vecs = tta.build_quasirandom_transforms(1, skip=0, color_sigma=0.0, **data_orig.no_augmentation_params) for i, (tf, color_vec) in enumerate(zip(tfs, color_vecs), start=1): pass
Gen_optimizer = optim.Adam(Gen_Model.parameters(), lr=lr) #lr = 1e-4 Dis_optimizer = optim.Adam(Dis_Model.parameters(), lr=lr) Vggloss = Perceptual_Loss.vggloss() # vgg(5,4) loss content_criterion = nn.MSELoss() adversal_criterion = nn.BCEWithLogitsLoss() PSNR_eval = np.zeros(TOTAL_EPOCH) PSNR_train = np.zeros(TOTAL_EPOCH) Train_Gen_loss = np.zeros(TOTAL_EPOCH) Train_Dis_loss = np.zeros(TOTAL_EPOCH) train_len = len(train_dataloader) start_epoch = 0 if PRETRAINED_PATH is not None: _, gen_modelpath = utils.load_module( os.path.join(PRETRAINED_PATH, "Generator")) start_epoch, dis_modelpath = utils.load_module( os.path.join(PRETRAINED_PATH, "Discriminator")) print(dis_modelpath) print("load module : saved on {} epoch".format(start_epoch)) Gen_Model.load_state_dict(torch.load(gen_modelpath)) Dis_Model.load_state_dict(torch.load(dis_modelpath)) if PRE_RESULT_DIR is not None: PSNR_eval = np.load("result_data/PSNR_eval.npy") PSNR_Train = np.load("result_data/PSNR_train.npy") Train_Dis_loss = np.load("result_data/Train_Dis_loss.npy") Train_Gen_loss = np.load("result_data/Train_Gen_loss.npy") Gen_Model = Gen_Model.to(device) Dis_Model = Dis_Model.to(device)
def main(orig_args): # load pretrained model fresume = os.path.abspath(orig_args.resume) # print('resuming from {}'.format(fresume)) assert os.path.isfile(fresume), '{} does not exist'.format(fresume) orig_args.input = os.path.abspath(orig_args.input) orig_args.tables = os.path.abspath(orig_args.tables) orig_args.db = os.path.abspath(orig_args.db) orig_args.dcache = os.path.abspath(orig_args.dcache) binary = torch.load(fresume, map_location=torch.device('cpu')) args = binary['args'] ext = binary['ext'] args.gpu = torch.cuda.is_available() args.tables = orig_args.tables args.db = orig_args.db args.dcache = orig_args.dcache args.batch = orig_args.batch Model = utils.load_module(args.model) if args.model == 'nl2sql': Reranker = utils.load_module(args.beam_rank) ext['reranker'] = Reranker(args, ext) m = Model(args, ext).place_on_device() m.load_save(fname=fresume) # preprocess data data = dataset.Dataset() if args.dataset == 'sparc': import preprocess_nl2sql_sparc as preprocess elif args.dataset == 'cosql': import preprocess_nl2sql_cosql as preprocess else: raise NotImplementedError() proc_errors = set() with open(orig_args.input) as f: C = preprocess.SQLDataset raw = json.load(f) # make contexts and populate vocab for i, ex in enumerate(raw): for turn_i, turn in enumerate(ex['interaction']): turn['id'] = '{}/{}:{}'.format(ex['database_id'], i, turn_i) turn['db_id'] = ex['database_id'] for k in ['query', 'query_toks', 'query_toks_no_value', 'sql']: if k in turn: del turn[k] turn['question'] = turn['utterance'] turn['g_question_toks'] = C.tokenize_question( turn['utterance'].split(), m.bert_tokenizer) turn['value_context'] = [ m.bert_tokenizer.cls_token ] + turn['g_question_toks'] + [m.bert_tokenizer.sep_token] turn['turn_i'] = turn_i data.append(turn) # run preds preds = m.run_interactive_pred(data, args, verbose=True) assert len(preds) == len( data), 'got {} predictions for {} examples'.format( len(preds), len(data)) # print('writing to {}'.format(orig_args.output)) with open(orig_args.output, 'wt') as f: for i, ex in enumerate(data): if i != 0 and ex['turn_i'] == 0: f.write('\n') if ex['id'] in proc_errors: s = 'ERROR' else: p = preds[ex['id']] s = p['query'] f.write(s + '\n') f.flush()
self.GC.Run() summary = c.summary() for k, v in summary.items(): print("%s: %s" % (str(k), str(v))) def __del__(self): self.GC.Stop() if __name__ == '__main__': parser = argparse.ArgumentParser() use_multi_process = int(os.environ.get("multi_process", 0)) sampler = Sampler() trainer = Trainer() game = load_module(os.environ["game"]).Loader() runner = MultiProcessRun() if use_multi_process else SingleProcessRun() model_file = load_module(os.environ["model_file"]) model_class, method_class = model_file.Models[os.environ["model"]] model_loader = ModelLoader(model_class) method = method_class() args_providers = [sampler, trainer, game, runner, model_loader, method] eval_only = os.environ.get("eval_only", False) has_eval_process = os.environ.get("eval_process", False) if has_eval_process or eval_only: eval_process = EvaluationProcess() evaluator = Eval()
conf.label_training = utils.abspath(conf.label_training) conf.input_validation = [utils.abspath(path) for path in conf.input_validation] conf.label_validation = utils.abspath(conf.label_validation) conf.one_hot_palette_input = utils.abspath(conf.one_hot_palette_input) conf.one_hot_palette_label = utils.abspath(conf.one_hot_palette_label) conf.model = utils.abspath(conf.model) conf.unetxst_homographies = utils.abspath( conf.unetxst_homographies ) if conf.unetxst_homographies is not None else conf.unetxst_homographies conf.model_weights = utils.abspath( conf.model_weights ) if conf.model_weights is not None else conf.model_weights conf.output_dir = utils.abspath(conf.output_dir) # load network architecture module architecture = utils.load_module(conf.model) # get max_samples_training random training samples n_inputs = len(conf.input_training) files_train_input = [ utils.get_files_in_folder(folder) for folder in conf.input_training ] files_train_label = utils.get_files_in_folder(conf.label_training) _, idcs = utils.sample_list(files_train_label, n_samples=conf.max_samples_training) files_train_input = [np.take(f, idcs) for f in files_train_input] files_train_label = np.take(files_train_label, idcs) image_shape_original_input = utils.load_image( files_train_input[0][0]).shape[0:2] image_shape_original_label = utils.load_image(files_train_label[0]).shape[0:2] print(f"Found {len(files_train_label)} training samples")
def template_engine(self, engine): if not callable(engine): engine = load_module(engine) self._template_engine = engine
#print("Arguments",args) # Read master config with open(args.master_config) as f: config = yaml.load(f) #print(json.dumps(config, indent=2)) # Get modular functions # Note that event lists have parsers and handlers; state lists only parsers event_parser = dict() event_handler = dict() state_parser = dict() for listname in config['event_lists'].keys(): print("Loading parser for event list {} from {}".format( listname, config['event_lists'][listname]['parser'])) module = load_module('parsers.events.' + listname, config['event_lists'][listname]['parser']) event_parser[listname] = module.parser print("Loading handler for event {} from {}".format( listname, config['event_lists'][listname]['event_handler'])) module = load_module('handlers.events.' + listname, config['event_lists'][listname]['event_handler']) event_handler[listname] = module.handler print("Event list parsers:", event_parser.keys()) # Note that state lists do not have event handlers for listname in config['state_lists'].keys(): print("Loading parser for state list {} from {}".format( listname, config['state_lists'][listname]['parser'])) module = load_module('parsers.state.' + listname, config['state_lists'][listname]['parser']) state_parser[listname] = module.parser print("State list parsers:", state_parser.keys())
def __init__(self, config_spec, cfg_spec_obj, hal_channel): self._config_spec = config_spec self._service_name = config_spec.Service # create client stub object from module module = utils.load_module(config_spec.ProtoObject) stub_method_name = self._service_name + "Stub" # connect to client stub to GRPC server stub = getattr(module, stub_method_name)(hal_channel) self._cfg_spec_obj_name = cfg_spec_obj.name self._max_objects = mbt_obj_store.default_max_objects() if 'max_objects' in dir(cfg_spec_obj): self._max_objects = int(cfg_spec_obj.max_objects) self._key_or_handle_str = "" if 'key_handle' in dir(cfg_spec_obj): self._key_or_handle_str = cfg_spec_obj.key_handle # config methods which are ignored ignore = {} ignore[ConfigMethodType.CREATE] = False ignore[ConfigMethodType.GET] = False ignore[ConfigMethodType.UPDATE] = False ignore[ConfigMethodType.DELETE] = False if 'ignore_v2' in dir(cfg_spec_obj): ignore_ops = cfg_spec_obj.ignore_v2 else: ignore_ops = cfg_spec_obj.ignore for index in ignore_ops or []: if index.op == 'Create': ignore[ConfigMethodType.CREATE] = True if index.op == 'Get': ignore[ConfigMethodType.GET] = True if index.op == 'Update': ignore[ConfigMethodType.UPDATE] = True if index.op == 'Delete': ignore[ConfigMethodType.DELETE] = True # configuration methods for this object self._config_methods = {} cfg_method_type = ConfigMethodType.CREATE self._config_methods[cfg_method_type] = ConfigMethod(stub, module, cfg_spec_obj.create, ignore[cfg_method_type]) cfg_method_type = ConfigMethodType.GET self._config_methods[cfg_method_type] = ConfigMethod(stub, module, cfg_spec_obj.get, ignore[cfg_method_type]) cfg_method_type = ConfigMethodType.UPDATE self._config_methods[cfg_method_type] = ConfigMethod(stub, module, cfg_spec_obj.update, ignore[cfg_method_type]) cfg_method_type = ConfigMethodType.DELETE self._config_methods[cfg_method_type] = ConfigMethod(stub, module, cfg_spec_obj.delete, ignore[cfg_method_type]) # constraints specified in the config spec object self._constraints = [] if 'constraints' in dir(cfg_spec_obj): cfg_spec_obj_constraints = getattr(cfg_spec_obj, 'constraints') for cfg_spec_obj_constraint in cfg_spec_obj_constraints: constraint = GrpcReqRspMsg.extract_constraints(cfg_spec_obj_constraint.constraint) self._constraints.append(constraint) # store external reference object for the specific key_handle # __DREPRECATED__ self._ext_ref_obj_list = [] # cache of mbt_handles of this type self._mbt_handle_list = [] # stats for this object self._num_create_ops = 0 self._num_get_ops = 0 self._num_update_ops = 0 self._num_delete_ops = 0