def predict_handler(params): text = params['text'] word = params['word'] print(util.red(text), util.yellow(word)) response = bert_masked_lm_prob(text, word) print(util.yellow(response)) return json.dumps(response)
def __init__(self, config, dynamics_model, solver): print(util.yellow("Using oneshot graph with lstm")) self._config = config self._dynamics = dynamics_model self._state_dim = dynamics_model.state_dim self._control_dim = dynamics_model.control_dim self._num_time_interval = config.num_time_interval self._total_time = config.total_time self._dtype = config.float_type self._initialization = config.initialization self._gradient_clip = config.max_gradient_norm self._solver = solver self._delta_t = config.delta_t self._sqrt_delta_t = np.sqrt(self._delta_t) self._saver = None self._random_seed = config.random_seed self._weight_V = config.weight_V # weight for value function difference self._weight_Vx = config.weight_Vx # weight for value function gradient difference self._weight_V_true = config.weight_V_true # weight for true value function self._weight_Vx_true = config.weight_Vx_true # weight for true value function gradient self._weight_Vxx_col = config.weight_Vxx_col self._weight_Vxx_col_true = config.weight_Vxx_col_true self._exploration = config.exploration # exploration factor during training self._use_TD_error = self._config.use_TD_error self._input_targets = self._config.input_targets self._use_lstm_batch_norm = self._config.use_lstm_batch_norm self._use_abs_loss = self._config.use_abs_loss print(util.yellow("%s" % (self._config.message))) ad_stoch_search_params_print ="For ad_stoch_search using,\n" + \ "n_sample_train = " + str(config.n_sample_train) + "\n"+\ "n_sample_infer = " + str(config.n_sample_infer) + "\n"+\ "n_iter_train = " + str(config.n_iter_train) + "\n" + \ "n_iter_infer = " + str(config.n_iter_infer) + "\n" + \ "shape_func = " + str(config.shape_func) + "\n" + \ "use_Hessian = " + str(config.use_Hessian) + "\n" + \ "kappa = " + str(config.kappa) + "\n" + \ "alpha = " + str(config.alpha) + "\n" + \ "use_gpu = " + str(config.use_gpu) + "\n" + \ "device = " + str(config.device) + "\n" + \ "train_kappa = " + str(config.train_kappa) + "\n" + \ "train_alpha = " + str(config.train_alpha) + "\n" + \ "use_linesearch = " + str(config.use_linesearch) + "\n" + \ "init_sigma = " + str(config.init_sigma) + "\n" print(util.yellow("%s" % (ad_stoch_search_params_print)))
def show_progress(self, opt, it, loss): time_elapsed = util.get_time(time.time() - self.time_start) print("it {0}/{1}, lr:{3}, loss:{4}, time:{2}".format( util.cyan("{}".format(it + 1)), opt.to_it, util.green("{0}:{1:02d}:{2:05.2f}".format(*time_elapsed)), util.yellow("{:.2e}".format(opt.lr_pmo)), util.red("{:.4e}".format(loss.all)), ))
def show_progress(self,opt,ep,loss): [lr] = self.sched.get_lr() time_elapsed = util.get_time(time.time()-self.time_start) print("ep {0}/{1}, lr:{3}, loss:{4}, time:{2}" .format(util.cyan("{}".format(ep+1)), opt.to_epoch, util.green("{0}:{1:02d}:{2:05.2f}".format(*time_elapsed)), util.yellow("{:.2e}".format(lr)), util.red("{:.4e}".format(loss.all)), ))
def load(self): """ This is an extremely important contextmanager. If you load do any attr mutation operation you probably want to use it. It holds a lock the whole time and loads the whole shared object into local memory for the duration of the contextmanager. Any getattr/setattr calls will use this local version. This contextmanager recurses with no issues. Nobody else can access the shared object as long as you have it loaded, since load() takes a lock(). Why this is so important: If you do sharedobj.mylist.pop() outside of this contextmanager the 'mylist' attr will be loaded from disk, then pop() will be called on it, but it won't be written back to disk. If you have it wrapped in a load() then all of the attributes will be loaded into self.attr_dict, then the 'mylist' attr will simply be taken from this local dict, so when pop() is called it will modify the mylist copy in the local dict, and at the very end everything will get flushed back to disk when load() ends. """ with self.lock(): self.load_count += 1 if self.fs.verbose: util.yellow(f'load(load_count={self.load_count})') if self.load_count == 1: # if this is first load since being unloaded if self.fs.verbose: util.green(f'--loaded--') self.attr_dict = self.fs.load_nosplay(self.abspath) try: yield None finally: self.load_count -= 1 if self.fs.verbose: util.yellow(f'unload(load_count={self.load_count})') if self.load_count == 0: # unload for real self.fs.save_nosplay(self.attr_dict, self.abspath) if self.fs.verbose: util.purple(f'--unloaded--')
def set(): # parse input arguments parser = argparse.ArgumentParser() parser.add_argument("--problem_name", default="Finance", help="name for system to control") parser.add_argument("--graph", default="oneshot", help="which graph to use") parser.add_argument("--debug", action="store_true", help="") parser.add_argument("--gpu", default="False", help="use cuda tensors") parser.add_argument("--recurrent", action="store_true", help="recurrent") parser.add_argument( "--load_timestamp", default=None, help="timestamp of previous experiment you want to load") parser.add_argument( "--test_only", default="False", help="Run test only. For this load_timestamp is mandatory") args = parser.parse_args() # --- below are automatically set --- # if args.seed is not None: # seed = args.seed # random.seed(seed) # os.environ['PYTHONHASHSEED'] = str(seed) # np.random.seed(seed) # args.problem_name += "_seed{}".format(args.seed) opt_dict = {name: value for (name, value) in args._get_kwargs()} opt = argparse.Namespace(**opt_dict) # print configurations for o in vars(opt): print(util.green(o), ":", util.yellow(getattr(opt, o))) print() return opt
def prompt(choices, mode='*'): if mode not in PROMPT_MODES: raise ValueError("mode '{}' is invalid".format(mode)) if len(choices) > 26: raise ValueError("too many choices") if mode == '*': header = "select zero or more:" max, min = float('inf'), 0 elif mode == '+': header = "select one or more:" max, min = float('inf'), 1 elif mode in [1, '1']: header = "select one:" max, min = 1, 1 elif mode == '?': header = "select zero or one:" max, min = 1, 0 letters = list(map(lambda x: chr(ord('a') + x), range(len(choices)))) num_selections = 0 selections = [] # unique indices into choices list while num_selections < min or num_selections < max: util.print(util.green(header)) for i in range(len(choices)): if i in selections: choice = " × " else: choice = " " choice += str(letters[i]) + '. ' + str(choices[i]) if i in selections: choice = util.yellow(choice) util.print(choice) try: sel = input(util.green("make a selection (or ! to commit): ")) except KeyboardInterrupt: util.exit(util.ERR_INTERRUPTED) if sel == '!': if num_selections < min: util.error("can't stop now; you must make " "{} {}".format(min, util.plural("selection", min))) continue else: break try: if letters.index(sel) in selections: selections.remove(letters.index(sel)) continue selections.append(letters.index(sel)) num_selections += 1 except ValueError: if sel == '': util.print("make a selection (or ! to commit)") else: util.error("invalid selection: not in list") continue return selections
import numpy as np import os, sys, time import torch import options, data, util import model print(util.yellow("=======================================================")) print(util.yellow("main.py (photometric mesh optimization)")) print(util.yellow("=======================================================")) print(util.magenta("setting configurations...")) opt = options.set() print(util.magenta("reading list of sequences...")) seq_list = data.load_sequence_list(opt, subset=1) seq_list = [("02958343", "eebbce8b77bdb53c82382fde2cafeb9")] with torch.cuda.device(opt.gpu): pmo = model.Model(opt) pmo.build_network(opt) pmo.restore_checkpoint(opt) print(util.yellow("======= OPTIMIZATION START =======")) for c, m in seq_list: pmo.load_sequence(opt, c, m) pmo.setup_visualizer(opt) pmo.setup_variables(opt) pmo.setup_optimizer(opt) pmo.time_start(opt) pmo.optimize(opt)
def set(): # parse input arguments parser = argparse.ArgumentParser() parser.add_argument("--eval", action="store_true", help="evaluation phase") parser.add_argument("--group", default="0", help="name for group") parser.add_argument("--name", default="debug", help="name for model instance") parser.add_argument("--seed", type=int, default=0, help="fix random seed") parser.add_argument("--gpu", type=int, default=0, help="GPU device") parser.add_argument("--cpu", action="store_true", help="use CPU only") parser.add_argument("--load", default=None, help="load (pre)trained model") # dataset parser.add_argument("--rendering-path", default="data/rendering", help="path to ShapeNet rendering") parser.add_argument("--pointcloud-path", default="data/customShapeNet", help="path to ShapeNet 3D point cloud") parser.add_argument("--sun360-path", default="data/background", help="path to SUN360 background") parser.add_argument("--seq-path", default="data/sequences", help="path to RGB sequences for evaluation") parser.add_argument("--category", default=None, help="train on specific category") parser.add_argument("--num-workers", type=int, default=8, help="number of data loading threads") parser.add_argument("--size", default="224x224", help="rendered image size") parser.add_argument("--sfm", action="store_true", help="use coordinate system mapping from SfM output") parser.add_argument("--init-idx", type=int, default=27, help="initial frame index") parser.add_argument("--noise", type=float, default=None, help="gaussian noise in coordinate system mapping") # visualization parser.add_argument("--log-tb", action="store_true", help="output loss in TensorBoard") parser.add_argument("--log-visdom", action="store_true", help="visualize mesh in Visdom") parser.add_argument("--vis-server", default="http://localhost", help="visdom port server") parser.add_argument("--vis-port", type=int, default=8097, help="visdom port number") parser.add_argument("--video", action="store_true", help="write video sequence with optimized mesh") # AtlasNet parser.add_argument("--num-prim", type=int, default=25, help="number of primitives") parser.add_argument("--num-points", type=int, default=100, help="number of points (per primitive)") parser.add_argument("--num-meshgrid", type=int, default=5, help="number of regular grids for mesh") parser.add_argument("--sphere", action="store_true", help="use closed sphere for AtlasNet") parser.add_argument("--sphere-densify", type=int, default=3, help="densify levels") parser.add_argument("--imagenet-enc", action="store_true", help="initialize with pretrained ResNet encoder") parser.add_argument("--pretrained-dec", default=None, help="initialize with pretrained AtlasNet decoder") # photometric optimization parser.add_argument( "--batch-size-pmo", type=int, default=-1, help="batch size for photometric optimization (-1 for all)") parser.add_argument("--lr-pmo", type=float, default=1e-3, help="base learning rate for photometric optimization") parser.add_argument("--code", type=float, default=None, help="penalty on code difference") parser.add_argument("--scale", type=float, default=None, help="penalty on scale") parser.add_argument("--to-it", type=int, default=100, help="run optimization to iteration number") parser.add_argument( "--avg-frame", action="store_true", help="average photo. loss across frames instead of sampled pixels") # AtlasNet training parser.add_argument("--batch-size", type=int, default=32, help="input batch size") parser.add_argument( "--aug-transl", type=int, default=None, help="augment with random translation (for new dataset)") parser.add_argument("--lr-pretrain", type=float, default=1e-4, help="base learning rate") parser.add_argument("--lr-decay", type=float, default=1.0, help="learning rate decay") parser.add_argument("--lr-step", type=int, default=100, help="learning rate decay step size") parser.add_argument("--from-epoch", type=int, default=0, help="train from epoch number") parser.add_argument("--to-epoch", type=int, default=500, help="train to epoch number") opt = parser.parse_args() # --- below are automatically set --- if opt.seed is not None: np.random.seed(opt.seed) torch.manual_seed(opt.seed) torch.cuda.manual_seed_all(opt.seed) opt.name += "_seed{}".format(opt.seed) opt.device = "cpu" if opt.cpu or not torch.cuda.is_available( ) else "cuda:{}".format(opt.gpu) opt.H, opt.W = [int(s) for s in opt.size.split("x")] if opt.sphere: opt.num_prim = 1 opt.num_points_all = opt.num_points * opt.num_prim # print configurations for o in sorted(vars(opt)): print(util.green(o), ":", util.yellow(getattr(opt, o))) print() return opt
def get_or_create_projects(): projects = [] # Loading projects from the Excel. df = pd.read_excel(ANNOTATED_FILE, keep_default_na=False) df = df[df.discardReason == ''].reset_index(drop=True) projects_excel = dict() for i, project_excel in df.iterrows(): projects_excel[(project_excel['owner'], project_excel['name'])] = project_excel # Loading projects from the database. projects_db = db.query(db.Project).options( load_only('id', 'owner', 'name'), selectinload(db.Project.versions).load_only('id')).all() status = { 'Excel': len(projects_excel), 'Database': len(projects_db), 'Added': 0, 'Deleted': 0, 'Repository not found': 0, 'Git error': 0 } i = 0 # Deleting projects that do not exist in the Excel file for project in projects_db: if projects_excel.pop((project.owner, project.name), None) is not None: # Print progress information i += 1 progress = '{:.2%}'.format(i / status['Excel']) print( f'[{progress}] Adding project {project.owner}/{project.name}:', end=' ') projects.append(project) print(yellow('already done.')) else: db.delete(project) status['Deleted'] += 1 # Adding missing projects in the database for project_excel in projects_excel.values(): # Print progress information i += 1 progress = '{:.2%}'.format(i / status['Excel']) print( f'[{progress}] Adding project {project_excel["owner"]}/{project_excel["name"]}:', end=' ') project_dict = { k: v for k, v in project_excel.to_dict().items() if k not in ['url', 'isSoftware', 'discardReason'] } project_dict['createdAt'] = str(project_dict['createdAt']) project_dict['pushedAt'] = str(project_dict['pushedAt']) try: os.chdir(REPOS_DIR + os.sep + project_dict['owner'] + os.sep + project_dict['name']) p = subprocess.run(REVPARSE_COMMAND, capture_output=True) if p.stderr: raise subprocess.CalledProcessError(p.returncode, REVPARSE_COMMAND, p.stdout, p.stderr) project = db.create(db.Project, **project_dict) db.create(db.Version, sha1=p.stdout.decode().strip(), isLast=True, project=project) projects.append(project) print(green('ok.')) status['Added'] += 1 except NotADirectoryError: print(red('repository not found.')) status['Repository not found'] += 1 except subprocess.CalledProcessError as ex: print(red('Git error.')) status['Git error'] += 1 if CODE_DEBUG: print(ex.stderr) status['Total'] = len(projects) print_results(status) commit() return sorted(projects, key=lambda item: (item.owner.lower(), item.name.lower()))
def main(): db.connect() print(f'Loading projects from {ANNOTATED_FILE}.') projects = get_or_create_projects() print(f'\nLoading heuristics from {HEURISTICS_DIR}.') labels = get_or_create_labels() # Indexing executions by label heuristic and project version. executions = index_executions(labels) status = { 'Success': 0, 'Skipped': 0, 'Repository not found': 0, 'Git error': 0, 'Total': len(labels) * len(projects) } print( f'\nProcessing {len(labels)} heuristics over {len(projects)} projects.' ) for i, label in enumerate(labels): heuristic = label.heuristic for j, project in enumerate(projects): version = project.versions[ 0] # TODO: fix this to deal with multiple versions # Print progress information progress = '{:.2%}'.format( (i * len(projects) + (j + 1)) / status['Total']) print( f'[{progress}] Searching for {label.name} in {project.owner}/{project.name}:', end=' ') # Try to get a previous execution execution = executions.get((heuristic, version), None) if not execution: try: os.chdir(REPOS_DIR + os.sep + project.owner + os.sep + project.name) cmd = GREP_COMMAND + [ HEURISTICS_DIR + os.sep + label.type + os.sep + label.name + '.txt' ] p = subprocess.run(cmd, capture_output=True) if p.stderr: raise subprocess.CalledProcessError( p.returncode, cmd, p.stdout, p.stderr) db.create(db.Execution, output=p.stdout.decode(errors='replace').replace( '\x00', '\uFFFD'), version=version, heuristic=heuristic, isValidated=False, isAccepted=False) print(green('ok.')) status['Success'] += 1 except NotADirectoryError: print(red('repository not found.')) status['Repository not found'] += 1 except subprocess.CalledProcessError as ex: print(red('Git error.')) status['Git error'] += 1 if CODE_DEBUG: print(ex.stderr) else: # Execution already exists print(yellow('already done.')) status['Skipped'] += 1 commit() print_results(status) db.close()
def get_or_create_labels(): labels = [] # Loading heuristics from the file system. labels_fs = dict() for label_type in os.scandir(HEURISTICS_DIR): if label_type.is_dir() and not label_type.name.startswith('.'): for label in os.scandir(label_type.path): if label.is_file() and not label.name.startswith('.'): with open(label.path) as file: pattern = file.read() label_fs = { 'name': os.path.splitext(label.name)[0], 'type': label_type.name, 'pattern': pattern, } labels_fs[(label_fs['type'], label_fs['name'])] = label_fs # Loading labels from the database. labels_db = db.query(db.Label).options( selectinload(db.Label.heuristic).options( selectinload( db.Heuristic.executions).defer('output').defer('user'))).all() status = { 'File System': len(labels_fs), 'Database': len(labels_db), 'Added': 0, 'Deleted': 0, 'Updated': 0, } i = 0 # Deleting labels that do not exist in the file system for label in labels_db: label_fs = labels_fs.pop((label.type, label.name), None) if label_fs is not None: # Print progress information i += 1 progress = '{:.2%}'.format(i / status['File System']) print(f'[{progress}] Adding label {label.type}/{label.name}:', end=' ') labels.append(label) # Check if pattern has changed and, in this case, remove executions that are not accepted and verified heuristic = label.heuristic if heuristic.pattern != label_fs['pattern']: heuristic.pattern = label_fs['pattern'] count = 0 for execution in list(heuristic.executions): if not (execution.isValidated and execution.isAccepted): # The commit does not invalidate the objects for performance reasons, # so we need to remove the relationships before deleting the execution. execution.heuristic = None execution.version = None db.delete(execution) count += 1 print( green(f'heuristic updated ({count} executions removed).')) status['Updated'] += 1 else: print(yellow('already done.')) else: db.delete(label) status['Deleted'] += 1 # Adding missing labels in the database for label_fs in labels_fs.values(): # Print progress information i += 1 progress = '{:.2%}'.format(i / status['File System']) print( f'[{progress}] Adding label {label_fs["type"]}/{label_fs["name"]}:', end=' ') label = db.create(db.Label, name=label_fs['name'], type=label_fs['type']) db.create(db.Heuristic, pattern=label_fs['pattern'], label=label, executions=[]) labels.append(label) print(green('ok.')) status['Added'] += 1 status['Total'] = len(labels) print_results(status) commit() return sorted(labels, key=lambda item: (item.type.lower(), item.name.lower()))
def main(): """ This program can be useful to fix collisions (due to data migrated from case insensitive to case sensitive file systems) and to update the workspace to the most recent version. """ print(f'Loading repositories from {ANNOTATED_FILE}.') info_repositories = pd.read_excel(ANNOTATED_FILE, keep_default_na=False) info_repositories = info_repositories[info_repositories.discardReason == ''].reset_index(drop=True) status = { 'Success': 0, 'Collided': 0, 'Repository not found': 0, 'Git error': 0, 'Total': len(info_repositories) } print(f'Resetting {status["Total"]} repositories...') for i, row in info_repositories.iterrows(): # Print progress information progress = '{:.2%}'.format(i / status["Total"]) print( f'[{progress}] Processing repository {row["owner"]}/{row["name"]}:', end=' ') try: target = REPOS_DIR + os.sep + row['owner'] + os.sep + row['name'] os.chdir(target) # Removes lock file, if they exist try: os.remove('.git/index.lock') except OSError: pass process = subprocess.run( ['git', 'config', 'core.precomposeunicode', 'false'], capture_output=True) if process.stderr: raise subprocess.CalledProcessError(process.stderr) cmd = ['git', 'reset', '--hard', '-q', 'origin/HEAD'] process = subprocess.run(cmd, capture_output=True) if process.stderr: raise subprocess.CalledProcessError(process.returncode, cmd, process.stdout, process.stderr) cmd = ['git', 'clean', '-d', '-f', '-x', '-q'] process = subprocess.run(cmd, capture_output=True) if process.stderr: raise subprocess.CalledProcessError(process.returncode, cmd, process.stdout, process.stderr) process = subprocess.run(['git', 'status', '-s'], capture_output=True) if process.stdout or process.stderr: print(yellow('collided.')) status['Collided'] += 1 if CODE_DEBUG: print(process.stdout) print(process.stderr) else: print(green('ok.')) status['Success'] += 1 except NotADirectoryError: print(red('repository not found.')) status['Repository not found'] += 1 except subprocess.CalledProcessError as ex: print(red('Git error.')) status['Git error'] += 1 if CODE_DEBUG: print(ex.stderr) print('\n*** Processing results ***') for k, v in status.items(): print(f'{k}: {v}') print("\nFinished.")
import numpy as np import os, sys, time import torch import options, util import model_pretrain print(util.yellow("=======================================================")) print( util.yellow( "main_pretrain.py (pretraining with AtlasNet reimplementation)")) print(util.yellow("=======================================================")) print(util.magenta("setting configurations...")) opt = options.set() with torch.cuda.device(opt.gpu): trainer = model_pretrain.Model(opt) trainer.load_dataset(opt) trainer.build_network(opt) trainer.setup_optimizer(opt) trainer.restore_checkpoint(opt) trainer.setup_visualizer(opt) print(util.yellow("======= TRAINING START =======")) trainer.time_start(opt) for ep in range(opt.from_epoch, opt.to_epoch): trainer.train_epoch(opt, ep) if (ep + 1) % 10 == 0: trainer.evaluate(opt, ep) if (ep + 1) % 50 == 0: trainer.save_checkpoint(opt, ep) print(util.yellow("======= TRAINING DONE ======="))
def dirtest2(): #util.green("Run wipetest before this, then launch this twice") dirtest() # set up some dirs/files for us to work with if os.fork() == 0: fs = SyncedFS('mg', 'test', verbose=True) util.yellow('FIRST try lock') fs.manual_lock('testdir/subdir') util.yellow('FIRST got lock') sleep(1) fs.unlock('testdir/subdir') util.yellow('FIRST released lock') fs.no_locks() else: fs = SyncedFS('mg', 'test', verbose=True) sleep(.2) util.yellow('SECOND try lock') fs.manual_lock('testdir/subdir/subdir/ss2') # will have to wait 1 sec util.yellow('SECOND got lock') fs.unlock('testdir/subdir/subdir/ss2') util.yellow('SECOND released lock') fs.no_locks()
def new(self): if self.fs.verbose: util.yellow('New sharedobject') with self.lock(): self.fs.save_nosplay( {}, self.abspath ) # save an empty dict at our path. This is our attribute dict.