def delete_simulation_files(sim_id,conf): """ Remove all files for given simulation. :param sim_id: the simulation id :param conf: configuration """ rm(simulation_paths(sim_id,conf).values())
def gen_imglst(names, prefix, train_file, test_file): os.chdir(prefix) imgs_train_l, imgs_test_l = [], [] for ind, cls in enumerate(names): if not osp.exists(cls): continue imgs = tf.gfile.Glob(cls + '/*.JPEG') if len(imgs) == 0: utils.rm(cls, True) continue assert len(imgs) >= 10 imgs = np.array(imgs) imgs_test = np.random.choice(imgs, max(3, imgs.shape[0] * 1 // 10), replace=False) imgs_train = np.setdiff1d(imgs, imgs_test) # imgs_train = imgs[:imgs.shape[0] * 9 // 10] # imgs_test = imgs[imgs.shape[0] * 9 // 10:] # imgs_test.shape, imgs_train.shape, imgs.shape imgs_train_l.append( np.stack((imgs_train, np.ones_like(imgs_train, dtype=int) * ind), axis=-1) ) imgs_test_l.append( np.stack((imgs_test, np.ones_like(imgs_test, dtype=int) * ind), axis=-1) ) imgs_train = np.concatenate(imgs_train_l, axis=0) np.random.shuffle(imgs_train) np.savetxt(train_file, imgs_train, delimiter=' ', fmt='%s') np.savetxt(test_file, np.concatenate(imgs_test_l, axis=0), delimiter=' ', fmt='%s')
def delete_simulation_files(sim_id, conf): """ Remove all files for given simulation. :param sim_id: the simulation id :param conf: configuration """ rm(simulation_paths(sim_id, conf).values())
def save(self, cur_loss, method, save_freq=None): """ Args: method: all / last all: save checkpoint by step last: save checkpoint to 'last.pth' all-last: save checkpoint by step per save_freq and save checkpoint to 'last.pth' always """ if method not in ['all', 'last', 'all-last']: return step_save = False last_save = False if method == 'all' or (method == 'all-last' and self.step % save_freq == 0): step_save = True if method == 'last' or method == 'all-last': last_save = True assert step_save or last_save save_dic = { 'generator': self.gen.state_dict(), 'generator_ema': self.gen_ema.state_dict(), 'optimizer': self.g_optim.state_dict(), 'epoch': self.step, 'loss': cur_loss } if self.disc is not None: save_dic['discriminator'] = self.disc.state_dict() save_dic['d_optimizer'] = self.d_optim.state_dict() if self.aux_clf is not None: save_dic['aux_clf'] = self.aux_clf.state_dict() save_dic['ac_optimizer'] = self.ac_optim.state_dict() ckpt_dir = self.cfg['work_dir'] / "checkpoints" / self.cfg[ 'unique_name'] step_ckpt_name = "{:06d}-{}.pth".format(self.step, self.cfg['name']) last_ckpt_name = "last.pth" step_ckpt_path = Path.cwd() / ckpt_dir / step_ckpt_name last_ckpt_path = ckpt_dir / last_ckpt_name log = "" if step_save: torch.save(save_dic, str(step_ckpt_path)) log = "Checkpoint is saved to {}".format(step_ckpt_path) if last_save: utils.rm(last_ckpt_path) last_ckpt_path.symlink_to(step_ckpt_path) log += " and symlink to {}".format(last_ckpt_path) if not step_save and last_save: utils.rm(last_ckpt_path) # last 가 symlink 일 경우 지우고 써줘야 함. torch.save(save_dic, str(last_ckpt_path)) log = "Checkpoint is saved to {}".format(last_ckpt_path) self.logger.info("{}\n".format(log))
def test_dir(self): with tempfile.TemporaryDirectory() as tmpdir: tmpdir = Path(tmpdir) (tmpdir / 'c').mkdir() (tmpdir / 'c' / 'a.txt').write_text("asdf_content") (tmpdir / 'c' / 'b.txt').write_text("bsdf_content") rm(tmpdir / 'c') self.assertFalse((tmpdir / 'c').exists())
def on_delete_buf(self, data): # TODO: somehow tell the user about this. maybe delete on disk too? del self.FLOO_BUFS[data["id"]] path = utils.get_full_path(data["path"]) if not G.DELETE_LOCAL_FILES: msg.log("Not deleting %s because delete_local_files is disabled" % path) return utils.rm(path) msg.warn("deleted %s because %s told me to." % (path, data.get("username", "the internet")))
def on_delete_buf(self, data): # TODO: somehow tell the user about this. maybe delete on disk too? del self.FLOO_BUFS[data['id']] path = utils.get_full_path(data['path']) if not G.DELETE_LOCAL_FILES: msg.log('Not deleting %s because delete_local_files is disabled' % path) return utils.rm(path) msg.warn('deleted %s because %s told me to.' % (path, data.get('username', 'the internet')))
def save(self, i): # 우선 저장 fpath = join(self.fdir, "{}-{}.{}".format(self.fname, i, self.fext)) # full path mkdir(self.fdir) torch.save(self.model.state_dict(), fpath) # 현재 폴더에 저장된 같은 종류의 모델 확인 files, idxs = ls_model(self.fdir, self.fname, self.fext) # 저장이 max_to_keep 보다 많으면 가장 오래된것 삭제 if self.max_to_keep < len(files): rm(files[0])
def update_backup(backup: Optional[str], i: int, time_stamp: str) -> str: tmp = save_dir + '{0}_{1}_{2}.pickle'.format(log_title, i, time_stamp) training_backup(net, optimizer, tmp, optim_kwargs) if backup is not None: if not rm(backup): print('Failed to delete {0}'.format(backup)) return tmp
def rename(self, name): msg.debug("renaming %s to %s" % (self.vim_buf.name, name)) current = vim.current.buffer text = self.get_text() old_name = self.vim_buf.name old_number = self.native_id with open(name, "wb") as fd: fd.write(text.encode("utf-8")) vim.command("edit! %s" % name) self.vim_buf = vim.current.buffer vim.command("edit! %s" % current.name) vim.command("bdelete! %s" % old_number) try: utils.rm(old_name) except Exception as e: msg.debug("couldn't delete %s... maybe thats OK?" % str(e))
def rename(self, name): msg.debug('renaming %s to %s' % (self.vim_buf.name, name)) current = vim.current.buffer text = self.get_text() old_name = self.vim_buf.name old_number = self.native_id with open(name, 'wb') as fd: fd.write(text.encode('utf-8')) vim.command('edit! %s' % name) self.vim_buf = vim.current.buffer vim.command('edit! %s' % current.name) vim.command('bdelete! %s' % old_number) try: utils.rm(old_name) except Exception as e: msg.debug("couldn't delete %s... maybe thats OK?" % str(e))
def check_individual(filename, filepath): if filename.endswith('JPEG') and osp.getsize(filepath) == 0: utils.rm(filepath) append_file(filepath, file='/home/wangxinglu/prj/few-shot/src/append.txt') return im = cv2.imread(filepath, cv2.CV_LOAD_IMAGE_COLOR) cv2.imwrite(filepath, im, [int(cv2.IMWRITE_JPEG_QUALITY), 100]) if im.shape[0] <= 10 or im.shape[1] <= 10: # utils.rm(filepath) append_file(filepath, './wrong.img.txt') print filepath assert im.shape[-1] == 3 assert im.shape[0] > 13 and im.shape[1] > 13 assert (len(im.shape) == 3 and im.shape[-1] == 3), 'img ' + filepath + str(im.shape) assert im.shape[1] != 0 and im.shape[0] != 0, 'width ' + filepath + str( im.shape)
def main(unused_argv): assert not FLAGS.train_shards % FLAGS.num_threads, ( 'Please make the FLAGS.num_threads commensurate with FLAGS.train_shards') assert not FLAGS.validation_shards % FLAGS.num_threads, ( 'Please make the FLAGS.num_threads commensurate with ' 'FLAGS.validation_shards') print('Saving results to %s' % FLAGS.output_directory) with tf.device('/cpu:0'): # Build a map from synset to human-readable label. synset_to_human = _build_synset_lookup(FLAGS.imagenet_metadata_file) # Run it! utils.rm(FLAGS.output_directory + '/train*', block=True) _process_dataset('train', FLAGS.validation_directory, FLAGS.validation_shards, synset_to_human) utils.rm(FLAGS.output_directory + '/validation*', block=True) _process_dataset('validation', FLAGS.train_directory, FLAGS.train_shards, synset_to_human)
def get_diffs(base1, base2, code, zoom): old = get_geojson(join(base1, code), zoom) tmp = get_geojson(join(base2, code), zoom) changed = [] for x in tmp: xtmp = join(base2, code, x) xold = join(base1, code, x) if x in old: if open(xtmp).read() != open(xold).read(): changed.append(x) cp(xtmp, xold) else: changed.append(x) cp(xtmp, xold) for x in old: if x not in tmp: xold = join(base1, code, x) changed.append(x) rm(xold) return changed
def get_diffs(base1,base2,code,zoom): old = get_geojson(join(base1,code),zoom) tmp = get_geojson(join(base2,code),zoom) changed = [] for x in tmp: xtmp = join(base2,code,x) xold = join(base1,code,x) if x in old: if open(xtmp).read() != open(xold).read(): changed.append(x) cp(xtmp,xold) else: changed.append(x) cp(xtmp,xold) for x in old: if x not in tmp: xold = join(base1,code,x) changed.append(x) rm(xold) return changed
def merge_json(M,base,out): print "BO",base,out if True: NEWM = {} check = set() for path in M: if not M[path]: ps = path.split("/") newpath = "/".join(ps[1:]) check.add(newpath) for code in M[path]: ps = path.split("/") newpath = "/".join(ps[1:]) newcode = code+"/"+ps[0] NEWM.setdefault(newpath,[]).append(newcode) for x in check: if x not in NEWM: NEWM[x] = [] M = NEWM for k in M: new = join(base,out,k) makedirs(dirname(new)) if len(M[k]) == 0: rm(new) elif len(M[k]) == 1: old = join(base,M[k][0],k) cp(old,new) else: print "K",k,"MK",M[k][:10] gs = [(x,geojson.load(open(join(base,x,k)))) for x in M[k]] x0,g = gs[0] for x,h in gs[1:]: print "merging",x0,x g["features"] += h["features"] f = open(new,"w") geojson.dump(g,f) f.close()
def merge_json(M, base, out): print "BO", base, out if True: NEWM = {} check = set() for path in M: if not M[path]: ps = path.split("/") newpath = "/".join(ps[1:]) check.add(newpath) for code in M[path]: ps = path.split("/") newpath = "/".join(ps[1:]) newcode = code + "/" + ps[0] NEWM.setdefault(newpath, []).append(newcode) for x in check: if x not in NEWM: NEWM[x] = [] M = NEWM for k in M: new = join(base, out, k) makedirs(dirname(new)) if len(M[k]) == 0: rm(new) elif len(M[k]) == 1: old = join(base, M[k][0], k) cp(old, new) else: print "K", k, "MK", M[k][:10] gs = [(x, geojson.load(open(join(base, x, k)))) for x in M[k]] x0, g = gs[0] for x, h in gs[1:]: print "merging", x0, x g["features"] += h["features"] f = open(new, "w") geojson.dump(g, f) f.close()
def write(self , tfile , package=None): """ Arguments: - `self`: """ if package == None: package = self._package if package == '*': utils.rm(tfile) os.mkdir(tfile) for i in self._js: self.write(os.path.basename(tfile) + os.sep + i + '.js' , i) #self.jsdoc( tfile ) self.addboot( tfile ) else: f = codecs.open(tfile , 'w' , self._encoding) f.write(self.getFile(package)) f.close()
def clean(self): """ 清空构建目录 """ print("start clean output dir") rm(os.path.join(self.local_dist_dir, "*")) rm(os.path.join(CUR_DIR, "__pycache__")) rm(os.path.join(CUR_DIR, "*.pyc")) print("end") return 0
def _process_image(filename, coder): """Process a single image file. Args: filename: string, path to an image file e.g., '/path/to/example.JPG'. coder: instance of ImageCoder to provide TensorFlow image coding utils. Returns: image_buffer: string, JPEG encoding of RGB image. height: integer, image height in pixels. width: integer, image width in pixels. """ # Read the image file. try: image_data = tf.gfile.FastGFile(filename, 'r').read() # Clean the dirty data. if _is_png(filename): utils.rm(filename) return # 1 image is a PNG. print('Converting PNG to JPEG for %s' % filename) image_data = coder.png_to_jpeg(image_data) elif _is_cmyk(filename): utils.rm(filename) return # 22 JPEG images are in CMYK colorspace. print('Converting CMYK to RGB for %s' % filename) image_data = coder.cmyk_to_rgb(image_data) # Decode the RGB JPEG. image = coder.decode_jpeg(image_data) except Exception as inst: utils.rm(filename) tf.logging.error(inst) tf.logging.error(filename) append_file(filename) # raise ValueError('rm file') # Check that image converted to RGB assert len(image.shape) == 3 height = image.shape[0] width = image.shape[1] assert image.shape[2] == 3 return image_data, height, width
def main(argv): # Extract command line options try: opts, args = getopt.gnu_getopt(argv[1:], 'stpSdi:nhz', [ "hdsafe", "onlyts", "onlyxy", "npar", "imas=", "strike=", "dc", "nont", "dip=", "rake=", "mom=", "noref", "xyz", "old", "help" ]) except getopt.GetoptError as err: usage() raise # Parse command line options i_master = IMASTER fastflag = True flagts = True flagxy = True flagxyz = False flagref = True sdrM0 = {} for o, a in opts: if o == '-h' or o == '--help': disphelp() sys.exit(0) if o == '-s' or o == '--hdsafe': fastflag = False if o == '-t' or o == '--onlyts': if not flagts: usage() raise getopt.GetoptError( 'options -t and -p cannot be used simultaneously') flagxy = False flagts = True if o == '-p' or o == '--onlyxy': if not flagxy: usage() raise getopt.GetoptError( 'options -t and -p cannot be used simultaneously') flagts = False fastflag = False flagxy = True if o == '--dc': sdrM0['-dc'] = '' if o == '--nont': sdrM0['-nont'] = '' if o == '--strike': sdrM0['-strike'] = a if o == '--dip': sdrM0['-dip'] = a if o == '--rake': sdrM0['-rake'] = a if o == '--mom': sdrM0['-mom'] = a if o == '-i' or o == '--imas': i_master = a if o == '-n' or o == '--noref': flagref = False if o == '-z' or o == '--xyz': flagxyz = True if o == '--old': WPINV_XY += ' -old' # Read i_master iconfig = utils.parseConfig(i_master) cmtref = iconfig['CMTFILE'] evname = iconfig['EVNAME'].replace(' ', '_').replace(',', '') # Set comments in output ps file Median = '-med ' if 'P2P_SCREENING' in iconfig: p2p_items = iconfig['P2P_SCREENING'].split() if p2p_items[0] != 'YES': Median = ' ' elif len(p2p_items) > 1: for p in p2p_items[1:]: Median += p + ' ' ths = '-th 5.0 3.0 0.9' if 'RMS_SCREENING' in iconfig: ths = '-th ' + iconfig['RMS_SCREENING'] comments = [VERSION, 'GF_PATH: ' + GF_PATH, 'Screening: ' + Median + ths] # Read reference CMTFILE eq = EarthQuake() eq.rcmtfile(cmtref) eq.title = evname.strip().replace(' ', '_').replace(',', '') cmtf = open(cmtref, 'r') L = cmtf.readlines() cmtf.close() if len(L) < 13: print('*** WARNING : no reference solution in %s' % (cmtref)) flagref = False # TS and/or LAT/LON Grid-search if (flagts or flagxy) and not flagxyz: grid_search(eq, cmtref, TS_NIT, TS_DT, TSBOUNDS, XY_NIT, XY_DX, XY_NX, XY_NOPT, fastflag, flagts, flagxy, sdrM0, ts_ofile=TS_OFILE, xy_ofile=XY_OFILE, comments=comments) # TS and LAT/LON/DEP Grid-search if flagxyz: grid_search(eq, cmtref, TS_NIT, TS_DT, TSBOUNDS, XYZ_NIT, XYZ_DX, XYZ_NX, XYZ_NOPT, fastflag, flagts, flagxyz, sdrM0, dz=DDEP, minz=MINDEP, ts_ofile=TS_OFILE, xy_ofile=XYZ_OFILE, comments=comments) if flagxy: eq.wcmtfile('_tmp_CMTSOLUTION.xyz') if flagref: addrefsol(cmtref, '_tmp_CMTSOLUTION.xyz') grid_search(eq, '_tmp_CMTSOLUTION.xyz', TS_NIT, TS_DT, TSBOUNDS, XY_NIT, XY_DX, XY_NX, XY_NOPT, 0, 0, 1, sdrM0, ts_ofile=TS_OFILE, xy_ofile=XY_OFILE, comments=comments) utils.rm('_tmp_CMTSOLUTION.xyz') # Cleaning up if os.path.exists('_tmp_ts_table'): utils.rm('_tmp_ts_table') if os.path.exists('_tmp_xy_table'): utils.rm('_tmp_xy_table')
def on_delete_buf(self, data): # TODO: somehow tell the user about this. maybe delete on disk too? del self.FLOO_BUFS[data['id']] path = utils.get_full_path(data['path']) utils.rm(path) msg.warn('deleted %s because %s told me to.' % (path, data.get('username', 'the internet')))
def protocol(self, req): self.buf += req.decode('utf-8') msg.debug('buf: %s' % self.buf) while True: before, sep, after = self.buf.partition('\n') if not sep: break try: data = json.loads(before) except Exception as e: msg.error('Unable to parse json: %s' % str(e)) msg.error('Data: %s' % before) raise e name = data.get('name') if name == 'patch': # TODO: we should do this in a separate thread Listener.apply_patch(data) elif name == 'get_buf': buf_id = data['id'] listener.BUFS[buf_id] = data view = listener.get_view(buf_id) if view: Listener.update_view(data, view) else: listener.save_buf(data) elif name == 'create_buf': listener.BUFS[data['id']] = data listener.save_buf(data) elif name == 'rename_buf': new = utils.get_full_path(data['path']) old = utils.get_full_path(data['old_path']) new_dir = os.path.split(new)[0] if new_dir: utils.mkdir(new_dir) os.rename(old, new) view = listener.get_view(data['id']) if view: view.retarget(new) elif name == 'delete_buf': path = utils.get_full_path(data['path']) try: utils.rm(path) except Exception: pass listener.delete_buf(data['id']) elif name == 'room_info': # Success! Reset counter self.retries = G.MAX_RETRIES self.room_info = data G.PERMS = data['perms'] if 'patch' not in data['perms']: msg.log( 'We don\'t have patch permission. Setting buffers to read-only' ) project_json = {'folders': [{'path': G.PROJECT_PATH}]} utils.mkdir(G.PROJECT_PATH) with open(os.path.join(G.PROJECT_PATH, '.sublime-project'), 'wb') as project_fd: project_fd.write( json.dumps(project_json, indent=4, sort_keys=True).encode('utf-8')) floo_json = { 'url': utils.to_room_url({ 'host': self.host, 'owner': self.owner, 'port': self.port, 'room': self.room, 'secure': self.secure, }) } with open(os.path.join(G.PROJECT_PATH, '.floo'), 'w') as floo_fd: floo_fd.write( json.dumps(floo_json, indent=4, sort_keys=True)) for buf_id, buf in data['bufs'].items(): buf_id = int(buf_id) # json keys must be strings buf_path = utils.get_full_path(buf['path']) new_dir = os.path.dirname(buf_path) utils.mkdir(new_dir) listener.BUFS[buf_id] = buf try: buf_fd = open(buf_path, 'rb') buf_buf = buf_fd.read().decode('utf-8') md5 = hashlib.md5(buf_buf.encode('utf-8')).hexdigest() if md5 == buf['md5']: msg.debug('md5 sums match. not getting buffer') buf['buf'] = buf_buf else: msg.debug( 'md5 for %s should be %s but is %s. getting buffer' % (buf['path'], buf['md5'], md5)) raise Exception('different md5') except Exception as e: msg.debug('Error calculating md5:', e) Listener.get_buf(buf_id) self.authed = True G.CONNECTED = True msg.log('Successfully joined room %s/%s' % (self.owner, self.room)) if self.on_connect: self.on_connect(self) self.on_connect = None elif name == 'join': msg.log('%s joined the room' % data['username']) self.room_info['users'][data['user_id']] = data['username'] elif name == 'part': msg.log('%s left the room' % data['username']) try: del self.room_info['users'][data['user_id']] except Exception as e: print('Unable to delete user %s from user list' % (data)) region_key = 'floobits-highlight-%s' % (data['user_id']) for window in sublime.windows(): for view in window.views(): view.erase_regions(region_key) elif name == 'highlight': region_key = 'floobits-highlight-%s' % (data['user_id']) Listener.highlight(data['id'], region_key, data['username'], data['ranges'], data.get('ping', False)) elif name == 'error': message = 'Floobits: Error! Message: %s' % str(data.get('msg')) msg.error(message) elif name == 'disconnect': message = 'Floobits: Disconnected! Reason: %s' % str( data.get('reason')) msg.error(message) sublime.error_message(message) self.stop() elif name == 'msg': self.on_msg(data) else: msg.debug('unknown name!', name, 'data:', data) self.buf = after
# parser for getpath tool parser_getpath = subparsers.add_parser('where', help='Display the path in which datasets are installed.') parser_getpath.add_argument('--config', dest='config', metavar='STRING', required=False, help='Absolute location to a specific config file') parser_getpath.set_defaults(func=utils.get_install_path) # parse the args and call the selected function args = parser.parse_args() # so we don't have multiple things running and writing data at the same # time. utils.check_set_lock(LOCK_PATH) try: args.func(args) except IOError, e: if e.errno != 32: # ignore SIGPIPE raise finally: utils.rm(LOCK_PATH) if __name__ == "__main__": main()
def eval_ckpt(): from train import (setup_language_dependent, setup_data, setup_cv_dset_loader, get_dset_loader) logger = Logger.get() parser = argparse.ArgumentParser('MaHFG-eval') parser.add_argument( "name", help= "name is used for directory name of the user-study generation results") parser.add_argument("resume") parser.add_argument("img_dir") parser.add_argument("config_paths", nargs="+") parser.add_argument("--show", action="store_true", default=False) parser.add_argument( "--mode", default="eval", help="eval (default) / cv-save / user-study / user-study-save. " "`eval` generates comparable grid and computes pixel-level CV scores. " "`cv-save` generates and saves all target characters in CV. " "`user-study` generates comparable grid for the ramdomly sampled target characters. " "`user-study-save` generates and saves all target characters in user-study." ) parser.add_argument("--deterministic", default=False, action="store_true") parser.add_argument("--debug", default=False, action="store_true") args, left_argv = parser.parse_known_args() cfg = Config(*args.config_paths) cfg.argv_update(left_argv) torch.backends.cudnn.benchmark = True cfg['data_dir'] = Path(cfg['data_dir']) if args.show: exit() # seed np.random.seed(cfg['seed']) torch.manual_seed(cfg['seed']) random.seed(cfg['seed']) if args.deterministic: torch.backends.cudnn.benchmark = False torch.backends.cudnn.deterministic = True cfg['n_workers'] = 0 logger.info("#" * 80) logger.info("# Deterministic option is activated !") logger.info( "# Deterministic evaluator only ensure the deterministic cross-validation" ) logger.info("#" * 80) else: torch.backends.cudnn.benchmark = True if args.mode.startswith('mix'): assert cfg['g_args']['style_enc']['use'], \ "Style mixing is only available with style encoder model" ##################################### # Dataset #################################### # setup language dependent values content_font, n_comp_types, n_comps = setup_language_dependent(cfg) # setup transform transform = transforms.Compose( [transforms.ToTensor(), transforms.Normalize([0.5], [0.5])]) # setup data hdf5_data, meta = setup_data(cfg, transform) # setup dataset trn_dset, loader = get_dset_loader(hdf5_data, meta['train']['fonts'], meta['train']['chars'], transform, True, cfg, content_font=content_font) val_loaders = setup_cv_dset_loader(hdf5_data, meta, transform, n_comp_types, content_font, cfg) ##################################### # Model #################################### # setup generator only g_kwargs = cfg.get('g_args', {}) gen = MACore(1, cfg['C'], 1, **g_kwargs, n_comps=n_comps, n_comp_types=n_comp_types, language=cfg['language']) gen.cuda() ckpt = torch.load(args.resume) logger.info("Use EMA generator as default") gen.load_state_dict(ckpt['generator_ema']) step = ckpt['epoch'] loss = ckpt['loss'] logger.info("Resumed checkpoint from {} (Step {}, Loss {:7.3f})".format( args.resume, step, loss)) writer = utils.DiskWriter(args.img_dir, 0.6) evaluator = Evaluator(hdf5_data, trn_dset.avails, logger, writer, cfg['batch_size'], content_font=content_font, transform=transform, language=cfg['language'], val_loaders=val_loaders, meta=meta) evaluator.n_cv_batches = -1 logger.info("Update n_cv_batches = -1 to evaluate about full data") if args.debug: evaluator.n_cv_batches = 10 logger.info("!!! DEBUG MODE: n_cv_batches = 10 !!!") if args.mode == 'eval': logger.info("Start validation ...") dic = evaluator.validation(gen, step) logger.info("Validation is done. Result images are saved to {}".format( args.img_dir)) elif args.mode.startswith('user-study'): meta = json.load(open('meta/kor-unrefined.json')) target_chars = meta['target_chars'] style_chars = meta['style_chars'] fonts = meta['fonts'] if args.mode == 'user-study': sampled_target_chars = uniform_sample(target_chars, 20) logger.info("Start generation kor-unrefined ...") logger.info("Sampled chars = {}".format(sampled_target_chars)) evaluator.handwritten_validation_2stage(gen, step, fonts, style_chars, sampled_target_chars, comparable=True, tag='userstudy-{}'.format( args.name)) elif args.mode == 'user-study-save': logger.info("Start generation & saving kor-unrefined ...") save_dir = Path(args.img_dir) / "{}-{}".format(args.name, step) evaluator.handwritten_validation_2stage(gen, step, fonts, style_chars, target_chars, comparable=True, save_dir=save_dir) logger.info("Validation is done. Result images are saved to {}".format( args.img_dir)) elif args.mode == 'cv-save': save_dir = Path(args.img_dir) / "cv_images_{}".format(step) logger.info("Save CV results to {} ...".format(save_dir)) utils.rm(save_dir) for tag, loader in val_loaders.items(): l1, ssim, msssim = evaluator.cross_validation( gen, step, loader, tag, n_batches=evaluator.n_cv_batches, save_dir=(save_dir / tag)) else: raise ValueError(args.mode)
def apply(self, hot_dir: os.PathLike, cold_dir: os.PathLike, index: Index): rm(cold_dir / self.name) cp(hot_dir / self.name, cold_dir / self.name) index[self.name] = self.index
def main_simulate(): parser = argparse.ArgumentParser(description='This script use Flex to simulate cloth dynamics.') # Global parser.add_argument('--randomSeed', type=int, default=14, help='Fix the random seed.') parser.add_argument('--flex_verbose', type=int, default=0, help='0 to inhibit printings from FleX') # Experiments parser.add_argument('--scenes_str', type=str, default="wind", help='Scene name [If there are multiple scenes, use --scenes_list]') parser.add_argument('--scenes_list', type=list, default=[], help='a list of scene names [If there is only one scecne, use --scenes_str]') parser.add_argument('--bstiff_float', type=float, default=1.0, help='bend_stiffness [If there are multiple bs values, use --bstiff_list]') parser.add_argument('--bstiff_list', type=list, default=[], help='a list of bend_stiffness values [If there is only one bs value, use --bstiff_float]') # Flex parser.add_argument('--flexRootPath', type=str, default="FleX/", help='path to FleX root directory') parser.add_argument('--flexConfig', type=str, default="FleXConfigs/flexConfig.yml", help='FleX config file') parser.add_argument('--flex_input_root_path', type=str, default="dataset/trialObjs/", help='root path to FleX input .obj file') parser.add_argument('--obj_input', type=str, default="model1.obj", help='name of the FleX input .obj file') parser.add_argument('--flex_output_root_path', type=str, default="experiments/simulation/trialObjs/", help='root path to FleX output .obj files') # Flex simulation paras parser.add_argument('--windstrength', type=float, default=0.3, help='wind strength') parser.add_argument('--scale', type=float, default=1.0, help='object scale') parser.add_argument('--rot', default=(0,0,0), help='Rotation of the object.') parser.add_argument('--local', type=bool, default=True, help='run Flex locally') #parser.add_argument('--clothColor', type=str, default="violet", help='') parser.add_argument('--floor', type=bool, default=False, help='') parser.add_argument('--offline', type=bool, default=True, help='') #parser.add_argument('--occluded', type=bool, default=True, help='') parser.add_argument('--useQuat', type=bool, default=False, help='') parser.add_argument('--psize', type=float, default=0.01, help='') parser.add_argument('--clothNumParticles', type=int, default=210, help='') parser.add_argument('--flexrandomSeed', type=int, default=-1, help='') parser.add_argument('--visSaveClothPerSimStep', type=bool, default=False, help='') parser.add_argument('--randomClothMinRes', type=int, default=145, help='') parser.add_argument('--randomClothMaxRes', type=int, default=215, help='') parser.add_argument('--mass', type=float, default=-1.0, help='invMass [Set -1 to use the value defined in --flexConfig]') parser.add_argument('--shstiff', type=float, default=-1.0, help='shear stiffness [Set -1 to use the value defined in --flexConfig]') parser.add_argument('--ststiff', type=float, default=-1.0, help='stretch stiffnessetch [Set -1 to use the value defined in --flexConfig]') parser.add_argument('--clothLift', type=float, default=-1.0, help='') parser.add_argument('--clothDrag', type=float, default=-1.0, help='') parser.add_argument('--clothFriction', type=float, default=1.1, help='') #parser.add_argument('--outputPath', type=str, default="", help='') parser.add_argument('--saveClothPerSimStep', type=int, default=1, help='') args = parser.parse_args() #print(args) # ---------------------------- np.random.seed(args.randomSeed) random.seed(args.randomSeed) torch.manual_seed(args.randomSeed) torch.cuda.manual_seed(args.randomSeed) # ---------------------------- # If there are multiple scenes if len(args.scenes_list) != 0: scenes = args.scenes_list else: scenes = [args.scenes_str] # If there are multiple bstiff values if len(args.bstiff_list) != 0: bstiff = [float(i) for i in args.bstiff_list] else: bstiff = [float(args.bstiff_float)] shstiff=float(args.shstiff) ststiff=float(args.ststiff) mass=float(args.mass) # Simulate .obj files for each scene for tmp_scene in scenes: objPath = os.path.join(args.flex_input_root_path, tmp_scene, args.obj_input) # ----- Setup environment ----- # FLEX_BIN_PATH = os.path.join(args.flexRootPath, "bin", "linux64", "NvFlexDemoReleaseCUDA_x64_" + tmp_scene) if not os.path.exists(FLEX_BIN_PATH): errorMessage="==> Error: No FleX binary found. Make sure you have set the right path and compiled FleX.\nTo compile FleX: singularity exec --nv kimImage.simg bash buildFleX.sh --build=true" sys.exit(errorMessage) else: os.environ["FLEX_PATH"] = args.flexRootPath # For libGLEW.so.1.10 os.environ["LD_LIBRARY_PATH"] += ":{}".format(os.path.join(args.flexRootPath, "external")) # ----- End Setup environment ----- # for i in bstiff: ''' Params: rot: Euler angles (in degrees) or quaternion (max unit norm) ''' # e.g., "experiments/renderings/trialObjs/wind/wind" The last wind is the base name for the output .obj files outputPath = os.path.join(args.flex_output_root_path, tmp_scene+"_mass_"+str(mass)+"_bs_"+str(i)+"_sh_"+str(shstiff)+"_st_"+str(ststiff), tmp_scene) outputFolder = ('/').join(outputPath.split('/')[0:-1]) rm(outputFolder) mkdirs(outputFolder) sim_cmd = [FLEX_BIN_PATH, "-obj={}".format(os.path.abspath(objPath)), "-config={}".format(os.path.abspath(args.flexConfig)), "-export={}".format(os.path.abspath(outputPath)), #"-iters={}".format(n_frames), #"-ss={}".format(n_substeps_per_frame), # rotate not args.useQuat and "-rx={} -ry={} -rz={}".format(*args.rot) or "-rx={} -ry={} -rz={} -rw={}".format(*args.rot), # cloth properties #"-stiff_scale={}".format(stiffness), # "-ccolor_id={}".format(COLOR_MAP[clothColor]), ] if not args.floor: sim_cmd.append("-nofloor") if args.offline: sim_cmd.append("-offline") # if not occluded: # sim_cmd.append("-clothsize=1") # sim_cmd.append("-cam_dist={}".format(4.5)) if args.useQuat: sim_cmd.append("-use_quat") else: sim_cmd.append("-use_euler") if args.visSaveClothPerSimStep: sim_cmd.append("-saveClothPerSimStep") if args.flex_verbose: sim_cmd.append("-v") sim_cmd.append("-bstiff={0:f}".format(i)) sim_cmd.append("-psize={0:f}".format(args.psize)) sim_cmd.append("-clothsize={0:d}".format(args.clothNumParticles)) sim_cmd.append("-randomSeed={0:d}".format(args.flexrandomSeed)) sim_cmd.append("-randomClothMinRes={0:d}".format(args.randomClothMinRes)) sim_cmd.append("-randomClothMaxRes={0:d}".format(args.randomClothMaxRes)) sim_cmd.append("-clothLift={0:f}".format(args.clothLift)) sim_cmd.append("-scale={0:f}".format(args.scale)) sim_cmd.append("-windstrength={0:f}".format(args.windstrength)) sim_cmd.append("-mass={0:f}".format(float(args.mass))) sim_cmd.append("-shstiff={0:f}".format(float(args.shstiff))) sim_cmd.append("-ststiff={0:f}".format(float(args.ststiff))) sim_cmd.append("-clothDrag={0:f}".format(args.clothDrag)) sim_cmd.append("-clothFriction={0:f}".format(args.clothFriction)) sim_cmd.append("-saveClothPerSimStep={0:d}".format(args.saveClothPerSimStep)) env = {} if args.local: env["DISPLAY"] = ":0" else: sim_cmd.insert(0, "vglrun") #print("---- Output Folder: ", outputFolder) # save params with open ((outputPath+'_main_simulate_paras.txt'), "w") as txt_file: for line in sim_cmd: txt_file.write("".join(line) + "\n") lines_stdout = runCmd(" ".join(sim_cmd), extra_vars=env, verbose=args.flex_verbose) if (len(lines_stdout) < 50): print(lines_stdout, file=sys.stderr) sys.exit(1)
def apply(self, hot_dir: os.PathLike, cold_dir: os.PathLike, index: Index): rm(cold_dir / self.name)
def convert_worker(target_format, message, url, config, bot): """Generic process spawned every time user sends a link or a file""" input_filename = "".join([config["temp_path"], utils.random_string()]) output_filename = "".join( [config["temp_path"], utils.random_string(), ".", target_format]) # Tell user that we are working status_message = bot.reply_to(message, text.starting, parse_mode="HTML") def update_status_message(new_text): bot.edit_message_text(chat_id=status_message.chat.id, message_id=status_message.message_id, text=new_text, parse_mode="HTML") # Try to download URL try: r = requests.get(url, stream=True) except: update_status_message(text.error.downloading) return # Check file size if int(r.headers.get("Content-Length", "0")) >= MAXIMUM_FILESIZE_ALLOWED: update_status_message(text.error.huge_file) return # Download the file update_status_message(text.downloading) chunk_size = 4096 raw_input_size = 0 try: with open(input_filename, "wb") as f: for chunk in r.iter_content(chunk_size=chunk_size): f.write(chunk) raw_input_size += chunk_size # Download files without Content-Length, but apply standard limit to them if raw_input_size >= MAXIMUM_FILESIZE_ALLOWED: update_status_message(text.error.huge_file) utils.rm(input_filename) return except: update_status_message(text.error.downloading) bot.reply_to(message, f"HTTP {r.status_code}") return # Start ffmpeg ffmpeg_process = None if target_format == "mp4": ffmpeg_process = subprocess.Popen([ "ffmpeg", "-v", "error", "-threads", str(config["ffmpeg_threads"]), "-i", input_filename, "-map", "V:0?", # select video stream "-map", "0:a?", # ignore audio if doesn't exist "-c:v", "libx264", # specify video encoder "-max_muxing_queue_size", "9999", # https://trac.ffmpeg.org/ticket/6375 "-movflags", "+faststart", # optimize for streaming "-preset", "veryslow", # https://trac.ffmpeg.org/wiki/Encode/H.264#a2.Chooseapresetandtune "-timelimit", "900", # prevent DoS (exit after 15 min) "-vf", "pad=ceil(iw/2)*2:ceil(ih/2)*2", # https://stackoverflow.com/questions/20847674/ffmpeg-libx264-height-not-divisible-by-2#20848224 output_filename, ]) elif target_format == "png": ffmpeg_process = subprocess.Popen([ "ffmpeg", "-v", "error", "-threads", str(config["ffmpeg_threads"]), "-thread_type", "slice", "-i", input_filename, "-timelimit", "60", # prevent DoS (exit after 15 min) output_filename, ]) # Update progress while ffmpeg is alive old_progress = "" while ffmpeg_process.poll() == None: try: raw_output_size = utils.filesize(output_filename) except FileNotFoundError: raw_output_size = 0 if raw_output_size >= MAXIMUM_FILESIZE_ALLOWED: update_status_message(text.error.huge_file) ffmpeg_process.kill() utils.rm(output_filename) input_size = utils.bytes2human(raw_input_size) output_size = utils.bytes2human(raw_output_size) progress = f"{output_size} / {input_size}" # Update progress only if it changed if progress != old_progress: update_status_message(text.converting.format(progress)) old_progress = progress time.sleep(2) # Exit in case of error with ffmpeg if ffmpeg_process.returncode != 0: update_status_message(text.error.converting) # Clean up and close pipe explicitly utils.rm(output_filename) return # Check output file size output_size = utils.filesize(output_filename) if output_size >= MAXIMUM_FILESIZE_ALLOWED: update_status_message(text.error.huge_file) # Clean up and close pipe explicitly utils.rm(output_filename) return # Default params for sending operation data = { "chat_id": message.chat.id, "reply_to_message_id": message.message_id } if target_format == "mp4": data.update({"supports_streaming": True}) # 1. Get video duration in seconds video_duration = subprocess.run( [ "ffprobe", "-v", "error", "-select_streams", "v:0", "-show_entries", "format=duration", "-of", "default=noprint_wrappers=1:nokey=1", output_filename, ], stdout=subprocess.PIPE, ).stdout.decode("utf-8").strip() video_duration = round(float(video_duration)) data.update({"duration": video_duration}) # 2. Get video height and width video_props = subprocess.run( [ "ffprobe", "-v", "error", "-select_streams", "v:0", "-show_entries", "stream=width,height", "-of", "csv=s=x:p=0", output_filename, ], stdout=subprocess.PIPE, ).stdout.decode("utf-8").strip() video_width, video_height = video_props.split("x") data.update({"width": video_width, "height": video_height}) # 3. Take one frame from the middle of the video update_status_message(text.generating_thumbnail) thumbnail = "".join( [config["temp_path"], utils.random_string(), ".jpg"]) generate_thumbnail_process = subprocess.Popen([ "ffmpeg", "-v", "error", "-i", output_filename, "-vcodec", "mjpeg", "-vframes", "1", "-an", "-f", "rawvideo", "-ss", str(int(video_duration / 2)), # keep the limit of 90px height/width (Telegram API) while preserving the aspect ratio "-vf", "scale='if(gt(iw,ih),90,trunc(oh*a/2)*2)':'if(gt(iw,ih),trunc(ow/a/2)*2,90)'", thumbnail, ]) # While process is alive (i.e. is working) while generate_thumbnail_process.poll() == None: time.sleep(1) # Exit in case of error with ffmpeg if generate_thumbnail_process.returncode != 0: update_status_message(text.error.generating_thumbnail) return update_status_message(text.uploading) requests.post( "https://api.telegram.org/bot{}/sendVideo".format( config["telegram_token"]), data=data, files=[ ("video", (utils.random_string() + ".mp4", open(output_filename, "rb"), "video/mp4")), ("thumb", (utils.random_string() + ".jpg", open(thumbnail, "rb"), "image/jpeg")), ], ) utils.rm(input_filename) utils.rm(output_filename) utils.rm(thumbnail) elif target_format == "png": # Upload to Telegram update_status_message(text.uploading) requests.post( "https://api.telegram.org/bot{}/sendPhoto".format( config["telegram_token"]), data=data, files=[("photo", (utils.random_string() + ".png", open(output_filename, "rb"), "image/png"))], ) requests.post( "https://api.telegram.org/bot{}/sendDocument".format( config["telegram_token"]), data=data, files=[("document", (utils.random_string() + ".png", open(output_filename, "rb"), "image/png"))], ) utils.rm(input_filename) utils.rm(output_filename) bot.delete_message(message.chat.id, status_message.message_id)
def test_non_existing(self): with self.assertRaises(AssertionError): rm(Path('asdf_path'))
def main(argv): # Input parameters (from Arguments.py) imaster = IMASTER length = LENGTH_GLOBAL syndir = 'SYNTH_traces' o_wpinversion = O_WPINVERSION nc = NC nl = NL solfile = None flagreg = False # Parse options try: opts, args = go.gnu_getopt(argv[1:], 'i:d:rh', ["icmtf=", "osydir=", "regional", "help"]) except go.GetoptError as err: sys.stderr.write('usage: %s [option] (for help see %s -h)\n' % (sys.argv[0], sys.argv[0])) raise for o, a in opts: if o == '-h' or o == '--help': disphelp(sys.argv[0], solfile, syndir) sys.exit(0) if o == '-r' or o == '--regional': length = LENGTH_REGIONAL flagreg = True if o == '-i' or o == '--icmtf': solfile = a if not os.path.exists(solfile): raise IOError('No wcmtfile named %s' % (solfile)) if o == '-d' or o == '--osyndir': syndir = a if not solfile: for f in ['xy_WCMTSOLUTION', 'ts_WCMTSOLUTION', 'WCMTSOLUTION']: if os.path.exists(f): solfile = f break if not solfile: raise IOError( 'No wcmtfile available, can be specified with --icmtf') eq = EarthQuake() eq.rcmtfile(solfile) cmtla, cmtlo = eq.lat, eq.lon # Title conf = utils.parseConfig(imaster) title = '_'.join(conf['EVNAME'].split()) title += ', filter = (%s, %s, %s, %s)' % ( conf['filt_cf1'], conf['filt_cf2'], conf['filt_order'], conf['filt_pass']) # Cleanup run dir if os.path.exists(syndir) and syndir != '.' and syndir != './': utils.rm(syndir) if syndir != '.' and syndir != './': os.mkdir(syndir) if not os.path.exists(LOGDIR): os.mkdir(LOGDIR) for l in os.listdir('.'): if l[:4] == 'page' and l[-4:] == '.pdf': utils.rm(l) # Compute synthetics cmd = SYNTHS + ' ' + imaster + ' ' + solfile + ' ' + o_wpinversion + ' ' + syndir print(cmd) #status = call(cmd, shell=True, stdin=sys.stdin, stdout=sys.stdout); status = os.system(SYNTHS + ' ' + imaster + ' ' + solfile + ' ' + o_wpinversion + ' ' + syndir + ' > ' + os.path.join(LOGDIR, '_tmp_synths')) if status: print('Error while running ' + SYNTHS) sys.exit(1) # Create Sac Objects sacdata = sacpy.sac() sacsynt = sacpy.sac() coords = [] L = open(o_wpinversion).readlines() for l in L: sacf = l.strip().split()[0] sacdata.read(sacf, datflag=0) coords.append([sacdata.stla, sacdata.stlo, sacdata.az, sacdata.dist]) coords = np.array(coords) # Main loop print('Input (W)CMTSOLUTION file is: %s' % (solfile)) print('Output synthetic directory is: %s' % (syndir)) perpage = nl * nc ntot = len(L) npages = np.ceil(float(ntot) / float(perpage)) nchan = 1 count = 1 pages = 1 fig = plt.figure() fig.subplots_adjust(bottom=0.06, top=0.87, left=0.06, right=0.95, wspace=0.25, hspace=0.4) print('All pages will be saved in %s' % (OPDFFILE)) pp = mpl.backends.backend_pdf.PdfPages(OPDFFILE) basem = None for l in L: # Parse line items = l.strip().split() fic1 = items[0] sacdata.read(fic1) chan = sacdata.kcmpnm[0:3] loc = sacdata.khole fic2 = syndir+'/%s.%s.%s.%s.complete_synth.bp.sac'\ %(sacdata.kstnm,sacdata.knetwk,chan,loc) sacsynt.read(fic2) # pages if count > perpage: plt.suptitle(title + ', p %d/%d' % (pages, npages), fontsize=16, y=0.95) print('page %d/%d' % (pages, npages)) #fig.set_rasterized(True) pp.savefig(orientation='landscape') plt.close() pages += 1 count = 1 fig = plt.figure() fig.subplots_adjust(bottom=0.06, top=0.87, left=0.06, right=0.95, wspace=0.25, hspace=0.4) # Time - W phase window t1 = np.arange(sacdata.npts, dtype='double') * sacdata.delta + sacdata.b - sacdata.o t2 = np.arange(sacsynt.npts, dtype='double') * sacsynt.delta + sacsynt.b - sacsynt.o wnb = float(items[5]) wne = float(items[6]) wtb = sacdata.b - sacdata.o + wnb * sacdata.delta wte = sacdata.b - sacdata.o + wne * sacdata.delta # Plot trace ax = plt.subplot(nl, nc, count) plt.plot(t1, sacdata.depvar * 1000., 'k') plt.plot(t2, sacsynt.depvar * 1000., 'r-') plt.plot([wtb, wte], [0., 0.], 'ro') # Axes limits B = wtb - 150.0 if B < 0: B = 0.0 plt.xlim([B, B + length * sacsynt.delta]) if YLIM_AUTO: a = np.absolute(sacsynt.depvar[:length]).max() * 1000. ymin = -1.1 * a ymax = 1.1 * a ylims = [ymin, ymax] else: ylims = YLIMFIXED plt.ylim(ylims) # Annotations plt.rcParams['font.family'] = 'sans-serif' plt.rcParams['font.sans-serif'] = 'Arial' plt.rcParams['mathtext.fontset'] = 'custom' plt.rcParams['mathtext.rm'] = 'sans' plt.rcParams['mathtext.it'] = 'sans:italic' plt.rcParams['mathtext.default'] = 'it' if sacdata.kcmpnm[2] == 'Z': if sys.version_info >= (2, 7): label = r'%s %s %s %s $(\phi,\Delta) = %6.1f\degree, %6.1f\degree$' % ( sacdata.knetwk, sacdata.kstnm, sacdata.kcmpnm, sacdata.khole, sacdata.az, sacdata.gcarc) else: label = r'%s %s %s %s Az=%3.0f, delta=%3.0f' % ( sacdata.knetwk, sacdata.kstnm, sacdata.kcmpnm, sacdata.khole, sacdata.az, sacdata.gcarc) else: if sys.version_info >= (2, 7): label = r'%s %s %s %s $(\phi,\Delta,\alpha) = %6.1f\degree,' label += '%6.1f\degree, %6.1f\degree$' label = label % (sacdata.knetwk, sacdata.kstnm, sacdata.kcmpnm, sacdata.khole, sacdata.az, sacdata.gcarc, sacdata.cmpaz) else: label = r'%s %s %s %s Az=%3.0f, delta=%3.0f' label = label % (sacdata.knetwk, sacdata.kstnm, sacdata.kcmpnm, sacdata.khole, sacdata.az, sacdata.gcarc) plt.title(label, fontsize=10.0, va='center', ha='center') if not (count - 1) % nc: plt.ylabel('mm', fontsize=10) if (count - 1) / nc == nl - 1 or nchan + nc > ntot: plt.xlabel('time, sec', fontsize=10) plt.grid() try: basem = showBasemap(ax, cmtla, cmtlo, sacdata.stla, sacdata.stlo, coords, flagreg, basem) except: showPolarmap(ax, sacdata.az, sacdata.dist, coords) print('Cannot use basemap') count += 1 nchan += 1 print('page %d/%d' % (pages, npages)) #fig.set_rasterized(True) plt.suptitle(title + ', p %d/%d' % (pages, npages), fontsize=16, y=0.95) pp.savefig(orientation='landscape') plt.close() pp.close()
def main(argv): # Extract command line options try: opts, args = getopt.gnu_getopt(argv[1:],'stpSdi:nhz',["hdsafe","onlyts","onlyxy","npar", "imas=","strike=","dc","nont","dip=", "rake=","mom=","noref","xyz","old", "help"]) except getopt.GetoptError as err: usage() raise # Parse command line options i_master = IMASTER fastflag = True flagts = True flagxy = True flagxyz = False flagref = True sdrM0 = {} for o, a in opts: if o == '-h' or o == '--help': disphelp() sys.exit(0) if o == '-s' or o == '--hdsafe': fastflag = False if o == '-t' or o == '--onlyts': if not flagts: usage() raise getopt.GetoptError('options -t and -p cannot be used simultaneously') flagxy = False flagts = True if o == '-p' or o == '--onlyxy': if not flagxy: usage() raise getopt.GetoptError('options -t and -p cannot be used simultaneously') flagts = False fastflag = False flagxy = True if o == '--dc': sdrM0['-dc']='' if o == '--nont': sdrM0['-nont']='' if o == '--strike': sdrM0['-strike']=a if o == '--dip': sdrM0['-dip']=a if o == '--rake': sdrM0['-rake']=a if o == '--mom': sdrM0['-mom']=a if o == '-i' or o == '--imas': i_master = a if o == '-n' or o == '--noref': flagref = False if o == '-z' or o == '--xyz': flagxyz = True if o == '--old': WPINV_XY += ' -old' # Read i_master iconfig = utils.parseConfig(i_master) cmtref = iconfig['CMTFILE'] evname = iconfig['EVNAME'].replace(' ','_').replace(',','') # Set comments in output ps file Median = '-med ' if 'P2P_SCREENING' in iconfig: if iconfig['P2P_SCREENING'] != 'YES': Median = ' ' ths = '5.0 3.0 0.9' if 'RMS_SCREENING' in iconfig: ths = iconfig['RMS_SCREENING'] comments = [VERSION,'GF_PATH: '+GF_PATH,'Screening: '+Median+ths] # Read reference CMTFILE eq = EarthQuake() eq.rcmtfile(cmtref) eq.title = evname.strip().replace(' ','_').replace(',','') cmtf = open(cmtref,'r') L=cmtf.readlines() cmtf.close() if len(L) < 13: print('*** WARNING : no reference solution in %s'%(cmtref)) flagref = False # TS and/or LAT/LON Grid-search if (flagts or flagxy) and not flagxyz: grid_search(eq,cmtref,TS_NIT,TS_DT,TSBOUNDS,XY_NIT,XY_DX,XY_NX,XY_NOPT,fastflag, flagts,flagxy,sdrM0,ts_ofile=TS_OFILE,xy_ofile=XY_OFILE,comments=comments) # TS and LAT/LON/DEP Grid-search if flagxyz: grid_search(eq,cmtref,TS_NIT,TS_DT,TSBOUNDS,XYZ_NIT,XYZ_DX,XYZ_NX,XYZ_NOPT,fastflag, flagts,flagxyz,sdrM0,dz=DDEP,minz=MINDEP,ts_ofile=TS_OFILE,xy_ofile=XYZ_OFILE, comments=comments) if flagxy: eq.wcmtfile('_tmp_CMTSOLUTION.xyz') if flagref: addrefsol(cmtref,'_tmp_CMTSOLUTION.xyz') grid_search(eq,'_tmp_CMTSOLUTION.xyz',TS_NIT,TS_DT,TSBOUNDS,XY_NIT,XY_DX,XY_NX,XY_NOPT, 0,0,1,sdrM0,ts_ofile=TS_OFILE,xy_ofile=XY_OFILE,comments=comments) utils.rm('_tmp_CMTSOLUTION.xyz') # Cleaning up if os.path.exists('_tmp_ts_table'): utils.rm('_tmp_ts_table') if os.path.exists('_tmp_xy_table'): utils.rm('_tmp_xy_table')
def protocol(self, req): self.buf += req while True: before, sep, after = self.buf.partition('\n') if not sep: break try: data = json.loads(before) except Exception as e: print('Unable to parse json:', e) print('Data:', before) raise e name = data.get('name') if name == 'patch': # TODO: we should do this in a separate thread Listener.apply_patch(data) elif name == 'get_buf': buf_id = data['id'] listener.BUFS[buf_id] = data view = listener.get_view(buf_id) if view: Listener.update_view(data, view) else: listener.save_buf(data) elif name == 'create_buf': listener.BUFS[data['id']] = data listener.save_buf(data) elif name == 'rename_buf': new = utils.get_full_path(data['path']) old = utils.get_full_path(data['old_path']) new_dir = os.path.split(new)[0] if new_dir: utils.mkdir(new_dir) os.rename(old, new) view = listener.get_view(data['id']) if view: view.retarget(new) elif name == 'delete_buf': path = utils.get_full_path(data['path']) utils.rm(path) listener.delete_buf(data['id']) elif name == 'room_info': # Success! Reset counter self.retries = G.MAX_RETRIES self.room_info = data G.PERMS = data['perms'] if 'patch' not in data['perms']: msg.log('We don\'t have patch permission. Setting buffers to read-only') project_json = { 'folders': [ {'path': G.PROJECT_PATH} ] } utils.mkdir(G.PROJECT_PATH) with open(os.path.join(G.PROJECT_PATH, '.sublime-project'), 'w') as project_fd: project_fd.write(json.dumps(project_json, indent=4, sort_keys=True)) floo_json = { 'url': utils.to_room_url({ 'host': self.host, 'owner': self.owner, 'port': self.port, 'room': self.room, 'secure': self.secure, }) } with open(os.path.join(G.PROJECT_PATH, '.floo'), 'w') as floo_fd: floo_fd.write(json.dumps(floo_json, indent=4, sort_keys=True)) for buf_id, buf in data['bufs'].iteritems(): buf_id = int(buf_id) # json keys must be strings buf_path = utils.get_full_path(buf['path']) new_dir = os.path.dirname(buf_path) utils.mkdir(new_dir) listener.BUFS[buf_id] = buf try: buf_fd = open(buf_path, 'r') buf_buf = buf_fd.read().decode('utf-8') md5 = hashlib.md5(buf_buf.encode('utf-8')).hexdigest() if md5 == buf['md5']: msg.debug('md5 sums match. not getting buffer') buf['buf'] = buf_buf else: msg.debug('md5 for %s should be %s but is %s. getting buffer' % (buf['path'], buf['md5'], md5)) raise Exception('different md5') except Exception as e: msg.debug('Error calculating md5:', e) Listener.get_buf(buf_id) self.authed = True G.CONNECTED = True msg.log('Successfully joined room %s/%s' % (self.owner, self.room)) if self.on_connect: self.on_connect(self) self.on_connect = None elif name == 'join': msg.log('%s joined the room' % data['username']) self.room_info['users'][data['user_id']] = data['username'] elif name == 'part': msg.log('%s left the room' % data['username']) try: del self.room_info['users'][data['user_id']] except Exception as e: print('Unable to delete user %s from user list' % (data)) region_key = 'floobits-highlight-%s' % (data['user_id']) for window in sublime.windows(): for view in window.views(): view.erase_regions(region_key) elif name == 'highlight': region_key = 'floobits-highlight-%s' % (data['user_id']) Listener.highlight(data['id'], region_key, data['username'], data['ranges'], data.get('ping', False)) elif name == 'error': message = 'Floobits: Error! Message: %s' % str(data.get('msg')) msg.error(message) elif name == 'disconnect': message = 'Floobits: Disconnected! Reason: %s' % str(data.get('reason')) msg.error(message) sublime.error_message(message) self.stop() elif name == 'msg': self.on_msg(data) else: msg.error('unknown name!', name, 'data:', data) self.buf = after
def main(argv): # Input parameters (from Arguments.py) imaster = IMASTER length = LENGTH_GLOBAL syndir = 'SYNTH_traces' o_wpinversion = O_WPINVERSION nc = NC nl = NL solfile = None flagreg = False # Parse options try: opts, args = go.gnu_getopt(argv[1:],'i:d:rh',["icmtf=","osydir=","regional","help"]) except go.GetoptError as err: sys.stderr.write('usage: %s [option] (for help see %s -h)\n'%(sys.argv[0],sys.argv[0])) raise for o, a in opts: if o == '-h' or o == '--help': disphelp(sys.argv[0],solfile,syndir) sys.exit(0) if o == '-r' or o == '--regional': length = LENGTH_REGIONAL flagreg = True if o == '-i' or o=='--icmtf': solfile = a if not os.path.exists(solfile): raise IOError('No wcmtfile named %s'%(solfile)) if o == '-d' or o == '--osyndir': syndir = a if not solfile: for f in ['xy_WCMTSOLUTION','ts_WCMTSOLUTION','WCMTSOLUTION']: if os.path.exists(f): solfile = f break if not solfile: raise IOError('No wcmtfile available, can be specified with --icmtf') eq = EarthQuake() eq.rcmtfile(solfile) cmtla,cmtlo = eq.lat, eq.lon # Title conf = utils.parseConfig(imaster) title = '_'.join(conf['EVNAME'].split()) title += ', filter = (%s, %s, %s, %s)'%(conf['filt_cf1'],conf['filt_cf2'],conf['filt_order'],conf['filt_pass']) # Cleanup run dir if os.path.exists(syndir) and syndir != '.' and syndir != './': utils.rm(syndir) if syndir != '.' and syndir != './': os.mkdir(syndir) if not os.path.exists(LOGDIR): os.mkdir(LOGDIR) for l in os.listdir('.'): if l[:4]=='page' and l[-4:]=='.pdf': utils.rm(l) # Compute synthetics cmd = SYNTHS+' '+imaster+' '+solfile+' '+o_wpinversion+' '+syndir print(cmd) #status = call(cmd, shell=True, stdin=sys.stdin, stdout=sys.stdout); status = os.system(SYNTHS+' '+imaster+' '+solfile+' '+o_wpinversion+' '+syndir+' > '+os.path.join(LOGDIR,'_tmp_synths')) if status: print('Error while running '+SYNTHS) sys.exit(1) # Create Sac Objects sacdata = sacpy.sac() sacsynt = sacpy.sac() coords = [] L = open(o_wpinversion).readlines() for l in L: sacf = l.strip().split()[0] sacdata.read(sacf,datflag=0) coords.append([sacdata.stla,sacdata.stlo,sacdata.az,sacdata.dist]) coords = np.array(coords) # Main loop print('Input (W)CMTSOLUTION file is: %s'%(solfile)) print('Output synthetic directory is: %s'%(syndir)) perpage = nl*nc ntot = len(L) npages = np.ceil(float(ntot)/float(perpage)) nchan = 1 count = 1 pages = 1 fig = plt.figure() fig.subplots_adjust(bottom=0.06,top=0.87,left=0.06,right=0.95,wspace=0.25,hspace=0.35) print('All pages will be saved in %s'%(OPDFFILE)) pp = mpl.backends.backend_pdf.PdfPages(OPDFFILE) basem = None for l in L: # Parse line items = l.strip().split() fic1 = items[0] sacdata.read(fic1) chan = sacdata.kcmpnm[0:3] loc = sacdata.khole fic2 = syndir+'/%s.%s.%s.%s.complete_synth.bp.sac'\ %(sacdata.kstnm,sacdata.knetwk,chan,loc) sacsynt.read(fic2) # pages if count > perpage: plt.suptitle(title+ ', p %d/%d'%(pages,npages), fontsize=16, y=0.95) print('page %d/%d'%(pages,npages)) #fig.set_rasterized(True) pp.savefig(orientation='landscape') plt.close() pages += 1 count = 1 fig = plt.figure() fig.subplots_adjust(bottom=0.06,top=0.87,left=0.06,right=0.95,wspace=0.25,hspace=0.35) # Time - W phase window t1 = np.arange(sacdata.npts,dtype='double')*sacdata.delta + sacdata.b - sacdata.o t2 = np.arange(sacsynt.npts,dtype='double')*sacsynt.delta + sacsynt.b - sacsynt.o wnb = float(items[5]) wne = float(items[6]) wtb = sacdata.b - sacdata.o + wnb * sacdata.delta wte = sacdata.b - sacdata.o + wne * sacdata.delta # Plot trace ax = plt.subplot(nl,nc,count) plt.plot(t1,sacdata.depvar*1000.,'k') plt.plot(t2,sacsynt.depvar*1000.,'r-') plt.plot([wtb,wte],[0.,0.],'ro') # Axes limits B=wtb-150.0 if B<0: B = 0.0 plt.xlim([B,B+length*sacsynt.delta]) if YLIM_AUTO: a = np.absolute(sacsynt.depvar[:length]).max()*1000. ymin = -1.1*a ymax = 1.1*a ylims = [ymin,ymax] else: ylims = YLIMFIXED plt.ylim(ylims) # Annotations if sacdata.kcmpnm[2] == 'Z': label = u'%s %s %s %s (\u03C6,\u0394) = %6.1f\u00B0, %5.1f\u00B0' label = label%(sacdata.knetwk,sacdata.kstnm, sacdata.kcmpnm, sacdata.khole, sacdata.az, sacdata.gcarc) else: label = u'%s %s %s %s (\u03C6,\u0394,\u03B1) = %6.1f\u00B0, %5.1f\u00B0, %6.1f\u00B0' label = label%(sacdata.knetwk,sacdata.kstnm, sacdata.kcmpnm, sacdata.khole, sacdata.az, sacdata.gcarc, sacdata.cmpaz) plt.title(label,fontsize=10.0,va='center',ha='center') if not (count-1)%nc: plt.ylabel('mm',fontsize=10) if (count-1)/nc == nl-1 or nchan+nc > ntot: plt.xlabel('time, sec',fontsize=10) plt.grid() try: basem = showBasemap(ax,cmtla,cmtlo,sacdata.stla,sacdata.stlo,coords,flagreg,basem) except: showPolarmap(ax,sacdata.az,sacdata.dist,coords) print('No basemap module') count += 1 nchan += 1 print('page %d/%d'%(pages,npages)) #fig.set_rasterized(True) plt.suptitle(title + ', p %d/%d'%(pages,npages), fontsize=16, y=0.95) pp.savefig(orientation='landscape') plt.close() pp.close()