def delete_simulation_files(sim_id,conf):
"""
Remove all files for given simulation.
:param sim_id: the simulation id
:param conf: configuration
"""
rm(simulation_paths(sim_id,conf).values())
def gen_imglst(names, prefix, train_file, test_file):
os.chdir(prefix)
imgs_train_l, imgs_test_l = [], []
for ind, cls in enumerate(names):
if not osp.exists(cls): continue
imgs = tf.gfile.Glob(cls + '/*.JPEG')
if len(imgs) == 0:
utils.rm(cls, True)
continue
assert len(imgs) >= 10
imgs = np.array(imgs)
imgs_test = np.random.choice(imgs, max(3, imgs.shape[0] * 1 // 10), replace=False)
imgs_train = np.setdiff1d(imgs, imgs_test)
# imgs_train = imgs[:imgs.shape[0] * 9 // 10]
# imgs_test = imgs[imgs.shape[0] * 9 // 10:]
# imgs_test.shape, imgs_train.shape, imgs.shape
imgs_train_l.append(
np.stack((imgs_train, np.ones_like(imgs_train, dtype=int) * ind), axis=-1)
)
imgs_test_l.append(
np.stack((imgs_test, np.ones_like(imgs_test, dtype=int) * ind), axis=-1)
)
imgs_train = np.concatenate(imgs_train_l, axis=0)
np.random.shuffle(imgs_train)
np.savetxt(train_file, imgs_train, delimiter=' ', fmt='%s')
np.savetxt(test_file, np.concatenate(imgs_test_l, axis=0), delimiter=' ', fmt='%s')
def delete_simulation_files(sim_id, conf):
"""
Remove all files for given simulation.
:param sim_id: the simulation id
:param conf: configuration
"""
rm(simulation_paths(sim_id, conf).values())
def save(self, cur_loss, method, save_freq=None):
"""
Args:
method: all / last
all: save checkpoint by step
last: save checkpoint to 'last.pth'
all-last: save checkpoint by step per save_freq and
save checkpoint to 'last.pth' always
"""
if method not in ['all', 'last', 'all-last']:
return
step_save = False
last_save = False
if method == 'all' or (method == 'all-last'
and self.step % save_freq == 0):
step_save = True
if method == 'last' or method == 'all-last':
last_save = True
assert step_save or last_save
save_dic = {
'generator': self.gen.state_dict(),
'generator_ema': self.gen_ema.state_dict(),
'optimizer': self.g_optim.state_dict(),
'epoch': self.step,
'loss': cur_loss
}
if self.disc is not None:
save_dic['discriminator'] = self.disc.state_dict()
save_dic['d_optimizer'] = self.d_optim.state_dict()
if self.aux_clf is not None:
save_dic['aux_clf'] = self.aux_clf.state_dict()
save_dic['ac_optimizer'] = self.ac_optim.state_dict()
ckpt_dir = self.cfg['work_dir'] / "checkpoints" / self.cfg[
'unique_name']
step_ckpt_name = "{:06d}-{}.pth".format(self.step, self.cfg['name'])
last_ckpt_name = "last.pth"
step_ckpt_path = Path.cwd() / ckpt_dir / step_ckpt_name
last_ckpt_path = ckpt_dir / last_ckpt_name
log = ""
if step_save:
torch.save(save_dic, str(step_ckpt_path))
log = "Checkpoint is saved to {}".format(step_ckpt_path)
if last_save:
utils.rm(last_ckpt_path)
last_ckpt_path.symlink_to(step_ckpt_path)
log += " and symlink to {}".format(last_ckpt_path)
if not step_save and last_save:
utils.rm(last_ckpt_path) # last 가 symlink 일 경우 지우고 써줘야 함.
torch.save(save_dic, str(last_ckpt_path))
log = "Checkpoint is saved to {}".format(last_ckpt_path)
self.logger.info("{}\n".format(log))
def test_dir(self):
with tempfile.TemporaryDirectory() as tmpdir:
tmpdir = Path(tmpdir)
(tmpdir / 'c').mkdir()
(tmpdir / 'c' / 'a.txt').write_text("asdf_content")
(tmpdir / 'c' / 'b.txt').write_text("bsdf_content")
rm(tmpdir / 'c')
self.assertFalse((tmpdir / 'c').exists())
def on_delete_buf(self, data):
# TODO: somehow tell the user about this. maybe delete on disk too?
del self.FLOO_BUFS[data["id"]]
path = utils.get_full_path(data["path"])
if not G.DELETE_LOCAL_FILES:
msg.log("Not deleting %s because delete_local_files is disabled" % path)
return
utils.rm(path)
msg.warn("deleted %s because %s told me to." % (path, data.get("username", "the internet")))
def on_delete_buf(self, data):
# TODO: somehow tell the user about this. maybe delete on disk too?
del self.FLOO_BUFS[data['id']]
path = utils.get_full_path(data['path'])
if not G.DELETE_LOCAL_FILES:
msg.log('Not deleting %s because delete_local_files is disabled' % path)
return
utils.rm(path)
msg.warn('deleted %s because %s told me to.' % (path, data.get('username', 'the internet')))
def save(self, i):
# 우선 저장
fpath = join(self.fdir, "{}-{}.{}".format(self.fname, i, self.fext)) # full path
mkdir(self.fdir)
torch.save(self.model.state_dict(), fpath)
# 현재 폴더에 저장된 같은 종류의 모델 확인
files, idxs = ls_model(self.fdir, self.fname, self.fext)
# 저장이 max_to_keep 보다 많으면 가장 오래된것 삭제
if self.max_to_keep < len(files):
rm(files[0])
def update_backup(backup: Optional[str], i: int, time_stamp: str) -> str:
tmp = save_dir + '{0}_{1}_{2}.pickle'.format(log_title, i, time_stamp)
training_backup(net, optimizer, tmp, optim_kwargs)
if backup is not None:
if not rm(backup):
print('Failed to delete {0}'.format(backup))
return tmp
def rename(self, name):
msg.debug("renaming %s to %s" % (self.vim_buf.name, name))
current = vim.current.buffer
text = self.get_text()
old_name = self.vim_buf.name
old_number = self.native_id
with open(name, "wb") as fd:
fd.write(text.encode("utf-8"))
vim.command("edit! %s" % name)
self.vim_buf = vim.current.buffer
vim.command("edit! %s" % current.name)
vim.command("bdelete! %s" % old_number)
try:
utils.rm(old_name)
except Exception as e:
msg.debug("couldn't delete %s... maybe thats OK?" % str(e))
def rename(self, name):
msg.debug('renaming %s to %s' % (self.vim_buf.name, name))
current = vim.current.buffer
text = self.get_text()
old_name = self.vim_buf.name
old_number = self.native_id
with open(name, 'wb') as fd:
fd.write(text.encode('utf-8'))
vim.command('edit! %s' % name)
self.vim_buf = vim.current.buffer
vim.command('edit! %s' % current.name)
vim.command('bdelete! %s' % old_number)
try:
utils.rm(old_name)
except Exception as e:
msg.debug("couldn't delete %s... maybe thats OK?" % str(e))
def check_individual(filename, filepath):
if filename.endswith('JPEG') and osp.getsize(filepath) == 0:
utils.rm(filepath)
append_file(filepath,
file='/home/wangxinglu/prj/few-shot/src/append.txt')
return
im = cv2.imread(filepath, cv2.CV_LOAD_IMAGE_COLOR)
cv2.imwrite(filepath, im, [int(cv2.IMWRITE_JPEG_QUALITY), 100])
if im.shape[0] <= 10 or im.shape[1] <= 10:
# utils.rm(filepath)
append_file(filepath, './wrong.img.txt')
print filepath
assert im.shape[-1] == 3
assert im.shape[0] > 13 and im.shape[1] > 13
assert (len(im.shape) == 3
and im.shape[-1] == 3), 'img ' + filepath + str(im.shape)
assert im.shape[1] != 0 and im.shape[0] != 0, 'width ' + filepath + str(
im.shape)
def main(unused_argv):
assert not FLAGS.train_shards % FLAGS.num_threads, (
'Please make the FLAGS.num_threads commensurate with FLAGS.train_shards')
assert not FLAGS.validation_shards % FLAGS.num_threads, (
'Please make the FLAGS.num_threads commensurate with '
'FLAGS.validation_shards')
print('Saving results to %s' % FLAGS.output_directory)
with tf.device('/cpu:0'):
# Build a map from synset to human-readable label.
synset_to_human = _build_synset_lookup(FLAGS.imagenet_metadata_file)
# Run it!
utils.rm(FLAGS.output_directory + '/train*', block=True)
_process_dataset('train', FLAGS.validation_directory,
FLAGS.validation_shards, synset_to_human)
utils.rm(FLAGS.output_directory + '/validation*', block=True)
_process_dataset('validation', FLAGS.train_directory, FLAGS.train_shards,
synset_to_human)
def get_diffs(base1, base2, code, zoom):
old = get_geojson(join(base1, code), zoom)
tmp = get_geojson(join(base2, code), zoom)
changed = []
for x in tmp:
xtmp = join(base2, code, x)
xold = join(base1, code, x)
if x in old:
if open(xtmp).read() != open(xold).read():
changed.append(x)
cp(xtmp, xold)
else:
changed.append(x)
cp(xtmp, xold)
for x in old:
if x not in tmp:
xold = join(base1, code, x)
changed.append(x)
rm(xold)
return changed
def get_diffs(base1,base2,code,zoom):
old = get_geojson(join(base1,code),zoom)
tmp = get_geojson(join(base2,code),zoom)
changed = []
for x in tmp:
xtmp = join(base2,code,x)
xold = join(base1,code,x)
if x in old:
if open(xtmp).read() != open(xold).read():
changed.append(x)
cp(xtmp,xold)
else:
changed.append(x)
cp(xtmp,xold)
for x in old:
if x not in tmp:
xold = join(base1,code,x)
changed.append(x)
rm(xold)
return changed
def merge_json(M,base,out):
print "BO",base,out
if True:
NEWM = {}
check = set()
for path in M:
if not M[path]:
ps = path.split("/")
newpath = "/".join(ps[1:])
check.add(newpath)
for code in M[path]:
ps = path.split("/")
newpath = "/".join(ps[1:])
newcode = code+"/"+ps[0]
NEWM.setdefault(newpath,[]).append(newcode)
for x in check:
if x not in NEWM:
NEWM[x] = []
M = NEWM
for k in M:
new = join(base,out,k)
makedirs(dirname(new))
if len(M[k]) == 0:
rm(new)
elif len(M[k]) == 1:
old = join(base,M[k][0],k)
cp(old,new)
else:
print "K",k,"MK",M[k][:10]
gs = [(x,geojson.load(open(join(base,x,k)))) for x in M[k]]
x0,g = gs[0]
for x,h in gs[1:]:
print "merging",x0,x
g["features"] += h["features"]
f = open(new,"w")
geojson.dump(g,f)
f.close()
def merge_json(M, base, out):
print "BO", base, out
if True:
NEWM = {}
check = set()
for path in M:
if not M[path]:
ps = path.split("/")
newpath = "/".join(ps[1:])
check.add(newpath)
for code in M[path]:
ps = path.split("/")
newpath = "/".join(ps[1:])
newcode = code + "/" + ps[0]
NEWM.setdefault(newpath, []).append(newcode)
for x in check:
if x not in NEWM:
NEWM[x] = []
M = NEWM
for k in M:
new = join(base, out, k)
makedirs(dirname(new))
if len(M[k]) == 0:
rm(new)
elif len(M[k]) == 1:
old = join(base, M[k][0], k)
cp(old, new)
else:
print "K", k, "MK", M[k][:10]
gs = [(x, geojson.load(open(join(base, x, k)))) for x in M[k]]
x0, g = gs[0]
for x, h in gs[1:]:
print "merging", x0, x
g["features"] += h["features"]
f = open(new, "w")
geojson.dump(g, f)
f.close()
def write(self , tfile , package=None):
"""
Arguments:
- `self`:
"""
if package == None:
package = self._package
if package == '*':
utils.rm(tfile)
os.mkdir(tfile)
for i in self._js:
self.write(os.path.basename(tfile) + os.sep + i + '.js' , i)
#self.jsdoc( tfile )
self.addboot( tfile )
else:
f = codecs.open(tfile , 'w' , self._encoding)
f.write(self.getFile(package))
f.close()
def clean(self):
"""
清空构建目录
"""
print("start clean output dir")
rm(os.path.join(self.local_dist_dir, "*"))
rm(os.path.join(CUR_DIR, "__pycache__"))
rm(os.path.join(CUR_DIR, "*.pyc"))
print("end")
return 0
def _process_image(filename, coder):
"""Process a single image file.
Args:
filename: string, path to an image file e.g., '/path/to/example.JPG'.
coder: instance of ImageCoder to provide TensorFlow image coding utils.
Returns:
image_buffer: string, JPEG encoding of RGB image.
height: integer, image height in pixels.
width: integer, image width in pixels.
"""
# Read the image file.
try:
image_data = tf.gfile.FastGFile(filename, 'r').read()
# Clean the dirty data.
if _is_png(filename):
utils.rm(filename)
return
# 1 image is a PNG.
print('Converting PNG to JPEG for %s' % filename)
image_data = coder.png_to_jpeg(image_data)
elif _is_cmyk(filename):
utils.rm(filename)
return
# 22 JPEG images are in CMYK colorspace.
print('Converting CMYK to RGB for %s' % filename)
image_data = coder.cmyk_to_rgb(image_data)
# Decode the RGB JPEG.
image = coder.decode_jpeg(image_data)
except Exception as inst:
utils.rm(filename)
tf.logging.error(inst)
tf.logging.error(filename)
append_file(filename)
# raise ValueError('rm file')
# Check that image converted to RGB
assert len(image.shape) == 3
height = image.shape[0]
width = image.shape[1]
assert image.shape[2] == 3
return image_data, height, width
def main(argv):
# Extract command line options
try:
opts, args = getopt.gnu_getopt(argv[1:], 'stpSdi:nhz', [
"hdsafe", "onlyts", "onlyxy", "npar", "imas=", "strike=", "dc",
"nont", "dip=", "rake=", "mom=", "noref", "xyz", "old", "help"
])
except getopt.GetoptError as err:
usage()
raise
# Parse command line options
i_master = IMASTER
fastflag = True
flagts = True
flagxy = True
flagxyz = False
flagref = True
sdrM0 = {}
for o, a in opts:
if o == '-h' or o == '--help':
disphelp()
sys.exit(0)
if o == '-s' or o == '--hdsafe':
fastflag = False
if o == '-t' or o == '--onlyts':
if not flagts:
usage()
raise getopt.GetoptError(
'options -t and -p cannot be used simultaneously')
flagxy = False
flagts = True
if o == '-p' or o == '--onlyxy':
if not flagxy:
usage()
raise getopt.GetoptError(
'options -t and -p cannot be used simultaneously')
flagts = False
fastflag = False
flagxy = True
if o == '--dc':
sdrM0['-dc'] = ''
if o == '--nont':
sdrM0['-nont'] = ''
if o == '--strike':
sdrM0['-strike'] = a
if o == '--dip':
sdrM0['-dip'] = a
if o == '--rake':
sdrM0['-rake'] = a
if o == '--mom':
sdrM0['-mom'] = a
if o == '-i' or o == '--imas':
i_master = a
if o == '-n' or o == '--noref':
flagref = False
if o == '-z' or o == '--xyz':
flagxyz = True
if o == '--old':
WPINV_XY += ' -old'
# Read i_master
iconfig = utils.parseConfig(i_master)
cmtref = iconfig['CMTFILE']
evname = iconfig['EVNAME'].replace(' ', '_').replace(',', '')
# Set comments in output ps file
Median = '-med '
if 'P2P_SCREENING' in iconfig:
p2p_items = iconfig['P2P_SCREENING'].split()
if p2p_items[0] != 'YES':
Median = ' '
elif len(p2p_items) > 1:
for p in p2p_items[1:]:
Median += p + ' '
ths = '-th 5.0 3.0 0.9'
if 'RMS_SCREENING' in iconfig:
ths = '-th ' + iconfig['RMS_SCREENING']
comments = [VERSION, 'GF_PATH: ' + GF_PATH, 'Screening: ' + Median + ths]
# Read reference CMTFILE
eq = EarthQuake()
eq.rcmtfile(cmtref)
eq.title = evname.strip().replace(' ', '_').replace(',', '')
cmtf = open(cmtref, 'r')
L = cmtf.readlines()
cmtf.close()
if len(L) < 13:
print('*** WARNING : no reference solution in %s' % (cmtref))
flagref = False
# TS and/or LAT/LON Grid-search
if (flagts or flagxy) and not flagxyz:
grid_search(eq,
cmtref,
TS_NIT,
TS_DT,
TSBOUNDS,
XY_NIT,
XY_DX,
XY_NX,
XY_NOPT,
fastflag,
flagts,
flagxy,
sdrM0,
ts_ofile=TS_OFILE,
xy_ofile=XY_OFILE,
comments=comments)
# TS and LAT/LON/DEP Grid-search
if flagxyz:
grid_search(eq,
cmtref,
TS_NIT,
TS_DT,
TSBOUNDS,
XYZ_NIT,
XYZ_DX,
XYZ_NX,
XYZ_NOPT,
fastflag,
flagts,
flagxyz,
sdrM0,
dz=DDEP,
minz=MINDEP,
ts_ofile=TS_OFILE,
xy_ofile=XYZ_OFILE,
comments=comments)
if flagxy:
eq.wcmtfile('_tmp_CMTSOLUTION.xyz')
if flagref:
addrefsol(cmtref, '_tmp_CMTSOLUTION.xyz')
grid_search(eq,
'_tmp_CMTSOLUTION.xyz',
TS_NIT,
TS_DT,
TSBOUNDS,
XY_NIT,
XY_DX,
XY_NX,
XY_NOPT,
0,
0,
1,
sdrM0,
ts_ofile=TS_OFILE,
xy_ofile=XY_OFILE,
comments=comments)
utils.rm('_tmp_CMTSOLUTION.xyz')
# Cleaning up
if os.path.exists('_tmp_ts_table'):
utils.rm('_tmp_ts_table')
if os.path.exists('_tmp_xy_table'):
utils.rm('_tmp_xy_table')
def on_delete_buf(self, data):
# TODO: somehow tell the user about this. maybe delete on disk too?
del self.FLOO_BUFS[data['id']]
path = utils.get_full_path(data['path'])
utils.rm(path)
msg.warn('deleted %s because %s told me to.' % (path, data.get('username', 'the internet')))
def protocol(self, req):
self.buf += req.decode('utf-8')
msg.debug('buf: %s' % self.buf)
while True:
before, sep, after = self.buf.partition('\n')
if not sep:
break
try:
data = json.loads(before)
except Exception as e:
msg.error('Unable to parse json: %s' % str(e))
msg.error('Data: %s' % before)
raise e
name = data.get('name')
if name == 'patch':
# TODO: we should do this in a separate thread
Listener.apply_patch(data)
elif name == 'get_buf':
buf_id = data['id']
listener.BUFS[buf_id] = data
view = listener.get_view(buf_id)
if view:
Listener.update_view(data, view)
else:
listener.save_buf(data)
elif name == 'create_buf':
listener.BUFS[data['id']] = data
listener.save_buf(data)
elif name == 'rename_buf':
new = utils.get_full_path(data['path'])
old = utils.get_full_path(data['old_path'])
new_dir = os.path.split(new)[0]
if new_dir:
utils.mkdir(new_dir)
os.rename(old, new)
view = listener.get_view(data['id'])
if view:
view.retarget(new)
elif name == 'delete_buf':
path = utils.get_full_path(data['path'])
try:
utils.rm(path)
except Exception:
pass
listener.delete_buf(data['id'])
elif name == 'room_info':
# Success! Reset counter
self.retries = G.MAX_RETRIES
self.room_info = data
G.PERMS = data['perms']
if 'patch' not in data['perms']:
msg.log(
'We don\'t have patch permission. Setting buffers to read-only'
)
project_json = {'folders': [{'path': G.PROJECT_PATH}]}
utils.mkdir(G.PROJECT_PATH)
with open(os.path.join(G.PROJECT_PATH, '.sublime-project'),
'wb') as project_fd:
project_fd.write(
json.dumps(project_json, indent=4,
sort_keys=True).encode('utf-8'))
floo_json = {
'url':
utils.to_room_url({
'host': self.host,
'owner': self.owner,
'port': self.port,
'room': self.room,
'secure': self.secure,
})
}
with open(os.path.join(G.PROJECT_PATH, '.floo'),
'w') as floo_fd:
floo_fd.write(
json.dumps(floo_json, indent=4, sort_keys=True))
for buf_id, buf in data['bufs'].items():
buf_id = int(buf_id) # json keys must be strings
buf_path = utils.get_full_path(buf['path'])
new_dir = os.path.dirname(buf_path)
utils.mkdir(new_dir)
listener.BUFS[buf_id] = buf
try:
buf_fd = open(buf_path, 'rb')
buf_buf = buf_fd.read().decode('utf-8')
md5 = hashlib.md5(buf_buf.encode('utf-8')).hexdigest()
if md5 == buf['md5']:
msg.debug('md5 sums match. not getting buffer')
buf['buf'] = buf_buf
else:
msg.debug(
'md5 for %s should be %s but is %s. getting buffer'
% (buf['path'], buf['md5'], md5))
raise Exception('different md5')
except Exception as e:
msg.debug('Error calculating md5:', e)
Listener.get_buf(buf_id)
self.authed = True
G.CONNECTED = True
msg.log('Successfully joined room %s/%s' %
(self.owner, self.room))
if self.on_connect:
self.on_connect(self)
self.on_connect = None
elif name == 'join':
msg.log('%s joined the room' % data['username'])
self.room_info['users'][data['user_id']] = data['username']
elif name == 'part':
msg.log('%s left the room' % data['username'])
try:
del self.room_info['users'][data['user_id']]
except Exception as e:
print('Unable to delete user %s from user list' % (data))
region_key = 'floobits-highlight-%s' % (data['user_id'])
for window in sublime.windows():
for view in window.views():
view.erase_regions(region_key)
elif name == 'highlight':
region_key = 'floobits-highlight-%s' % (data['user_id'])
Listener.highlight(data['id'], region_key, data['username'],
data['ranges'], data.get('ping', False))
elif name == 'error':
message = 'Floobits: Error! Message: %s' % str(data.get('msg'))
msg.error(message)
elif name == 'disconnect':
message = 'Floobits: Disconnected! Reason: %s' % str(
data.get('reason'))
msg.error(message)
sublime.error_message(message)
self.stop()
elif name == 'msg':
self.on_msg(data)
else:
msg.debug('unknown name!', name, 'data:', data)
self.buf = after
# parser for getpath tool
parser_getpath = subparsers.add_parser('where',
help='Display the path in which datasets are installed.')
parser_getpath.add_argument('--config',
dest='config',
metavar='STRING',
required=False,
help='Absolute location to a specific config file')
parser_getpath.set_defaults(func=utils.get_install_path)
# parse the args and call the selected function
args = parser.parse_args()
# so we don't have multiple things running and writing data at the same
# time.
utils.check_set_lock(LOCK_PATH)
try:
args.func(args)
except IOError, e:
if e.errno != 32: # ignore SIGPIPE
raise
finally:
utils.rm(LOCK_PATH)
if __name__ == "__main__":
main()
def eval_ckpt():
from train import (setup_language_dependent, setup_data,
setup_cv_dset_loader, get_dset_loader)
logger = Logger.get()
parser = argparse.ArgumentParser('MaHFG-eval')
parser.add_argument(
"name",
help=
"name is used for directory name of the user-study generation results")
parser.add_argument("resume")
parser.add_argument("img_dir")
parser.add_argument("config_paths", nargs="+")
parser.add_argument("--show", action="store_true", default=False)
parser.add_argument(
"--mode",
default="eval",
help="eval (default) / cv-save / user-study / user-study-save. "
"`eval` generates comparable grid and computes pixel-level CV scores. "
"`cv-save` generates and saves all target characters in CV. "
"`user-study` generates comparable grid for the ramdomly sampled target characters. "
"`user-study-save` generates and saves all target characters in user-study."
)
parser.add_argument("--deterministic", default=False, action="store_true")
parser.add_argument("--debug", default=False, action="store_true")
args, left_argv = parser.parse_known_args()
cfg = Config(*args.config_paths)
cfg.argv_update(left_argv)
torch.backends.cudnn.benchmark = True
cfg['data_dir'] = Path(cfg['data_dir'])
if args.show:
exit()
# seed
np.random.seed(cfg['seed'])
torch.manual_seed(cfg['seed'])
random.seed(cfg['seed'])
if args.deterministic:
torch.backends.cudnn.benchmark = False
torch.backends.cudnn.deterministic = True
cfg['n_workers'] = 0
logger.info("#" * 80)
logger.info("# Deterministic option is activated !")
logger.info(
"# Deterministic evaluator only ensure the deterministic cross-validation"
)
logger.info("#" * 80)
else:
torch.backends.cudnn.benchmark = True
if args.mode.startswith('mix'):
assert cfg['g_args']['style_enc']['use'], \
"Style mixing is only available with style encoder model"
#####################################
# Dataset
####################################
# setup language dependent values
content_font, n_comp_types, n_comps = setup_language_dependent(cfg)
# setup transform
transform = transforms.Compose(
[transforms.ToTensor(),
transforms.Normalize([0.5], [0.5])])
# setup data
hdf5_data, meta = setup_data(cfg, transform)
# setup dataset
trn_dset, loader = get_dset_loader(hdf5_data,
meta['train']['fonts'],
meta['train']['chars'],
transform,
True,
cfg,
content_font=content_font)
val_loaders = setup_cv_dset_loader(hdf5_data, meta, transform,
n_comp_types, content_font, cfg)
#####################################
# Model
####################################
# setup generator only
g_kwargs = cfg.get('g_args', {})
gen = MACore(1,
cfg['C'],
1,
**g_kwargs,
n_comps=n_comps,
n_comp_types=n_comp_types,
language=cfg['language'])
gen.cuda()
ckpt = torch.load(args.resume)
logger.info("Use EMA generator as default")
gen.load_state_dict(ckpt['generator_ema'])
step = ckpt['epoch']
loss = ckpt['loss']
logger.info("Resumed checkpoint from {} (Step {}, Loss {:7.3f})".format(
args.resume, step, loss))
writer = utils.DiskWriter(args.img_dir, 0.6)
evaluator = Evaluator(hdf5_data,
trn_dset.avails,
logger,
writer,
cfg['batch_size'],
content_font=content_font,
transform=transform,
language=cfg['language'],
val_loaders=val_loaders,
meta=meta)
evaluator.n_cv_batches = -1
logger.info("Update n_cv_batches = -1 to evaluate about full data")
if args.debug:
evaluator.n_cv_batches = 10
logger.info("!!! DEBUG MODE: n_cv_batches = 10 !!!")
if args.mode == 'eval':
logger.info("Start validation ...")
dic = evaluator.validation(gen, step)
logger.info("Validation is done. Result images are saved to {}".format(
args.img_dir))
elif args.mode.startswith('user-study'):
meta = json.load(open('meta/kor-unrefined.json'))
target_chars = meta['target_chars']
style_chars = meta['style_chars']
fonts = meta['fonts']
if args.mode == 'user-study':
sampled_target_chars = uniform_sample(target_chars, 20)
logger.info("Start generation kor-unrefined ...")
logger.info("Sampled chars = {}".format(sampled_target_chars))
evaluator.handwritten_validation_2stage(gen,
step,
fonts,
style_chars,
sampled_target_chars,
comparable=True,
tag='userstudy-{}'.format(
args.name))
elif args.mode == 'user-study-save':
logger.info("Start generation & saving kor-unrefined ...")
save_dir = Path(args.img_dir) / "{}-{}".format(args.name, step)
evaluator.handwritten_validation_2stage(gen,
step,
fonts,
style_chars,
target_chars,
comparable=True,
save_dir=save_dir)
logger.info("Validation is done. Result images are saved to {}".format(
args.img_dir))
elif args.mode == 'cv-save':
save_dir = Path(args.img_dir) / "cv_images_{}".format(step)
logger.info("Save CV results to {} ...".format(save_dir))
utils.rm(save_dir)
for tag, loader in val_loaders.items():
l1, ssim, msssim = evaluator.cross_validation(
gen,
step,
loader,
tag,
n_batches=evaluator.n_cv_batches,
save_dir=(save_dir / tag))
else:
raise ValueError(args.mode)
def apply(self, hot_dir: os.PathLike, cold_dir: os.PathLike, index: Index):
rm(cold_dir / self.name)
cp(hot_dir / self.name, cold_dir / self.name)
index[self.name] = self.index
def main_simulate():
parser = argparse.ArgumentParser(description='This script use Flex to simulate cloth dynamics.')
# Global
parser.add_argument('--randomSeed', type=int, default=14, help='Fix the random seed.')
parser.add_argument('--flex_verbose', type=int, default=0, help='0 to inhibit printings from FleX')
# Experiments
parser.add_argument('--scenes_str', type=str, default="wind", help='Scene name [If there are multiple scenes, use --scenes_list]')
parser.add_argument('--scenes_list', type=list, default=[], help='a list of scene names [If there is only one scecne, use --scenes_str]')
parser.add_argument('--bstiff_float', type=float, default=1.0, help='bend_stiffness [If there are multiple bs values, use --bstiff_list]')
parser.add_argument('--bstiff_list', type=list, default=[], help='a list of bend_stiffness values [If there is only one bs value, use --bstiff_float]')
# Flex
parser.add_argument('--flexRootPath', type=str, default="FleX/", help='path to FleX root directory')
parser.add_argument('--flexConfig', type=str, default="FleXConfigs/flexConfig.yml", help='FleX config file')
parser.add_argument('--flex_input_root_path', type=str, default="dataset/trialObjs/", help='root path to FleX input .obj file')
parser.add_argument('--obj_input', type=str, default="model1.obj", help='name of the FleX input .obj file')
parser.add_argument('--flex_output_root_path', type=str, default="experiments/simulation/trialObjs/", help='root path to FleX output .obj files')
# Flex simulation paras
parser.add_argument('--windstrength', type=float, default=0.3, help='wind strength')
parser.add_argument('--scale', type=float, default=1.0, help='object scale')
parser.add_argument('--rot', default=(0,0,0), help='Rotation of the object.')
parser.add_argument('--local', type=bool, default=True, help='run Flex locally')
#parser.add_argument('--clothColor', type=str, default="violet", help='')
parser.add_argument('--floor', type=bool, default=False, help='')
parser.add_argument('--offline', type=bool, default=True, help='')
#parser.add_argument('--occluded', type=bool, default=True, help='')
parser.add_argument('--useQuat', type=bool, default=False, help='')
parser.add_argument('--psize', type=float, default=0.01, help='')
parser.add_argument('--clothNumParticles', type=int, default=210, help='')
parser.add_argument('--flexrandomSeed', type=int, default=-1, help='')
parser.add_argument('--visSaveClothPerSimStep', type=bool, default=False, help='')
parser.add_argument('--randomClothMinRes', type=int, default=145, help='')
parser.add_argument('--randomClothMaxRes', type=int, default=215, help='')
parser.add_argument('--mass', type=float, default=-1.0, help='invMass [Set -1 to use the value defined in --flexConfig]')
parser.add_argument('--shstiff', type=float, default=-1.0, help='shear stiffness [Set -1 to use the value defined in --flexConfig]')
parser.add_argument('--ststiff', type=float, default=-1.0, help='stretch stiffnessetch [Set -1 to use the value defined in --flexConfig]')
parser.add_argument('--clothLift', type=float, default=-1.0, help='')
parser.add_argument('--clothDrag', type=float, default=-1.0, help='')
parser.add_argument('--clothFriction', type=float, default=1.1, help='')
#parser.add_argument('--outputPath', type=str, default="", help='')
parser.add_argument('--saveClothPerSimStep', type=int, default=1, help='')
args = parser.parse_args()
#print(args)
# ----------------------------
np.random.seed(args.randomSeed)
random.seed(args.randomSeed)
torch.manual_seed(args.randomSeed)
torch.cuda.manual_seed(args.randomSeed)
# ----------------------------
# If there are multiple scenes
if len(args.scenes_list) != 0:
scenes = args.scenes_list
else:
scenes = [args.scenes_str]
# If there are multiple bstiff values
if len(args.bstiff_list) != 0:
bstiff = [float(i) for i in args.bstiff_list]
else:
bstiff = [float(args.bstiff_float)]
shstiff=float(args.shstiff)
ststiff=float(args.ststiff)
mass=float(args.mass)
# Simulate .obj files for each scene
for tmp_scene in scenes:
objPath = os.path.join(args.flex_input_root_path, tmp_scene, args.obj_input)
# ----- Setup environment ----- #
FLEX_BIN_PATH = os.path.join(args.flexRootPath, "bin", "linux64", "NvFlexDemoReleaseCUDA_x64_" + tmp_scene)
if not os.path.exists(FLEX_BIN_PATH):
errorMessage="==> Error: No FleX binary found. Make sure you have set the right path and compiled FleX.\nTo compile FleX: singularity exec --nv kimImage.simg bash buildFleX.sh --build=true"
sys.exit(errorMessage)
else:
os.environ["FLEX_PATH"] = args.flexRootPath
# For libGLEW.so.1.10
os.environ["LD_LIBRARY_PATH"] += ":{}".format(os.path.join(args.flexRootPath, "external"))
# ----- End Setup environment ----- #
for i in bstiff:
'''
Params:
rot: Euler angles (in degrees) or quaternion (max unit norm)
'''
# e.g., "experiments/renderings/trialObjs/wind/wind" The last wind is the base name for the output .obj files
outputPath = os.path.join(args.flex_output_root_path, tmp_scene+"_mass_"+str(mass)+"_bs_"+str(i)+"_sh_"+str(shstiff)+"_st_"+str(ststiff), tmp_scene)
outputFolder = ('/').join(outputPath.split('/')[0:-1])
rm(outputFolder)
mkdirs(outputFolder)
sim_cmd = [FLEX_BIN_PATH,
"-obj={}".format(os.path.abspath(objPath)),
"-config={}".format(os.path.abspath(args.flexConfig)),
"-export={}".format(os.path.abspath(outputPath)),
#"-iters={}".format(n_frames),
#"-ss={}".format(n_substeps_per_frame),
# rotate
not args.useQuat and "-rx={} -ry={} -rz={}".format(*args.rot) or "-rx={} -ry={} -rz={} -rw={}".format(*args.rot),
# cloth properties
#"-stiff_scale={}".format(stiffness),
# "-ccolor_id={}".format(COLOR_MAP[clothColor]),
]
if not args.floor:
sim_cmd.append("-nofloor")
if args.offline:
sim_cmd.append("-offline")
# if not occluded:
# sim_cmd.append("-clothsize=1")
# sim_cmd.append("-cam_dist={}".format(4.5))
if args.useQuat:
sim_cmd.append("-use_quat")
else:
sim_cmd.append("-use_euler")
if args.visSaveClothPerSimStep:
sim_cmd.append("-saveClothPerSimStep")
if args.flex_verbose:
sim_cmd.append("-v")
sim_cmd.append("-bstiff={0:f}".format(i))
sim_cmd.append("-psize={0:f}".format(args.psize))
sim_cmd.append("-clothsize={0:d}".format(args.clothNumParticles))
sim_cmd.append("-randomSeed={0:d}".format(args.flexrandomSeed))
sim_cmd.append("-randomClothMinRes={0:d}".format(args.randomClothMinRes))
sim_cmd.append("-randomClothMaxRes={0:d}".format(args.randomClothMaxRes))
sim_cmd.append("-clothLift={0:f}".format(args.clothLift))
sim_cmd.append("-scale={0:f}".format(args.scale))
sim_cmd.append("-windstrength={0:f}".format(args.windstrength))
sim_cmd.append("-mass={0:f}".format(float(args.mass)))
sim_cmd.append("-shstiff={0:f}".format(float(args.shstiff)))
sim_cmd.append("-ststiff={0:f}".format(float(args.ststiff)))
sim_cmd.append("-clothDrag={0:f}".format(args.clothDrag))
sim_cmd.append("-clothFriction={0:f}".format(args.clothFriction))
sim_cmd.append("-saveClothPerSimStep={0:d}".format(args.saveClothPerSimStep))
env = {}
if args.local:
env["DISPLAY"] = ":0"
else:
sim_cmd.insert(0, "vglrun")
#print("---- Output Folder: ", outputFolder)
# save params
with open ((outputPath+'_main_simulate_paras.txt'), "w") as txt_file:
for line in sim_cmd:
txt_file.write("".join(line) + "\n")
lines_stdout = runCmd(" ".join(sim_cmd), extra_vars=env, verbose=args.flex_verbose)
if (len(lines_stdout) < 50):
print(lines_stdout, file=sys.stderr)
sys.exit(1)
def apply(self, hot_dir: os.PathLike, cold_dir: os.PathLike, index: Index):
rm(cold_dir / self.name)
def convert_worker(target_format, message, url, config, bot):
"""Generic process spawned every time user sends a link or a file"""
input_filename = "".join([config["temp_path"], utils.random_string()])
output_filename = "".join(
[config["temp_path"],
utils.random_string(), ".", target_format])
# Tell user that we are working
status_message = bot.reply_to(message, text.starting, parse_mode="HTML")
def update_status_message(new_text):
bot.edit_message_text(chat_id=status_message.chat.id,
message_id=status_message.message_id,
text=new_text,
parse_mode="HTML")
# Try to download URL
try:
r = requests.get(url, stream=True)
except:
update_status_message(text.error.downloading)
return
# Check file size
if int(r.headers.get("Content-Length", "0")) >= MAXIMUM_FILESIZE_ALLOWED:
update_status_message(text.error.huge_file)
return
# Download the file
update_status_message(text.downloading)
chunk_size = 4096
raw_input_size = 0
try:
with open(input_filename, "wb") as f:
for chunk in r.iter_content(chunk_size=chunk_size):
f.write(chunk)
raw_input_size += chunk_size
# Download files without Content-Length, but apply standard limit to them
if raw_input_size >= MAXIMUM_FILESIZE_ALLOWED:
update_status_message(text.error.huge_file)
utils.rm(input_filename)
return
except:
update_status_message(text.error.downloading)
bot.reply_to(message, f"HTTP {r.status_code}")
return
# Start ffmpeg
ffmpeg_process = None
if target_format == "mp4":
ffmpeg_process = subprocess.Popen([
"ffmpeg",
"-v",
"error",
"-threads",
str(config["ffmpeg_threads"]),
"-i",
input_filename,
"-map",
"V:0?", # select video stream
"-map",
"0:a?", # ignore audio if doesn't exist
"-c:v",
"libx264", # specify video encoder
"-max_muxing_queue_size",
"9999", # https://trac.ffmpeg.org/ticket/6375
"-movflags",
"+faststart", # optimize for streaming
"-preset",
"veryslow", # https://trac.ffmpeg.org/wiki/Encode/H.264#a2.Chooseapresetandtune
"-timelimit",
"900", # prevent DoS (exit after 15 min)
"-vf",
"pad=ceil(iw/2)*2:ceil(ih/2)*2", # https://stackoverflow.com/questions/20847674/ffmpeg-libx264-height-not-divisible-by-2#20848224
output_filename,
])
elif target_format == "png":
ffmpeg_process = subprocess.Popen([
"ffmpeg",
"-v",
"error",
"-threads",
str(config["ffmpeg_threads"]),
"-thread_type",
"slice",
"-i",
input_filename,
"-timelimit",
"60", # prevent DoS (exit after 15 min)
output_filename,
])
# Update progress while ffmpeg is alive
old_progress = ""
while ffmpeg_process.poll() == None:
try:
raw_output_size = utils.filesize(output_filename)
except FileNotFoundError:
raw_output_size = 0
if raw_output_size >= MAXIMUM_FILESIZE_ALLOWED:
update_status_message(text.error.huge_file)
ffmpeg_process.kill()
utils.rm(output_filename)
input_size = utils.bytes2human(raw_input_size)
output_size = utils.bytes2human(raw_output_size)
progress = f"{output_size} / {input_size}"
# Update progress only if it changed
if progress != old_progress:
update_status_message(text.converting.format(progress))
old_progress = progress
time.sleep(2)
# Exit in case of error with ffmpeg
if ffmpeg_process.returncode != 0:
update_status_message(text.error.converting)
# Clean up and close pipe explicitly
utils.rm(output_filename)
return
# Check output file size
output_size = utils.filesize(output_filename)
if output_size >= MAXIMUM_FILESIZE_ALLOWED:
update_status_message(text.error.huge_file)
# Clean up and close pipe explicitly
utils.rm(output_filename)
return
# Default params for sending operation
data = {
"chat_id": message.chat.id,
"reply_to_message_id": message.message_id
}
if target_format == "mp4":
data.update({"supports_streaming": True})
# 1. Get video duration in seconds
video_duration = subprocess.run(
[
"ffprobe",
"-v",
"error",
"-select_streams",
"v:0",
"-show_entries",
"format=duration",
"-of",
"default=noprint_wrappers=1:nokey=1",
output_filename,
],
stdout=subprocess.PIPE,
).stdout.decode("utf-8").strip()
video_duration = round(float(video_duration))
data.update({"duration": video_duration})
# 2. Get video height and width
video_props = subprocess.run(
[
"ffprobe",
"-v",
"error",
"-select_streams",
"v:0",
"-show_entries",
"stream=width,height",
"-of",
"csv=s=x:p=0",
output_filename,
],
stdout=subprocess.PIPE,
).stdout.decode("utf-8").strip()
video_width, video_height = video_props.split("x")
data.update({"width": video_width, "height": video_height})
# 3. Take one frame from the middle of the video
update_status_message(text.generating_thumbnail)
thumbnail = "".join(
[config["temp_path"],
utils.random_string(), ".jpg"])
generate_thumbnail_process = subprocess.Popen([
"ffmpeg",
"-v",
"error",
"-i",
output_filename,
"-vcodec",
"mjpeg",
"-vframes",
"1",
"-an",
"-f",
"rawvideo",
"-ss",
str(int(video_duration / 2)),
# keep the limit of 90px height/width (Telegram API) while preserving the aspect ratio
"-vf",
"scale='if(gt(iw,ih),90,trunc(oh*a/2)*2)':'if(gt(iw,ih),trunc(ow/a/2)*2,90)'",
thumbnail,
])
# While process is alive (i.e. is working)
while generate_thumbnail_process.poll() == None:
time.sleep(1)
# Exit in case of error with ffmpeg
if generate_thumbnail_process.returncode != 0:
update_status_message(text.error.generating_thumbnail)
return
update_status_message(text.uploading)
requests.post(
"https://api.telegram.org/bot{}/sendVideo".format(
config["telegram_token"]),
data=data,
files=[
("video", (utils.random_string() + ".mp4",
open(output_filename, "rb"), "video/mp4")),
("thumb", (utils.random_string() + ".jpg",
open(thumbnail, "rb"), "image/jpeg")),
],
)
utils.rm(input_filename)
utils.rm(output_filename)
utils.rm(thumbnail)
elif target_format == "png":
# Upload to Telegram
update_status_message(text.uploading)
requests.post(
"https://api.telegram.org/bot{}/sendPhoto".format(
config["telegram_token"]),
data=data,
files=[("photo", (utils.random_string() + ".png",
open(output_filename, "rb"), "image/png"))],
)
requests.post(
"https://api.telegram.org/bot{}/sendDocument".format(
config["telegram_token"]),
data=data,
files=[("document", (utils.random_string() + ".png",
open(output_filename, "rb"), "image/png"))],
)
utils.rm(input_filename)
utils.rm(output_filename)
bot.delete_message(message.chat.id, status_message.message_id)
def test_non_existing(self):
with self.assertRaises(AssertionError):
rm(Path('asdf_path'))
def main(argv):
# Input parameters (from Arguments.py)
imaster = IMASTER
length = LENGTH_GLOBAL
syndir = 'SYNTH_traces'
o_wpinversion = O_WPINVERSION
nc = NC
nl = NL
solfile = None
flagreg = False
# Parse options
try:
opts, args = go.gnu_getopt(argv[1:], 'i:d:rh',
["icmtf=", "osydir=", "regional", "help"])
except go.GetoptError as err:
sys.stderr.write('usage: %s [option] (for help see %s -h)\n' %
(sys.argv[0], sys.argv[0]))
raise
for o, a in opts:
if o == '-h' or o == '--help':
disphelp(sys.argv[0], solfile, syndir)
sys.exit(0)
if o == '-r' or o == '--regional':
length = LENGTH_REGIONAL
flagreg = True
if o == '-i' or o == '--icmtf':
solfile = a
if not os.path.exists(solfile):
raise IOError('No wcmtfile named %s' % (solfile))
if o == '-d' or o == '--osyndir':
syndir = a
if not solfile:
for f in ['xy_WCMTSOLUTION', 'ts_WCMTSOLUTION', 'WCMTSOLUTION']:
if os.path.exists(f):
solfile = f
break
if not solfile:
raise IOError(
'No wcmtfile available, can be specified with --icmtf')
eq = EarthQuake()
eq.rcmtfile(solfile)
cmtla, cmtlo = eq.lat, eq.lon
# Title
conf = utils.parseConfig(imaster)
title = '_'.join(conf['EVNAME'].split())
title += ', filter = (%s, %s, %s, %s)' % (
conf['filt_cf1'], conf['filt_cf2'], conf['filt_order'],
conf['filt_pass'])
# Cleanup run dir
if os.path.exists(syndir) and syndir != '.' and syndir != './':
utils.rm(syndir)
if syndir != '.' and syndir != './':
os.mkdir(syndir)
if not os.path.exists(LOGDIR):
os.mkdir(LOGDIR)
for l in os.listdir('.'):
if l[:4] == 'page' and l[-4:] == '.pdf':
utils.rm(l)
# Compute synthetics
cmd = SYNTHS + ' ' + imaster + ' ' + solfile + ' ' + o_wpinversion + ' ' + syndir
print(cmd)
#status = call(cmd, shell=True, stdin=sys.stdin, stdout=sys.stdout);
status = os.system(SYNTHS + ' ' + imaster + ' ' + solfile + ' ' +
o_wpinversion + ' ' + syndir + ' > ' +
os.path.join(LOGDIR, '_tmp_synths'))
if status:
print('Error while running ' + SYNTHS)
sys.exit(1)
# Create Sac Objects
sacdata = sacpy.sac()
sacsynt = sacpy.sac()
coords = []
L = open(o_wpinversion).readlines()
for l in L:
sacf = l.strip().split()[0]
sacdata.read(sacf, datflag=0)
coords.append([sacdata.stla, sacdata.stlo, sacdata.az, sacdata.dist])
coords = np.array(coords)
# Main loop
print('Input (W)CMTSOLUTION file is: %s' % (solfile))
print('Output synthetic directory is: %s' % (syndir))
perpage = nl * nc
ntot = len(L)
npages = np.ceil(float(ntot) / float(perpage))
nchan = 1
count = 1
pages = 1
fig = plt.figure()
fig.subplots_adjust(bottom=0.06,
top=0.87,
left=0.06,
right=0.95,
wspace=0.25,
hspace=0.4)
print('All pages will be saved in %s' % (OPDFFILE))
pp = mpl.backends.backend_pdf.PdfPages(OPDFFILE)
basem = None
for l in L:
# Parse line
items = l.strip().split()
fic1 = items[0]
sacdata.read(fic1)
chan = sacdata.kcmpnm[0:3]
loc = sacdata.khole
fic2 = syndir+'/%s.%s.%s.%s.complete_synth.bp.sac'\
%(sacdata.kstnm,sacdata.knetwk,chan,loc)
sacsynt.read(fic2)
# pages
if count > perpage:
plt.suptitle(title + ', p %d/%d' % (pages, npages),
fontsize=16,
y=0.95)
print('page %d/%d' % (pages, npages))
#fig.set_rasterized(True)
pp.savefig(orientation='landscape')
plt.close()
pages += 1
count = 1
fig = plt.figure()
fig.subplots_adjust(bottom=0.06,
top=0.87,
left=0.06,
right=0.95,
wspace=0.25,
hspace=0.4)
# Time - W phase window
t1 = np.arange(sacdata.npts,
dtype='double') * sacdata.delta + sacdata.b - sacdata.o
t2 = np.arange(sacsynt.npts,
dtype='double') * sacsynt.delta + sacsynt.b - sacsynt.o
wnb = float(items[5])
wne = float(items[6])
wtb = sacdata.b - sacdata.o + wnb * sacdata.delta
wte = sacdata.b - sacdata.o + wne * sacdata.delta
# Plot trace
ax = plt.subplot(nl, nc, count)
plt.plot(t1, sacdata.depvar * 1000., 'k')
plt.plot(t2, sacsynt.depvar * 1000., 'r-')
plt.plot([wtb, wte], [0., 0.], 'ro')
# Axes limits
B = wtb - 150.0
if B < 0:
B = 0.0
plt.xlim([B, B + length * sacsynt.delta])
if YLIM_AUTO:
a = np.absolute(sacsynt.depvar[:length]).max() * 1000.
ymin = -1.1 * a
ymax = 1.1 * a
ylims = [ymin, ymax]
else:
ylims = YLIMFIXED
plt.ylim(ylims)
# Annotations
plt.rcParams['font.family'] = 'sans-serif'
plt.rcParams['font.sans-serif'] = 'Arial'
plt.rcParams['mathtext.fontset'] = 'custom'
plt.rcParams['mathtext.rm'] = 'sans'
plt.rcParams['mathtext.it'] = 'sans:italic'
plt.rcParams['mathtext.default'] = 'it'
if sacdata.kcmpnm[2] == 'Z':
if sys.version_info >= (2, 7):
label = r'%s %s %s %s $(\phi,\Delta) = %6.1f\degree, %6.1f\degree$' % (
sacdata.knetwk, sacdata.kstnm, sacdata.kcmpnm,
sacdata.khole, sacdata.az, sacdata.gcarc)
else:
label = r'%s %s %s %s Az=%3.0f, delta=%3.0f' % (
sacdata.knetwk, sacdata.kstnm, sacdata.kcmpnm,
sacdata.khole, sacdata.az, sacdata.gcarc)
else:
if sys.version_info >= (2, 7):
label = r'%s %s %s %s $(\phi,\Delta,\alpha) = %6.1f\degree,'
label += '%6.1f\degree, %6.1f\degree$'
label = label % (sacdata.knetwk, sacdata.kstnm, sacdata.kcmpnm,
sacdata.khole, sacdata.az, sacdata.gcarc,
sacdata.cmpaz)
else:
label = r'%s %s %s %s Az=%3.0f, delta=%3.0f'
label = label % (sacdata.knetwk, sacdata.kstnm, sacdata.kcmpnm,
sacdata.khole, sacdata.az, sacdata.gcarc)
plt.title(label, fontsize=10.0, va='center', ha='center')
if not (count - 1) % nc:
plt.ylabel('mm', fontsize=10)
if (count - 1) / nc == nl - 1 or nchan + nc > ntot:
plt.xlabel('time, sec', fontsize=10)
plt.grid()
try:
basem = showBasemap(ax, cmtla, cmtlo, sacdata.stla, sacdata.stlo,
coords, flagreg, basem)
except:
showPolarmap(ax, sacdata.az, sacdata.dist, coords)
print('Cannot use basemap')
count += 1
nchan += 1
print('page %d/%d' % (pages, npages))
#fig.set_rasterized(True)
plt.suptitle(title + ', p %d/%d' % (pages, npages), fontsize=16, y=0.95)
pp.savefig(orientation='landscape')
plt.close()
pp.close()
def main(argv):
# Extract command line options
try:
opts, args = getopt.gnu_getopt(argv[1:],'stpSdi:nhz',["hdsafe","onlyts","onlyxy","npar",
"imas=","strike=","dc","nont","dip=",
"rake=","mom=","noref","xyz","old",
"help"])
except getopt.GetoptError as err:
usage()
raise
# Parse command line options
i_master = IMASTER
fastflag = True
flagts = True
flagxy = True
flagxyz = False
flagref = True
sdrM0 = {}
for o, a in opts:
if o == '-h' or o == '--help':
disphelp()
sys.exit(0)
if o == '-s' or o == '--hdsafe':
fastflag = False
if o == '-t' or o == '--onlyts':
if not flagts:
usage()
raise getopt.GetoptError('options -t and -p cannot be used simultaneously')
flagxy = False
flagts = True
if o == '-p' or o == '--onlyxy':
if not flagxy:
usage()
raise getopt.GetoptError('options -t and -p cannot be used simultaneously')
flagts = False
fastflag = False
flagxy = True
if o == '--dc':
sdrM0['-dc']=''
if o == '--nont':
sdrM0['-nont']=''
if o == '--strike':
sdrM0['-strike']=a
if o == '--dip':
sdrM0['-dip']=a
if o == '--rake':
sdrM0['-rake']=a
if o == '--mom':
sdrM0['-mom']=a
if o == '-i' or o == '--imas':
i_master = a
if o == '-n' or o == '--noref':
flagref = False
if o == '-z' or o == '--xyz':
flagxyz = True
if o == '--old':
WPINV_XY += ' -old'
# Read i_master
iconfig = utils.parseConfig(i_master)
cmtref = iconfig['CMTFILE']
evname = iconfig['EVNAME'].replace(' ','_').replace(',','')
# Set comments in output ps file
Median = '-med '
if 'P2P_SCREENING' in iconfig:
if iconfig['P2P_SCREENING'] != 'YES':
Median = ' '
ths = '5.0 3.0 0.9'
if 'RMS_SCREENING' in iconfig:
ths = iconfig['RMS_SCREENING']
comments = [VERSION,'GF_PATH: '+GF_PATH,'Screening: '+Median+ths]
# Read reference CMTFILE
eq = EarthQuake()
eq.rcmtfile(cmtref)
eq.title = evname.strip().replace(' ','_').replace(',','')
cmtf = open(cmtref,'r')
L=cmtf.readlines()
cmtf.close()
if len(L) < 13:
print('*** WARNING : no reference solution in %s'%(cmtref))
flagref = False
# TS and/or LAT/LON Grid-search
if (flagts or flagxy) and not flagxyz:
grid_search(eq,cmtref,TS_NIT,TS_DT,TSBOUNDS,XY_NIT,XY_DX,XY_NX,XY_NOPT,fastflag,
flagts,flagxy,sdrM0,ts_ofile=TS_OFILE,xy_ofile=XY_OFILE,comments=comments)
# TS and LAT/LON/DEP Grid-search
if flagxyz:
grid_search(eq,cmtref,TS_NIT,TS_DT,TSBOUNDS,XYZ_NIT,XYZ_DX,XYZ_NX,XYZ_NOPT,fastflag,
flagts,flagxyz,sdrM0,dz=DDEP,minz=MINDEP,ts_ofile=TS_OFILE,xy_ofile=XYZ_OFILE,
comments=comments)
if flagxy:
eq.wcmtfile('_tmp_CMTSOLUTION.xyz')
if flagref:
addrefsol(cmtref,'_tmp_CMTSOLUTION.xyz')
grid_search(eq,'_tmp_CMTSOLUTION.xyz',TS_NIT,TS_DT,TSBOUNDS,XY_NIT,XY_DX,XY_NX,XY_NOPT,
0,0,1,sdrM0,ts_ofile=TS_OFILE,xy_ofile=XY_OFILE,comments=comments)
utils.rm('_tmp_CMTSOLUTION.xyz')
# Cleaning up
if os.path.exists('_tmp_ts_table'):
utils.rm('_tmp_ts_table')
if os.path.exists('_tmp_xy_table'):
utils.rm('_tmp_xy_table')
def protocol(self, req):
self.buf += req
while True:
before, sep, after = self.buf.partition('\n')
if not sep:
break
try:
data = json.loads(before)
except Exception as e:
print('Unable to parse json:', e)
print('Data:', before)
raise e
name = data.get('name')
if name == 'patch':
# TODO: we should do this in a separate thread
Listener.apply_patch(data)
elif name == 'get_buf':
buf_id = data['id']
listener.BUFS[buf_id] = data
view = listener.get_view(buf_id)
if view:
Listener.update_view(data, view)
else:
listener.save_buf(data)
elif name == 'create_buf':
listener.BUFS[data['id']] = data
listener.save_buf(data)
elif name == 'rename_buf':
new = utils.get_full_path(data['path'])
old = utils.get_full_path(data['old_path'])
new_dir = os.path.split(new)[0]
if new_dir:
utils.mkdir(new_dir)
os.rename(old, new)
view = listener.get_view(data['id'])
if view:
view.retarget(new)
elif name == 'delete_buf':
path = utils.get_full_path(data['path'])
utils.rm(path)
listener.delete_buf(data['id'])
elif name == 'room_info':
# Success! Reset counter
self.retries = G.MAX_RETRIES
self.room_info = data
G.PERMS = data['perms']
if 'patch' not in data['perms']:
msg.log('We don\'t have patch permission. Setting buffers to read-only')
project_json = {
'folders': [
{'path': G.PROJECT_PATH}
]
}
utils.mkdir(G.PROJECT_PATH)
with open(os.path.join(G.PROJECT_PATH, '.sublime-project'), 'w') as project_fd:
project_fd.write(json.dumps(project_json, indent=4, sort_keys=True))
floo_json = {
'url': utils.to_room_url({
'host': self.host,
'owner': self.owner,
'port': self.port,
'room': self.room,
'secure': self.secure,
})
}
with open(os.path.join(G.PROJECT_PATH, '.floo'), 'w') as floo_fd:
floo_fd.write(json.dumps(floo_json, indent=4, sort_keys=True))
for buf_id, buf in data['bufs'].iteritems():
buf_id = int(buf_id) # json keys must be strings
buf_path = utils.get_full_path(buf['path'])
new_dir = os.path.dirname(buf_path)
utils.mkdir(new_dir)
listener.BUFS[buf_id] = buf
try:
buf_fd = open(buf_path, 'r')
buf_buf = buf_fd.read().decode('utf-8')
md5 = hashlib.md5(buf_buf.encode('utf-8')).hexdigest()
if md5 == buf['md5']:
msg.debug('md5 sums match. not getting buffer')
buf['buf'] = buf_buf
else:
msg.debug('md5 for %s should be %s but is %s. getting buffer' % (buf['path'], buf['md5'], md5))
raise Exception('different md5')
except Exception as e:
msg.debug('Error calculating md5:', e)
Listener.get_buf(buf_id)
self.authed = True
G.CONNECTED = True
msg.log('Successfully joined room %s/%s' % (self.owner, self.room))
if self.on_connect:
self.on_connect(self)
self.on_connect = None
elif name == 'join':
msg.log('%s joined the room' % data['username'])
self.room_info['users'][data['user_id']] = data['username']
elif name == 'part':
msg.log('%s left the room' % data['username'])
try:
del self.room_info['users'][data['user_id']]
except Exception as e:
print('Unable to delete user %s from user list' % (data))
region_key = 'floobits-highlight-%s' % (data['user_id'])
for window in sublime.windows():
for view in window.views():
view.erase_regions(region_key)
elif name == 'highlight':
region_key = 'floobits-highlight-%s' % (data['user_id'])
Listener.highlight(data['id'], region_key, data['username'], data['ranges'], data.get('ping', False))
elif name == 'error':
message = 'Floobits: Error! Message: %s' % str(data.get('msg'))
msg.error(message)
elif name == 'disconnect':
message = 'Floobits: Disconnected! Reason: %s' % str(data.get('reason'))
msg.error(message)
sublime.error_message(message)
self.stop()
elif name == 'msg':
self.on_msg(data)
else:
msg.error('unknown name!', name, 'data:', data)
self.buf = after
def main(argv):
# Input parameters (from Arguments.py)
imaster = IMASTER
length = LENGTH_GLOBAL
syndir = 'SYNTH_traces'
o_wpinversion = O_WPINVERSION
nc = NC
nl = NL
solfile = None
flagreg = False
# Parse options
try:
opts, args = go.gnu_getopt(argv[1:],'i:d:rh',["icmtf=","osydir=","regional","help"])
except go.GetoptError as err:
sys.stderr.write('usage: %s [option] (for help see %s -h)\n'%(sys.argv[0],sys.argv[0]))
raise
for o, a in opts:
if o == '-h' or o == '--help':
disphelp(sys.argv[0],solfile,syndir)
sys.exit(0)
if o == '-r' or o == '--regional':
length = LENGTH_REGIONAL
flagreg = True
if o == '-i' or o=='--icmtf':
solfile = a
if not os.path.exists(solfile):
raise IOError('No wcmtfile named %s'%(solfile))
if o == '-d' or o == '--osyndir':
syndir = a
if not solfile:
for f in ['xy_WCMTSOLUTION','ts_WCMTSOLUTION','WCMTSOLUTION']:
if os.path.exists(f):
solfile = f
break
if not solfile:
raise IOError('No wcmtfile available, can be specified with --icmtf')
eq = EarthQuake()
eq.rcmtfile(solfile)
cmtla,cmtlo = eq.lat, eq.lon
# Title
conf = utils.parseConfig(imaster)
title = '_'.join(conf['EVNAME'].split())
title += ', filter = (%s, %s, %s, %s)'%(conf['filt_cf1'],conf['filt_cf2'],conf['filt_order'],conf['filt_pass'])
# Cleanup run dir
if os.path.exists(syndir) and syndir != '.' and syndir != './':
utils.rm(syndir)
if syndir != '.' and syndir != './':
os.mkdir(syndir)
if not os.path.exists(LOGDIR):
os.mkdir(LOGDIR)
for l in os.listdir('.'):
if l[:4]=='page' and l[-4:]=='.pdf':
utils.rm(l)
# Compute synthetics
cmd = SYNTHS+' '+imaster+' '+solfile+' '+o_wpinversion+' '+syndir
print(cmd)
#status = call(cmd, shell=True, stdin=sys.stdin, stdout=sys.stdout);
status = os.system(SYNTHS+' '+imaster+' '+solfile+' '+o_wpinversion+' '+syndir+' > '+os.path.join(LOGDIR,'_tmp_synths'))
if status:
print('Error while running '+SYNTHS)
sys.exit(1)
# Create Sac Objects
sacdata = sacpy.sac()
sacsynt = sacpy.sac()
coords = []
L = open(o_wpinversion).readlines()
for l in L:
sacf = l.strip().split()[0]
sacdata.read(sacf,datflag=0)
coords.append([sacdata.stla,sacdata.stlo,sacdata.az,sacdata.dist])
coords = np.array(coords)
# Main loop
print('Input (W)CMTSOLUTION file is: %s'%(solfile))
print('Output synthetic directory is: %s'%(syndir))
perpage = nl*nc
ntot = len(L)
npages = np.ceil(float(ntot)/float(perpage))
nchan = 1
count = 1
pages = 1
fig = plt.figure()
fig.subplots_adjust(bottom=0.06,top=0.87,left=0.06,right=0.95,wspace=0.25,hspace=0.35)
print('All pages will be saved in %s'%(OPDFFILE))
pp = mpl.backends.backend_pdf.PdfPages(OPDFFILE)
basem = None
for l in L:
# Parse line
items = l.strip().split()
fic1 = items[0]
sacdata.read(fic1)
chan = sacdata.kcmpnm[0:3]
loc = sacdata.khole
fic2 = syndir+'/%s.%s.%s.%s.complete_synth.bp.sac'\
%(sacdata.kstnm,sacdata.knetwk,chan,loc)
sacsynt.read(fic2)
# pages
if count > perpage:
plt.suptitle(title+ ', p %d/%d'%(pages,npages), fontsize=16, y=0.95)
print('page %d/%d'%(pages,npages))
#fig.set_rasterized(True)
pp.savefig(orientation='landscape')
plt.close()
pages += 1
count = 1
fig = plt.figure()
fig.subplots_adjust(bottom=0.06,top=0.87,left=0.06,right=0.95,wspace=0.25,hspace=0.35)
# Time - W phase window
t1 = np.arange(sacdata.npts,dtype='double')*sacdata.delta + sacdata.b - sacdata.o
t2 = np.arange(sacsynt.npts,dtype='double')*sacsynt.delta + sacsynt.b - sacsynt.o
wnb = float(items[5])
wne = float(items[6])
wtb = sacdata.b - sacdata.o + wnb * sacdata.delta
wte = sacdata.b - sacdata.o + wne * sacdata.delta
# Plot trace
ax = plt.subplot(nl,nc,count)
plt.plot(t1,sacdata.depvar*1000.,'k')
plt.plot(t2,sacsynt.depvar*1000.,'r-')
plt.plot([wtb,wte],[0.,0.],'ro')
# Axes limits
B=wtb-150.0
if B<0:
B = 0.0
plt.xlim([B,B+length*sacsynt.delta])
if YLIM_AUTO:
a = np.absolute(sacsynt.depvar[:length]).max()*1000.
ymin = -1.1*a
ymax = 1.1*a
ylims = [ymin,ymax]
else:
ylims = YLIMFIXED
plt.ylim(ylims)
# Annotations
if sacdata.kcmpnm[2] == 'Z':
label = u'%s %s %s %s (\u03C6,\u0394) = %6.1f\u00B0, %5.1f\u00B0'
label = label%(sacdata.knetwk,sacdata.kstnm, sacdata.kcmpnm, sacdata.khole,
sacdata.az, sacdata.gcarc)
else:
label = u'%s %s %s %s (\u03C6,\u0394,\u03B1) = %6.1f\u00B0, %5.1f\u00B0, %6.1f\u00B0'
label = label%(sacdata.knetwk,sacdata.kstnm, sacdata.kcmpnm, sacdata.khole,
sacdata.az, sacdata.gcarc, sacdata.cmpaz)
plt.title(label,fontsize=10.0,va='center',ha='center')
if not (count-1)%nc:
plt.ylabel('mm',fontsize=10)
if (count-1)/nc == nl-1 or nchan+nc > ntot:
plt.xlabel('time, sec',fontsize=10)
plt.grid()
try:
basem = showBasemap(ax,cmtla,cmtlo,sacdata.stla,sacdata.stlo,coords,flagreg,basem)
except:
showPolarmap(ax,sacdata.az,sacdata.dist,coords)
print('No basemap module')
count += 1
nchan += 1
print('page %d/%d'%(pages,npages))
#fig.set_rasterized(True)
plt.suptitle(title + ', p %d/%d'%(pages,npages), fontsize=16, y=0.95)
pp.savefig(orientation='landscape')
plt.close()
pp.close()
