def do_delete(self):
'delete compiled output files'
dst_dir = os.path.join(self.cfg.dst_dir, self.nav.test_id)
for fname in utils.glob_files(dst_dir, ['*']):
fullname = os.path.join(dst_dir, fname)
log.debug('del %s', fullname)
os.remove(fullname)
self.msgs.append('rm {}'.format(fullname))
shutil.rmtree(dst_dir)
self.msgs.append('rmtree {}'.format(dst_dir))
return self
style_config=conf,
no_local_style=None,
in_place=change_file,
print_diff=False,
verify=True,
parallel=True,
verbose=True)
if __name__ == "__main__":
BAZEL_ROOT = utils.find_bazel_root()
STYLE_CONF_PATH = BAZEL_ROOT + "/.style.yapf"
USER = pwd.getpwuid(os.getuid())[0]
projects = utils.CHECK_PROJECTS(sys.argv[1:])
if "//..." in projects:
projects = [p.replace(BAZEL_ROOT, "/")[:-1] for p in glob.glob(BAZEL_ROOT + '/*/')]
projects = [p for p in projects if p not in utils.BLACKLISTED_BAZEL_TARGETS]
for p in projects:
if p.endswith("/..."):
p = p[:-4]
path = BAZEL_ROOT + '/' + p[2:]
files = utils.glob_files(path, VALID_PY_FILE_TYPES)
if files != []:
changed = lint(USER, files, STYLE_CONF_PATH)
if changed:
print(
"\033[93mWARNING:\033[0m This command modified your files with the recommended linting, you should review the changes before committing"
)
sys.exit(1)
wlog("Reading Column Density Map Data", log, verb, t=True)
# Column Density Maps ata format types. The column density maps can be in
# either .npz (numpy) or .h5 (hdf5) file formats.
npz_format = [".npz", "npz"]
h5_format = [".h5", "h5"]
# Need to speficy the redshifts of the npz files
ColDensMapsZVals = params["ColDensMapZVals"]
# If the Column Density Maps are .npz files convert them to .h5 files
if params["ColDensMapSuffix"] in npz_format:
wlog("Column Density Maps in .npz format", log, verb)
wlog("Converting files to .h5 files", log, verb)
ColDensMaps = utils.glob_files(params["ColDensMapDir"],
"*" + params["ColDensMapSuffix"])
for i in range(len(ColDensMaps)):
old_fn = ColDensMaps[i].split(".npz")[0].split("/")[-1]
new_fn = "{0}/{1}.h5".format(params["OutputDataDir"], old_fn)
redshift = np.array([ColDensMapsZVals[i]])
wlog("Converting {0} to .h5 format".format(old_fn), log, verb)
utils.convert_npz_to_h5(ColDensMaps[i], new_fn, z=redshift)
wlog("Created: {0}".format(new_fn), log, verb)
# Change file path to the path of the new .h5 file
ColDensMaps[i] = new_fn
elif params["ColDensMapSuffix"] in h5_format:
wlog("Column Density Maps in .h5 format", log, verb)
def _run(self):
'run the compile job in a new thread'
self.running = True
log.debug('Start compile job for %s', self.job)
# pick up job details via test_id in Mantra Index
idxs = utils.MantraIdx(self.cfg.src_dir, self.cfg.dst_dir).sync()
idx = idxs.test_id(self.job)
if idx is None:
log.debug('No idx entry for job %r', self.job)
self.running = False
return
# add Handler, captures job log msgs to specific logfile
dst_dir = os.path.join(self.cfg.dst_dir, idx.test_id)
os.makedirs(dst_dir, exist_ok=True)
self.logfile = os.path.join(dst_dir, 'mtr.log') # for this job
log.debug('logging to %s', self.logfile)
handler = logging.FileHandler(self.logfile)
handler.setFormatter(FORMAT)
handler.addFilter(ThreadFilter(self.job))
log.addHandler(handler)
log.info('Parsing source: %s', idx.src)
# clear output directory (carefully)
log.info('Delete (most) files:')
for fname in utils.glob_files(dst_dir,
includes=['*'],
excludes=['./mtr.log', '*.png']
):
fname = os.path.join(dst_dir, fname)
if fname != self.logfile and not fname.endswith('.png'):
log.info('- del %s', fname)
os.remove(fname)
# parse the source file -> p.meta, p.tags, p.qstn
p = Parser(idx).parse()
# save to dst_dir
log.info('Add new files:')
for nr, q in enumerate(p.qstn):
qfname = os.path.join(dst_dir, 'q{:03d}.json'.format(nr))
q.save(qfname)
log.debug('- add %s', qfname)
log.info('Copy images (if needed):')
_copy_files(p.imgs) # copy images
# create mtr.idx
idx = idx._replace(flag='P')
fname = os.path.join(dst_dir, 'mtr.idx')
with open(fname, 'wt') as fh:
fh.write(json.dumps(idx))
log.info('Created %s', fname)
for fld in idx._fields:
log.info('- %-8s: %s', fld, getattr(idx, fld))
# create quiz.json
log.debug('meta:')
for k, v in p.meta.items():
log.debug('%-12s: %s', k, v)
log.info('All done!')
# give UI some time to display logfile contents & remove handler
log.info('Going to sleep')
time.sleep(4)
log.info('Dying now')
log.removeHandler(handler)
log = logging.getLogger('werkzeug')
log.setLevel(logging.ERROR)
# - Module logger
logger.setup(cfg.app_name, cfg.log_file)
log = logging.getLogger(cfg.app_name)
log.debug('logging via %s', log.name)
for msg in config.errors:
log.error(msg)
for msg in config.warnings:
log.warning(msg)
utils.MantraIdx(config.cfg.src_dir, config.cfg.dst_dir).save()
# remove lingering mtr.log files (server interrupted during compile)
for fname in utils.glob_files(cfg.dst_dir, ['[!.]*/mtr.log']):
log.debug('removing residue (%s)', fname)
os.remove(os.path.join(cfg.dst_dir, fname))
# - Helpers
def urlparms(href):
'turn href into dict with mantra-fields'
# Mantra uses the following scheme:
# - url = scheme://usr:pwd@netloc:8050/path;param?query=arg#frag
# - parts after path are optional
# - param, if present, is always a unique test_id
# - qry=arg[;q2=arg2,..], if any, is meaningful only to app_<path>
# - frag, if any, is a single fragment, meaningful only to app_<path>
# - examples:
# - /compile;test_id