def copy_lib_files(data, source, target): """Copy auxiliary files to target Args: data (dict): simulation db target (py.path): destination directory """ for f in lib_files(data, source): path = target.join(f.basename) pkio.mkdir_parent_only(path) if not path.exists(): if not f.exists(): sim_resource = resource_dir(data.simulationType) r = sim_resource.join(f.basename) # the file doesn't exist in the simulation lib, check the resource lib if r.exists(): pkio.mkdir_parent_only(f) r.copy(f) else: pkdlog('No file in lib or resource: {}', f) continue if source: # copy files from another session f.copy(path) else: # symlink into the run directory path.mksymlinkto(f, absolute=False)
def api_uploadFile(simulation_type, simulation_id, file_type): f = flask.request.files['file'] filename = werkzeug.secure_filename(f.filename) p = _lib_filepath(simulation_type, filename, file_type) err = None file_list = None if p.check(): confirm = flask.request.form['confirm'] if 'confirm' in flask.request.form else None if not confirm: search_name = _lib_filename(simulation_type, filename, file_type) file_list = _simulations_using_file(simulation_type, file_type, search_name, ignore_sim_id=simulation_id) if file_list: err = 'File is in use in other simulations. Please confirm you would like to replace the file for all simulations.' if not err: pkio.mkdir_parent_only(p) f.save(str(p)) template = sirepo.template.import_module(simulation_type) if hasattr(template, 'validate_file'): err = template.validate_file(file_type, str(p)) if err: pkio.unchecked_remove(p) if err: return http_reply.gen_json({ 'error': err, 'filename': filename, 'fileList': file_list, 'fileType': file_type, 'simulationId': simulation_id, }) return http_reply.gen_json({ 'filename': filename, 'fileType': file_type, 'simulationId': simulation_id, })
def _before_request(fc): target = srunit.server.app.sirepo_db_dir.join('beaker', 'container_file', filename[0:1], filename[0:2], filename) pkio.mkdir_parent_only(target) shutil.copy(str(pkunit.data_dir().join(filename)), str(target))
def _find_tls_crt(j2_ctx, domain): from rsconf.pkcli import tls from rsconf import db d = db.secret_path(j2_ctx, TLS_SECRET_SUBDIR, visibility='global') for crt, domains in j2_ctx.component.tls_crt.items(): if domain in domains: return d.join(crt), domains for s in ( domain, domain.replace('.', '_'), '.'.join([_WILDCARD_TLS] + domain.split('.')[1:]), # sirepo.com is in wildcard cert star.sirepo.com _WILDCARD_TLS + '.' + domain, ): src = d.join(s) # due to dots in domain, we can't use ext= if (src + tls.KEY_EXT).check(): return src, domain assert j2_ctx.component.tls_crt_create, \ f'{domain}: tls crt for domain not found' src = d.join(domain) pkio.mkdir_parent_only(src) tls.gen_self_signed_crt(str(src), *domain) return src, domain
def _create_example(example): data = simulation_db.save_new_example(example) # ensure all datafiles for the new example exist in the sim lib dir for f in template_common.lib_files(data): if not f.exists(): r = template_common.resource_dir(data.simulationType).join(f.basename) assert r.exists(), 'Example missing resource file: {}'.format(f) pkio.mkdir_parent_only(f) r.copy(f)
def _before_request(fc): target = srunit.server._app.sirepo_db_dir.join('beaker', 'container_file', filename[0:1], filename[0:2], filename) pkio.mkdir_parent_only(target) shutil.copy(str(pkunit.data_dir().join(filename)), str(target)) for h in header.split('; '): x = h.split('=') fc.set_cookie('', *x)
def _crt_create(basename, op): from rsconf.pkcli import tls res = pkcollections.Dict( key=basename + tls.KEY_EXT, crt=basename + tls.CRT_EXT, ) if res.crt.check(): return res pkio.mkdir_parent_only(res.crt) return op()
def secret_path(hdb, filename, visibility=None, qualifier=None): if visibility: assert visibility in VISIBILITY_LIST, \ '{}: invalid visibility, must be {}'.format( visibility, VISIBILITY_LIST, ) else: visibility = VISIBILITY_DEFAULT p = [] if visibility == VISIBILITY_GLOBAL else [qualifier or hdb.rsconf_db[visibility]] p.append(filename) res = hdb.rsconf_db.secret_d.join(*p) pkio.mkdir_parent_only(res) return res
def import_file(req, tmp_dir=None, **kwargs): if not pkio.has_file_extension(req.filename, 'zip'): raise sirepo.util.UserAlert('unsupported import filename: {}'.format(filename)) #TODO(pjm): writing to simulation lib for now, tmp_dir will get removed after this request filepath = str(simulation_db.simulation_lib_dir(SIM_TYPE).join(_ZIP_FILE_NAME)) pkio.mkdir_parent_only(filepath) with open(filepath, 'wb') as f: f.write(req.file_stream.read()) data = simulation_db.default_data(SIM_TYPE) data['models']['simulation']['name'] = req.filename data['models']['simulation'][_TMP_INPUT_FILE_FIELD] = filepath # more processing occurs in prepare_for_client() via: # import_file => _save_new_and_reply => api_simulationData => prepare_for_client return data
def tls_key_and_crt(j2_ctx, domain): from rsconf.pkcli import tls from rsconf import db src, domains = _find_tls_crt(j2_ctx, domain) src_key = src + tls.KEY_EXT src_crt = src + tls.CRT_EXT if not src_crt.check(): assert j2_ctx.component.tls_crt_create, \ '{}: missing crt for: {}'.format(src_crt, domain) pkio.mkdir_parent_only(src_crt) # https://stackoverflow.com/a/42730929 # Cannot pass (basename=str(src), *domains) tls.gen_self_signed_crt(str(src), *domains) assert src_key.check(), \ '{}: missing key for: {}'.format(src_key, domain) return PKDict(key=src_key, crt=src_crt)
def init_tree(name, author, author_email, description, license, url): """Setup a project tree with: docs, tests, etc., and checkin to git. Creates: setup.py, index.rst, project dir, <name>_console.py, etc. Overwrites files if they exist without checking. Args: name (str): short name of the project, e.g. ``pykern``. author (str): copyright holder, e.g. ``RadiaSoft LLC`` author_email (str): how to reach author, e.g. ``[email protected]`` description (str): one-line summary of project license (str): url of license url (str): website for project, e.g. http://pykern.org """ assert os.path.isdir('.git'), \ 'Must be run from the root directory of the repo' assert not os.path.isdir(name), \ '{}: already exists, only works on fresh repos'.format(name) assert name == py.path.local().basename, \ '{}: name must be the name of the current directory'.format(name) license = license.lower() base = pkresource.filename('projex') values = copy.deepcopy(DEFAULTS) values.update({ 'name': name, 'author': author, 'description': description, 'author_email': author_email, 'url': url, 'license': _license(license, 0), 'classifier_license': _license(license, 1), }) values['copyright_license_rst'] = values['copyright_license_rst'].format(**values) suffix_re = r'\.jinja$' for src in pkio.walk_tree(base, file_re=suffix_re): dst = py.path.local(src).relto(str(base)) dst = dst.replace('projex', name).replace('dot-', '.') dst = re.sub(suffix_re, '', dst) pkio.mkdir_parent_only(dst) _render(src, values, output=dst) src = py.path.local(pkresource.filename('projex-licenses')) src = src.join(license + '.jinja') _render(src, values, output='LICENSE')
def setup_cluster(compt, hosts, tls_d, run_u, j2_ctx): from rsconf import db compt.install_access(mode='700', owner=run_u) compt.install_directory(tls_d) b = db.secret_path(j2_ctx, compt.name + '_' + _TLS_BASENAME, visibility='host') pkio.mkdir_parent_only(b) for h in hosts: c, ca = _self_signed_crt(j2_ctx, h) c = _signed_crt(j2_ctx, ca, b.join(h)) d = tls_d.join(h) compt.install_access(mode='700') compt.install_directory(d) compt.install_access(mode='400') # POSIT: sirepo.runner.docker uses {cacert,cert,key}.pem compt.install_abspath(ca.crt, d.join('cacert.pem')) compt.install_abspath(c.crt, d.join('cert.pem')) compt.install_abspath(c.key, d.join('key.pem'))
def _bash_append_and_dst(self, host_path, ignore_exists=False, file_contents=None, file_src=None): dst = self.hdb.build.dst_d.join(host_path) if dst.check(): if ignore_exists: return None raise AssertionError('{}: dst already exists'.format(dst)) pkio.mkdir_parent_only(dst) md5 = None if file_src: assert file_contents is None, \ '{}: do not pass both file_contents and file_src'.format(host_path) file_contents = file_src.read_binary() if file_contents: md5 = _md5(self._write_binary(dst, file_contents)) self._bash_append(host_path, md5=md5) return dst
def _flash_create_sim_files(cls, data, run_dir): import sirepo.mpi import sirepo.template.flash subprocess.check_output( [ 'tar', '--extract', '--gunzip', f'--file={cls._flash_src_tarball_basename()}', f'--directory={run_dir}', ], stderr=subprocess.STDOUT, ) s = run_dir.join(cls.sim_type()) d = [] if data.models.problemFiles.archive: for i, r in cls._flash_extract_problem_files_archive( run_dir.join(cls._flash_problem_files_archive_basename(data)), ): b = pkio.py_path(i.filename).basename #TODO(pjm): zip file also includes required datafiles # if not re.match(r'(\w+\.F90)|(Makefile)', b): # continue p = cls.flash_simulation_unit_file_path(run_dir, data, b) pkio.mkdir_parent_only(p) pkio.write_text(p, r()) d.append(p.basename) cls._flash_check_datafiles(data, d) sirepo.template.flash.generate_config_file(run_dir, data) t = s.join(cls.schema().constants.flashAppName) c = sirepo.template.flash.setup_command(data) pkdc('setup_command={}', ' '.join(c)) cls._flash_run_command_and_parse_log_on_error( c, s, cls._SETUP_LOG, r'(.*PPDEFINE.*$)|(^\s+\*.*$(\n\w+.*)?)', ) flash_schema = flash_parser.SetupParameterParser( run_dir.join(cls.sim_type(), cls.schema().constants.flashAppName) ).generate_schema() cls._add_default_views(flash_schema) pkio.write_text( run_dir.join(cls.SETUP_PARAMS_SCHEMA_FILE), pkjson.dump_pretty(PKDict( flashSchema=flash_schema, )) ) datafiles = flash_schema.enum.SetupDatafiles cls._flash_run_command_and_parse_log_on_error( ['make', f'-j{sirepo.mpi.cfg.cores}'], t, cls._COMPILE_LOG, r'^(?:Error): (.*)', ) for c, b in PKDict({ v: v for v in [f[0] for f in datafiles] }).pkupdate( # POSIT: values match cls._sim_file_basenames flash4=cls.flash_exe_basename(data), ).items(): p = t.join(c) cls.delete_sim_file(cls._flash_file_prefix(b), data) cls.put_sim_file(p, b, data) if p.check(link=1): p.copy(run_dir.join(b)) else: p.move(run_dir.join(b))
def _before_request(fc): target = srunit.server.app.sirepo_db_dir.join( 'beaker', 'container_file', filename[0:1], filename[0:2], filename) pkio.mkdir_parent_only(target) shutil.copy(str(pkunit.data_dir().join(filename)), str(target))
def default_command(): import rsconf.component import rsconf.component.rsconf import rsconf.pkcli.tls root_d = db.cfg.root_d if root_d.check(): return '{}: already exists'.format(root_d) srv = pkio.mkdir_parent(root_d.join(db.SRV_SUBDIR)) def _sym(old, new_base=None): old = pkio.py_path(old) if not new_base: new_base = old.basename assert old.check(), \ '{}: does not exist'.format(old) srv.join(new_base).mksymlinkto(old, absolute=False) # ssh-keygen -q -N '' -C rsconf -t rsa -b 4096 -f /var/tmp/foo # -- don't need this db_d = pkio.mkdir_parent(root_d.join(db.DB_SUBDIR)) secret_d = pkio.mkdir_parent(db_d.join(db.SECRET_SUBDIR)) nginx_d = pkio.mkdir_parent(root_d.join(NGINX_SUBDIR)) boot_hdb = pkcollections.Dict(rsconf_db=pkcollections.Dict( secret_d=secret_d, channel='dev', )) j2_ctx = pkcollections.Dict( all_host='v9.radia.run', group=grp.getgrgid(os.getgid())[0], host='v4.radia.run', master='v3.radia.run', port=2916, root_d=root_d, srv_d=str(srv), uid=os.getuid(), user=pwd.getpwuid(os.getuid())[0], worker5_host='v5.radia.run', worker6_host='v6.radia.run', ) hosts = [h for h in j2_ctx.values() if str(h).endswith('.radia.run')] # bootstrap j2_ctx.update(boot_hdb) j2_ctx.rsconf_db.http_host = 'http://{}:{}'.format(j2_ctx.master, j2_ctx.port) j2_ctx.bkp = pkcollections.Dict(primary=j2_ctx.host) j2_ctx.passwd_f = rsconf.component.rsconf.passwd_secret_f(j2_ctx) for h in hosts: _add_host(j2_ctx, srv, h) _sym('~/src/radiasoft/download/bin/install.sh', 'index.html') _sym(pkresource.filename('rsconf/rsconf.sh'), 'rsconf.sh') dev_d = pkio.py_path(pkresource.filename('dev')) for f in pkio.walk_tree(dev_d): if str(f).endswith('~') or str(f).startswith('#'): continue x = str(f.relto(dev_d)) if not ('local/' in x and x.endswith('.sh.jinja')): x = re.sub('.jinja$', '', x) dst = root_d.join(x) pkio.mkdir_parent_only(dst) if f.basename == dst.basename: f.copy(dst) else: pkjinja.render_file(f, j2_ctx, output=dst, strict_undefined=True) n = [] for e in 'rpm', 'proprietary': d = pkio.py_path(root_d.dirname).join(e) pkio.mkdir_parent(d) root_d.join(e).mksymlinkto(d, absolute=False) n.append(str(d)) subprocess.check_call(['bash', str(secret_d.join('setup_dev.sh')), *n], ) # dev only, really insecure, but makes consistent builds easy _sym('~/src/radiasoft') _sym('~/src/biviosoftware') for h in hosts: # needed to be able to talk setup certs for registry so we can # pull private images from all hosts. Only used in dev, because # private registry doesn't protect against pushes from these hosts. if h != j2_ctx.master: subprocess.check_call( ['rsconf', 'host', 'init_docker_registry', h]) tls_d = secret_d.join(rsconf.component.TLS_SECRET_SUBDIR) tls_d.ensure(dir=True) for h in ( 'jupyter.' + j2_ctx.all_host, 'jupyter.' + j2_ctx.host, j2_ctx.all_host, j2_ctx.master, j2_ctx.worker5_host, j2_ctx.worker6_host, ): rsconf.pkcli.tls.gen_self_signed_crt( tls_d.join(h), h, ) rsconf.pkcli.tls.gen_self_signed_crt( tls_d.join('star.' + j2_ctx.host), '*.' + j2_ctx.host, j2_ctx.host, )