def default_command(in_file): """Reads `in_file` passes to `msg.jobCmd` Must be called in run_dir Writes its output on stdout. Args: in_file (str): json parsed to msg Returns: str: json output of command, e.g. status msg """ try: job.init() f = pkio.py_path(in_file) msg = pkjson.load_any(f) #TODO(e-carlin): find common place to serialize/deserialize paths msg.runDir = pkio.py_path(msg.runDir) f.remove() res = globals()['_do_' + msg.jobCmd](msg, sirepo.template.import_module( msg.simulationType)) if res is None: return r = PKDict(res).pksetdefault(state=job.COMPLETED) except Exception as e: r = PKDict( state=job.ERROR, error=e.sr_args.error if isinstance(e, sirepo.util.UserAlert) else str(e), stack=pkdexc(), ) return pkjson.dump_pretty(r, pretty=False)
def _init_git(): """Init git locally and to bitbucket""" from pykern import pkcli from pykern import pkio import datetime import re import subprocess title = pkio.py_path().basename v = datetime.datetime.utcnow().strftime('%Y%m%d-%H%M%S') name = 'sim-{}-{}'.format(pkio.py_path().basename, v).lower() r, ctx = _git_api_request( 'post', 'repositories/{user}/{repo}', dict( repo=name, json=dict( scm='git', is_private=True, fork_policy='no_public_forks', name=name, ), ), ) repo_url = r.json()['links']['clone'][0]['href'] #TODO(robnagler) add README.md if not already there subprocess.check_call(['git', 'init']) subprocess.check_call(['git', 'remote', 'add', 'origin', repo_url]) subprocess.check_call(['git', 'config', 'user.name', ctx['user']]) if pkio.pkunit_prefix: _pkunit_setup(ctx) subprocess.check_call(['git', 'checkout', '-b', 'master']) _out_dir() _git_commit('init')
async def _fastcgi_op(self, msg): if msg.runDir: _assert_run_dir_exists(pkio.py_path(msg.runDir)) if not self.fastcgi_cmd: m = msg.copy() m.jobCmd = 'fastcgi' self._fastcgi_file = cfg.fastcgi_sock_dir.join( f'sirepo_job_cmd-{cfg.agent_id:8}.sock', ) self._fastcgi_msg_q = sirepo.tornado.Queue(1) pkio.unchecked_remove(self._fastcgi_file) m.fastcgiFile = self._fastcgi_file # Runs in an agent's directory and chdirs to real runDirs. # Except in stateless_compute which doesn't interact with the db. m.runDir = pkio.py_path() # Kind of backwards, but it makes sense since we need to listen # so _do_fastcgi can connect self._fastcgi_remove_handler = tornado.netutil.add_accept_handler( tornado.netutil.bind_unix_socket(str(self._fastcgi_file)), self._fastcgi_accept, ) # last thing, because of await: start fastcgi process await self._cmd(m, send_reply=False) self._fastcgi_msg_q.put_nowait(msg) self.fastcgi_cmd.op_id = msg.opId return None
def _init_root(*args): global _root if args: assert not cfg.root, \ 'Cannot set both SIREPO_SRDB_ROOT ({}) and SIREPO_SERVER_DB_DIR ({})'.format( cfg.root, args[0], ) cfg.root = args[0] v = cfg.root if v: assert os.path.isabs(v), \ '{}: SIREPO_SRDB_ROOT must be absolute'.format(v) assert os.path.isdir(v), \ '{}: SIREPO_SRDB_ROOT must be a directory and exist'.format(v) v = pkio.py_path(v) else: assert pkconfig.channel_in('dev'), \ 'SIREPO_SRDB_ROOT must be configured except in DEV' fn = sys.modules[pkinspect.root_package(_init_root)].__file__ root = pkio.py_path(pkio.py_path(pkio.py_path(fn).dirname).dirname) # Check to see if we are in our dev directory. This is a hack, # but should be reliable. if not root.join('requirements.txt').check(): # Don't run from an install directory root = pkio.py_path.local('.') v = pkio.mkdir_parent(root.join(_DEFAULT_ROOT)) _root = v
def dump_pretty(obj, filename=None, pretty=True, **kwargs): """Formats as json as string If an object is not encoded by default, will call str() on the object. Args: obj (object): any Python object filename (str or py.path): where to write [None] pretty (bool): pretty print [True] kwargs (object): other arguments to `json.dumps` Returns: str: sorted and formatted JSON """ if pretty: res = json.dumps(obj, indent=4, separators=(',', ': '), sort_keys=True, cls=Encoder, **kwargs) + '\n' else: res = json.dumps(obj, separators=(',', ':'), cls=Encoder, **kwargs) if filename: from pykern import pkio pkio.py_path(filename).write(res) return res
def init_apis(*args, **kwargs): global cfg cfg = pkconfig.init( dst_db_root=( pkio.py_path(sirepo.srdb.root()).join('jupyterhub'), pkio.py_path, 'new jupyter user db', ), rs_jupyter_migrate=( False, bool, 'give user option to migrate data from jupyter.radiasoft.org'), src_db_root=( pkio.py_path('/var/empty'), pkio.py_path, 'existing jupyter user db (ex /srv/jupyterhub)', ), uri_root=('jupyter', str, 'the root uri of jupyterhub'), ) sirepo.auth_db.init_model(_init_model) sirepo.events.register({ 'auth_logout': _event_auth_logout, 'end_api_call': _event_end_api_call, 'github_authorized': _event_github_authorized, })
def _init_root(): global _cfg, _root def _cfg_root(v): """Config value or root package's parent or cwd with `_DEFAULT_ROOT`""" if not os.path.isabs(v): pkconfig.raise_error(f'{v}: SIREPO_SRDB_ROOT must be absolute') if not os.path.isdir(v): pkconfig.raise_error(f'{v}: SIREPO_SRDB_ROOT must be a directory and exist') return pkio.py_path(v) _cfg = pkconfig.init( root=(None, _cfg_root, 'where database resides'), ) _root = _cfg.root if _root: return _root assert pkconfig.channel_in('dev'), \ 'SIREPO_SRDB_ROOT must be configured except in dev' r = pkio.py_path( sys.modules[pkinspect.root_package(_init_root)].__file__, ).dirpath().dirpath() # Check to see if we are in our dev directory. This is a hack, # but should be reliable. if not r.join('requirements.txt').check(): # Don't run from an install directory r = pkio.py_path('.') _root = pkio.mkdir_parent(r.join(_DEFAULT_ROOT)) return _root
def internal_build(self): from rsconf import systemd from rsconf.component import network from rsconf.component import logrotate from rsconf.component import bop self.buildt.require_component('base_all') jc = self.hdb.j2_ctx_copy() z = jc.spamd conf_d = pkio.py_path('/etc/mail/spamassassin') nc = self.buildt.get_component('network') z.sa_update_keys_d = conf_d.join('sa-update-keys') z.trusted_networks = ' '.join(nc.trusted_nets()) watch = bop.install_perl_rpms(self, jc) + [conf_d] z.run_d = systemd.custom_unit_prepare(self, jc, watch) z.log_postrotate_f = z.run_d.join('log_postrotate') socket_d = pkio.py_path('/run/spamd') z.socket_path = pkio.py_path('/run/spamd/spamd.sock') self.install_access(mode='755', owner=jc.rsconf_db.run_u) self.install_directory(conf_d) self.install_directory(socket_d) self.install_access(mode='500') self.install_resource('spamd/log_postrotate.sh', jc, z.log_postrotate_f) self.install_access(mode='444') self.install_resource( 'spamd/spamc.conf', jc, conf_d.join('spamc.conf'), ) logrotate.install_conf(self, jc) self.append_root_bash_with_main(jc) systemd.custom_unit_enable(self, jc)
def get_data_file(run_dir, model, frame, **kwargs): data = simulation_db.read_json( run_dir.join(template_common.INPUT_BASE_NAME)) path = pkio.py_path( _SIM_DATA.lib_file_for_sim(data, _FRAME_FILENAME[frame])) if not path.exists() and frame == _SCHEMA.constants.dose2FrameId: # no alternate dose exists, use main dose instead path = pkio.py_path(_SIM_DATA.lib_file_for_sim(data, RTDOSE_FILE)) return PKDict(filename=path)
def copy_related_files(data, source_path, target_path): # copy results and log for the long-running simulations for m in ('animation', ): # copy any simulation output s = pkio.py_path(source_path).join(m) if not s.exists(): continue t = pkio.py_path(target_path).join(m) pkio.mkdir_parent(str(t)) for f in pkio.sorted_glob('*'): f.copy(t)
def test_read_all(): from pykern import pkio from pykern import pkjson from pykern import pkunit from pykern.pkunit import pkok, pkeq, pkre from pykern.pkdebug import pkdp from pykern.pkcli import rsmanifest import re with pkunit.save_chdir_work(is_pkunit_prefix=True) as d: rsmanifest.add_code( 'code1', version='1.1', uri='http://x.com', source_d='/tmp', pyenv='py2', ) v = pkjson.load_any(pkio.py_path(rsmanifest.USER_FILE)).version pkjson.dump_pretty( {'version': v, 'image': {'type': 'docker'}}, filename=rsmanifest.CONTAINER_FILE, ) m = rsmanifest.read_all() pkeq(v, m.version) pkeq('docker', m.image.type) pkeq('1.1', m.codes.py2.code1.version)
async def run_extract_job(run_dir, cmd, backend_info): # no output redirection here - we want to let the docker daemon collect it quoted_bash_cmd = ' '.join(shlex.quote(c) for c in cmd) # We never run more than one extract job at a time in a given run_dir, so # no need to give them unique names. working_dir = pkio.py_path(backend_info.working_dir) container = await _make_container( run_dir, working_dir, quoted_bash_cmd, 'extract', ) try: result = await _container_wait(container) stdout = await trio.run_sync_in_worker_thread( functools.partial( container.logs, stdout=True, stderr=False, ) ) stderr = await trio.run_sync_in_worker_thread( functools.partial( container.logs, stdout=False, stderr=True, ) ) return pkcollections.Dict( returncode=result['StatusCode'], stdout=stdout, stderr=stderr, ) finally: await trio.run_sync_in_worker_thread(container.remove)
def _flash_run_command_and_parse_log_on_error( cls, command, work_dir, log_file, regex, ): p = pkio.py_path(log_file) with pkio.open_text(p.ensure(), mode='r+') as l: try: subprocess.run( command, check=True, cwd=work_dir, stderr=l, stdout=l, ) except subprocess.CalledProcessError as e: l.seek(0) c = l.read() m = [x.group().strip() for x in re.finditer( regex, c, re.MULTILINE, )] if m: r = '\n'.join(m) else: r = c.splitlines()[-1] raise sirepo.util.UserAlert( r, '{}', e )
def _do_compute(msg, template): msg.runDir = pkio.py_path(msg.runDir) with msg.runDir.join(template_common.RUN_LOG).open('w') as run_log: p = subprocess.Popen( _do_prepare_simulation(msg, template).cmd, stdout=run_log, stderr=run_log, ) while True: for j in range(20): time.sleep(.1) r = p.poll() i = r is None if not i: break if msg.isParallel: # TODO(e-carlin): This has a potential to fail. We likely # don't want the job to fail in this case _write_parallel_status(msg, template, i) if i: continue if r != 0: return PKDict(state=job.ERROR, error='non zero returncode={}'.format(r)) return PKDict(state=job.COMPLETED)
def merge_app_vars(j2_ctx, app_name): z = j2_ctx.bop z.setdefault('is_production', j2_ctx.rsconf_db.channel == 'prod') z.is_test = not z.is_production z.run_u = j2_ctx.rsconf_db.run_u db.merge_dict(z, j2_ctx[app_name]) z.app_name = app_name # all apps are secured by TLS now z.can_secure = True z.setdefault('bconf_aux', '') z.setdefault('block_goto', False) z.setdefault('client_max_body_size', _DEFAULT_CLIENT_MAX_BODY_SIZE) z.source_code_d = SOURCE_CODE_D # Assumed by PetShop's crm-ticket-deletion.btest z.want_status_email = z.perl_root == PETSHOP_ROOT if z.is_test: z.http_host = _domain(j2_ctx, z.vhosts[0])[0] z.mail_host = z.http_host z.run_d = systemd.unit_run_d(j2_ctx, app_name) z.conf_f = z.run_d.join('httpd.conf') z.bconf_f = z.run_d.join('bivio.bconf') z.initdb_f = z.run_d.join('initdb.sh') z.initdb_cmd = 'source {}'.format(z.initdb_f) z.log_postrotate_f = z.run_d.join('reload') z.httpd_cmd = "/usr/sbin/httpd -d '{}' -f '{}'".format(z.run_d, z.conf_f) z.local_file_root_d = pkio.py_path('/var/www/facades') return z
def start(): #TODO(robnagler) commands need their own init hook like the server has job.init() global cfg cfg = pkconfig.init( agent_id=pkconfig.Required(str, 'id of this agent'), fastcgi_sock_dir=( pkio.py_path('/tmp'), pkio.py_path, 'directory of fastcfgi socket, must be less than 50 chars'), start_delay=(0, pkconfig.parse_seconds, 'delay startup in internal_test mode'), supervisor_uri=pkconfig.Required( str, 'how to connect to the supervisor', ), ) pkdlog('{}', cfg) if pkconfig.channel_in_internal_test() and cfg.start_delay: pkdlog('start_delay={}', cfg.start_delay) time.sleep(cfg.start_delay) i = tornado.ioloop.IOLoop.current() d = _Dispatcher() def s(*args): return i.add_callback_from_signal(_terminate, d) signal.signal(signal.SIGTERM, s) signal.signal(signal.SIGINT, s) i.spawn_callback(d.loop) i.start()
def __init__(self, *args, **kwargs): super(T, self).__init__(*args, **kwargs) self.root_d = pkio.py_path(cfg.root_d) self.db_d = self.root_d.join(DB_SUBDIR) self.proprietary_source_d = self.root_d.join(PROPRIETARY_SUBDIR) self.rpm_source_d = self.root_d.join(RPM_SUBDIR) self.tmp_d = self.root_d.join(TMP_SUBDIR) self.secret_d = self.db_d.join(SECRET_SUBDIR) self.srv_d = self.root_d.join(SRV_SUBDIR) self.srv_host_d = self.srv_d.join(HOST_SUBDIR) self.base = PKDict() f = None try: for d in self.db_d, self.secret_d: for f in pkio.sorted_glob(d.join(ZERO_YML)): v = pkyaml.load_str( pkjinja.render_file( f, self.base, strict_undefined=True, ), ) merge_dict(self.base, v) except Exception: pkdlog('error rendering db={}', f) raise
def _init(): import sirepo.mpi global SCHEMA_COMMON, cfg, JOB_RUN_MODE_MAP cfg = pkconfig.init( nfs_tries=(10, int, 'How many times to poll in hack_nfs_write_status'), nfs_sleep=(0.5, float, 'Seconds sleep per hack_nfs_write_status poll'), sbatch_display=(None, str, 'how to display sbatch cluster to user'), tmp_dir=(None, pkio.py_path, 'Used by utilities (not regular config)'), ) fn = STATIC_FOLDER.join('json/schema-common{}'.format(JSON_SUFFIX)) with open(str(fn)) as f: SCHEMA_COMMON = json_load(f) # In development, any schema update creates a new version if pkconfig.channel_in('dev'): SCHEMA_COMMON.version = max([ _timestamp(pkio.py_path(fn).mtime()) \ for fn in glob.glob(str(STATIC_FOLDER.join('json/*{}'.format(JSON_SUFFIX)))) ]) else: SCHEMA_COMMON.version = sirepo.__version__ JOB_RUN_MODE_MAP = PKDict( sequential='Serial', parallel='{} cores (SMP)'.format(sirepo.mpi.cfg.cores), ) if cfg.sbatch_display: JOB_RUN_MODE_MAP.sbatch = cfg.sbatch_display
def _prepare_hosts(self, jc, z): nc = self.buildt.get_component('network') z.ip, _ = nc.ip_and_net_for_host(jc.rsconf_db.host) if not z.is_first: return res = [] for h in z.hosts: ip, net = nc.ip_and_net_for_host(h) if 'net' in z: assert net == z.net, \ 'net={} for host={} not on same net={}'.format( net, h, z.net, ) else: z.net = net s = self._gen_keys(jc, z, h) res.append( pkcollections.Dict( host=h, ip=ip, host_key_pub=pkio.read_text(s.host_key_pub_f).rstrip(), # replace this (first) host with current host (h) so we don't # have to recompute identity_f=pkio.py_path( str(z.guest.identity_f).replace(jc.rsconf_db.host, h), )), ) # Convert to CIDR z.net = str(z.net.name) z.max_slots = z.slots_per_host * len(res) z.hosts_sorted = sorted(res, key=lambda x: x.ip)
def api_copyNonSessionSimulation(): req = http_request.parse_post(id=True, template=True) simulation_db.verify_app_directory(req.type) src = pkio.py_path( simulation_db.find_global_simulation( req.type, req.id, checked=True, ), ) data = simulation_db.open_json_file( req.type, src.join(simulation_db.SIMULATION_DATA_FILE), ) data.pkdel('report') data.models.simulation.isExample = False data.models.simulation.outOfSessionSimulationId = req.id res = _save_new_and_reply(data) sirepo.sim_data.get_class(req.type).lib_files_from_other_user( data, simulation_db.lib_dir_from_sim_dir(src), ) target = simulation_db.simulation_dir(req.type, data.models.simulation.simulationId) #TODO(robnagler) does not work, supervisor needs to be notified to # copy the simulation state. # if hasattr(req.template, 'copy_related_files'): # req.template.copy_related_files(data, str(src), str(target)) return res
def _sym(old, new_base=None): old = pkio.py_path(old) if not new_base: new_base = old.basename assert old.check(), \ '{}: does not exist'.format(old) srv.join(new_base).mksymlinkto(old, absolute=False)
def api_staticFile(path_info=None): """flask.send_from_directory for static folder. Uses safe_join which doesn't allow indexing, paths with '..', or absolute paths. Args: path_info (str): relative path to join Returns: flask.Response: flask.send_from_directory response """ if not path_info: raise util.raise_not_found('empty path info') p = pkio.py_path( flask.safe_join(str(simulation_db.STATIC_FOLDER), path_info)) r = None if _google_tag_manager and re.match(r'^en/[^/]+html$', path_info): return http_reply.headers_for_cache( flask.make_response( _google_tag_manager_re.sub( _google_tag_manager, pkio.read_text(p), ), ), path=p, ) if re.match(r'^html/[^/]+html$', path_info): return http_reply.render_html(p) return flask.send_file(p, conditional=True)
def _do_compute(msg, template): msg.runDir = pkio.py_path(msg.runDir) with msg.runDir.join(template_common.RUN_LOG).open('w') as run_log: p = subprocess.Popen( _do_prepare_simulation(msg, template).cmd, stdout=run_log, stderr=run_log, ) while True: for j in range(20): time.sleep(.1) r = p.poll() i = r is None if not i: break if msg.isParallel: # TODO(e-carlin): This has a potential to fail. We likely # don't want the job to fail in this case _write_parallel_status(msg, template, i) if i: continue return _on_do_compute_exit( r == 0, msg.isParallel, template, msg.runDir, )
def _cfg_root(v): """Config value or root package's parent or cwd with `_DEFAULT_ROOT`""" if not os.path.isabs(v): pkconfig.raise_error(f'{v}: SIREPO_SRDB_ROOT must be absolute') if not os.path.isdir(v): pkconfig.raise_error(f'{v}: SIREPO_SRDB_ROOT must be a directory and exist') return pkio.py_path(v)
def test_read_all(): from pykern import pkio from pykern import pkjson from pykern import pkunit from pykern.pkunit import pkok, pkeq, pkre from pykern.pkdebug import pkdp from pykern.pkcli import rsmanifest import re with pkunit.save_chdir_work(is_pkunit_prefix=True) as d: rsmanifest.add_code( 'code1', version='1.1', uri='http://x.com', source_d='/tmp', pyenv='py2', ) v = pkjson.load_any(pkio.py_path(rsmanifest.USER_FILE)).version pkjson.dump_pretty( { 'version': v, 'image': { 'type': 'docker' } }, filename=rsmanifest.CONTAINER_FILE, ) m = rsmanifest.read_all() pkeq(v, m.version) pkeq('docker', m.image.type) pkeq('1.1', m.codes.py2.code1.version)
def test_py_path(): from pykern import pkunit from pykern import pkio from pykern.pkunit import pkeq with pkunit.save_chdir_work(): d = pkunit.data_dir() pkeq(d, pkio.py_path(d))
def _find(paths): from pykern import pkio import re i = re.compile(r'(?:_work|_data)/') res = [] cwd = pkio.py_path() for t in paths or ('tests', ): t = pkio.py_path(t) if t.check(file=True): res.append(str(cwd.bestrelpath(t))) continue for p in pkio.walk_tree(t, re.compile(r'_test\.py$')): p = str(cwd.bestrelpath(p)) if not i.search(p): res.append(p) return res
def _gen_req(which, basename, domains, is_ca=False): first = domains[0] if not basename: basename = first basename = pkio.py_path(basename) if is_ca: # pathlen:0 means can only be used for signing certs, not for # signing intermediate certs. alt = '''x509_extensions = x509_req [x509_req] basicConstraints=critical,CA:true,pathlen:0''' else: # Must always provide subjectAltName except for is_ca # see https://github.com/urllib3/urllib3/issues/497 # which points to RFC 2818: # Although the use of the Common Name is existing practice, it is deprecated and # Certification Authorities are encouraged to use the dNSName instead. alt = '{}_extensions = v3_req\n[v3_req]\n{}'.format( 'req' if which == 'csr' else 'x509', _alt_names(domains), ) c = ''' [req] distinguished_name = subj prompt = no {} [subj] C = US ST = Colorado L = Boulder CN = {}'''.format(alt, first) cfg = basename + '.cfg' try: cfg.write(c) key = basename + KEY_EXT out = basename + '.' + which cmd = [ 'openssl', 'req', '-nodes', '-newkey', 'rsa:2048', '-keyout', str(key), '-out', str(out), '-config', str(cfg), ] if which == _CRT: cmd += ['-x509'] + _signing_args() _run(cmd) finally: pkio.unchecked_remove(cfg) res = pkcollections.Dict(key=key) res[which] = out return res
def _pkunit_setup(ctx): from pykern import pkio import subprocess f = pkio.py_path('git-credentials') f.write('https://{user}:{pass}@{host}'.format(**ctx)) f.chmod(0600) subprocess.check_call(['git', 'config', 'credential.helper', 'cache']) subprocess.check_call(['git', 'config', 'credential.helper', 'store --file ' + str(f)])
def __slots_from_dump(cls, pool_name): p = pkio.py_path(_POOLS_DUMP_FILE) if not p.exists(): return PKDict() slots = PKDict() for s in pkjson.load_any(p).pkunchecked_nested_get( f'{pool_name}.slots', ) or []: slots[s.cname] = s return slots
def _cfg_tls_dir(value): if not value: assert pkconfig.channel_in('dev'), \ 'required config' return None res = pkio.py_path(value) assert res.check(dir=True), \ 'directory does not exist; value={}'.format(value) return res
def run(cfg_dir): cfg_dir = pkio.py_path(cfg_dir) data = simulation_db.read_json(template_common.INPUT_BASE_NAME) if data['report'] == 'doseCalculation': _run_dose_calculation(data, cfg_dir) elif data['report'] == 'dvhReport': _run_dvh(data, cfg_dir) else: raise RuntimeError('unknown report: {}'.format(data['report']))
def run(cfg_dir): cfg_dir = pkio.py_path(cfg_dir) data = simulation_db.read_json(template_common.INPUT_BASE_NAME) if data['report'] == 'doseCalculation': _run_dose_calculation(data, cfg_dir) elif data['report'] == 'dvhReport': _run_dvh(data, cfg_dir) else: raise AssertionError('unknown report: {}'.format(data['report']))
def dump_pretty(obj, filename=None, pretty=True): """Formats as json as string Args: obj (object): any Pyton object filename (str or py.path): where to write [None] pretty (bool): pretty print [True] Returns: str: sorted and formatted JSON """ from pykern import pkio import json import py.path if pretty: res = json.dumps(obj, indent=4, separators=(',', ': '), sort_keys=True) + '\n' else: res = json.dumps(obj) if filename: pkio.py_path(filename).write(res) return res
def _call(args): """Run a command with the proper local python and path environment Args: args (tuple): what to run (flags and all) """ from pykern import pkio import subprocess import os ub = pkio.py_path(_PYTHON_USER_BASE) env = os.environ.copy() env['PATH'] = str(ub.join('bin')) + ':' + env['PATH'] env['PYTHONUSERBASE'] = str(ub) subprocess.check_call(args, env=env)
def restore(git_txz): """Restores the git directory (only) to a new directory with the .git.txz suffix """ m = re.search('(([^/]+)\.git)\.txz$', git_txz) if not m: raise ValueError(git_txz, ': does not end in .git.txz') git_txz = pkio.py_path(git_txz) d = m.group(2) pkdc('restore: {}', d) g = m.group(1) with pkio.save_chdir(d, mkdir=True): _shell(['tar', 'xJf', str(git_txz)]) os.rename(g, '.git') _shell(['git', 'config', 'core.bare', 'false']) _shell(['git', 'config', 'core.logallrefupdates', 'true']) _shell(['git', 'checkout'])
def test_add_code(): from pykern import pkio from pykern import pkjson from pykern import pkunit from pykern.pkunit import pkok, pkeq, pkre from pykern.pkdebug import pkdp from pykern.pkcli import rsmanifest import re with pkunit.save_chdir_work(is_pkunit_prefix=True) as d: rsmanifest.add_code('A', 'b', 'c', 'd', pyenv='v') j = pkjson.load_any(pkio.py_path(rsmanifest.USER_FILE).read()) pkok(20170101.0 < float(j.version), 'version must be after 2017') pkeq('A', j.codes.v.a.name) pkeq('b', j.codes.v.a.version) rsmanifest.add_code('a', 'bb', 'cc', 'dd') j = pkjson.load_any(pkio.expand_user_path(rsmanifest.USER_FILE).read()) pkeq('A', j.codes.v.a.name) pkeq('a', j.codes[''].a.name) pkeq('bb', j.codes[''].a.version) pkre('20.*T.*Z', j.codes[''].a.installed)
def _out_dir(): from pykern import pkio p = pkio.py_path(_OUT_DIR).ensure_dir() p.join('.gitignore').write('*\n!.gitignore\n')
def _init_python_user_base(): """Ensure all python_user_base files are committed""" from pykern import pkio ub = pkio.py_path(_PYTHON_USER_BASE).ensure_dir() ub.join('.gitignore').write('!*\n')
def test_get_data_file(): from sirepo import srunit from pykern import pkunit from pykern import pkio import sdds fc = srunit.flask_client() fc.get('/elegant') data = fc.sr_post( 'listSimulations', {'simulationType': 'elegant', 'search': {'simulationName': 'fourDipoleCSR'}}, ) data = data[0].simulation data = fc.sr_get( 'simulationData', params=dict( pretty='1', simulation_id=data.simulationId, simulation_type='elegant', ), ) run = fc.sr_post( 'runSimulation', dict( forceRun=False, models=data.models, report='bunchReport1', simulationId=data.models.simulation.simulationId, simulationType=data.simulationType, ), ) for _ in range(10): run = fc.sr_post( 'runStatus', run.nextRequest ) if run.state == 'completed': break time.sleep(1) else: pkunit.pkfail('runStatus: failed to complete: {}', run) resp = fc.sr_get( 'downloadDataFile', dict( simulation_type=data.simulationType, simulation_id=data.models.simulation.simulationId, model='bunchReport1', frame='-1', suffix='csv', ), raw_response=True, ) rows = csv.reader(StringIO.StringIO(resp.get_data())) assert len(list(rows)) == 5001 resp = fc.sr_get( 'downloadDataFile', dict( simulation_type=data.simulationType, simulation_id=data.models.simulation.simulationId, model='bunchReport1', frame='-1', ), raw_response=True, ) m = re.search(r'attachment; filename="([^"]+)"', resp.headers['Content-Disposition']) with pkunit.save_chdir_work(): path = pkio.py_path(m.group(1)) with open(str(path), 'w') as f: f.write(resp.get_data()) assert sdds.sddsdata.InitializeInput(0, str(path)) == 1, \ '{}: sdds failed to open'.format(path) # Verify we can read something assert 0 <= len(sdds.sddsdata.GetColumnNames(0)) sdds.sddsdata.Terminate(0)
def _parent_file(cfg_dir, filename): return str(pkio.py_path(cfg_dir.dirname).join(filename))
import sirepo.template import threading import time import werkzeug.exceptions #: Json files JSON_SUFFIX = '.json' #: Schema common values, e.g. version SCHEMA_COMMON = None #: Simulation file name is globally unique to avoid collisions with simulation output SIMULATION_DATA_FILE = 'sirepo-data' + JSON_SUFFIX #: The root of the pkresource tree (package_data) RESOURCE_FOLDER = pkio.py_path(pkresource.filename('')) #: Where server files and static files are found STATIC_FOLDER = RESOURCE_FOLDER.join('static') #: Verify ID _IS_PARALLEL_RE = re.compile('animation', re.IGNORECASE) #: How to find examples in resources _EXAMPLE_DIR = 'examples' #: Valid characters in ID _ID_CHARS = numconv.BASE62 #: length of ID _ID_LEN = 8