def parse_user_config(alias_d, colour_d): from accelerator.compat import open from os import environ cfgdir = environ.get('XDG_CONFIG_HOME') if not cfgdir: home = environ.get('HOME') if not home: return None cfgdir = join(home, '.config') fn = join(cfgdir, 'accelerator', 'config') try: fh = open(fn, 'r', encoding='utf-8') except IOError: return None with fh: from configparser import ConfigParser cfg = ConfigParser() cfg.optionxform = str # case sensitive (don't downcase aliases) cfg.read_file(fh) if 'alias' in cfg: alias_d.update(cfg['alias']) if 'colour' in cfg: colour_d.update({ k: [_unesc_re.sub(_unesc, e) for e in v.split()] for k, v in cfg['colour'].items() })
def open_input(self, filename, mode='r', encoding=None, errors=None): assert 'r' in mode, "Don't write to input files" if 'b' not in mode and encoding is None: encoding = 'utf-8' return open(self.input_filename(filename), mode, encoding=encoding, errors=errors)
def open(self, filename, mode='r', sliceno=None, encoding=None, errors=None): assert 'r' in mode, "Don't write to other jobs" return open(self.filename(filename, sliceno), mode, encoding=encoding, errors=errors)
def open(self, filename, mode='r', sliceno=None, encoding=None, errors=None): assert 'r' in mode, "Don't write to other jobs" if 'b' not in mode and encoding is None: encoding = 'utf-8' return open(self.filename(filename, sliceno), mode, encoding=encoding, errors=errors)
def __exit__(self, type, value, tb): # We don't care if an exception occured, we still want to save # the report. # But if saving the report produces an exception we want to # ignore that and re-raise the original exception (or raise # our own exception if no original exception exists). try: if tb is None: self.line() with open('report.txt', 'w', encoding='utf-8') as F: F.write(uni(self.s)) if self.stdout: print(self.s) except Exception: # This logic looks backwards, but it isn't if tb is None: raise finally: self._closed = True
def output(self, what=None): if isinstance(what, int): fns = [str(what)] else: assert what in (None, 'prepare', 'analysis', 'synthesis'), 'Unknown output %r' % (what, ) if what in (None, 'analysis'): fns = [str(sliceno) for sliceno in range(self.params.slices)] if what is None: fns = ['prepare'] + fns + ['synthesis'] else: fns = [what] res = [] for fn in fns: fn = self.filename('OUTPUT/' + fn) if os.path.exists(fn): with open(fn, 'rt', encoding='utf-8', errors='backslashreplace') as fh: res.append(fh.read()) return ''.join(res)
def parse_user_config(): from accelerator.compat import open from os import environ cfgdir = environ.get('XDG_CONFIG_HOME') if not cfgdir: home = environ.get('HOME') if not home: return None cfgdir = join(home, '.config') fn = join(cfgdir, 'accelerator', 'config') try: fh = open(fn, 'r', encoding='utf-8') except IOError: return None with fh: from configparser import ConfigParser cfg = ConfigParser() cfg.optionxform = str # case sensitive (don't downcase aliases) cfg.read_file(fh) if 'alias' in cfg: return cfg['alias'] return None
def _open(fn, _mode): # ignore the passed mode, use the one we have return open(fn, mode, encoding=encoding, errors=errors)
def load_config(filename): from accelerator.shell import UserError key = None multivalued = {'workdirs', 'method packages', 'interpreters'} required = {'slices', 'logfile', 'workdirs', 'method packages'} known = { 'target workdir', 'listen', 'urd', 'result directory', 'source directory', 'project directory' } | required | multivalued cfg = {key: [] for key in multivalued} cfg['listen'] = '.socket.dir/daemon', None class _E(Exception): pass def parse_pair(thing, val): a = val.split() if len(a) != 2 or not a[1].startswith('/'): raise _E("Invalid %s specification %r (expected 'name /path')" % ( thing, val, )) return a def check_interpreter(val): if val[0] == 'DEFAULT': raise _E("Don't override DEFAULT interpreter") if not os.path.isfile(val[1]): raise _E('%r does not exist' % (val, )) parsers = dict( slices=int, workdirs=partial(parse_pair, 'workdir'), interpreters=partial(parse_pair, 'interpreter'), listen=resolve_listen, urd=resolve_listen, ) checkers = dict(interpreter=check_interpreter, ) try: with open(filename, 'r', encoding='utf-8') as fh: for lineno, line in enumerate(fh, 1): line = line.split('#', 1)[0].rstrip() if not line.strip(): continue if line == line.strip(): if ':' not in line: raise _E('Expected a ":"') key, val = line.split(':', 1) if key not in known: raise _E('Unknown key %r' % (key, )) else: if not key: raise _E('First line indented') val = line val = interpolate(val).strip() if val: if key in parsers: val = parsers[key](val) if key in checkers: checkers[key](val) if key in multivalued: cfg[key].append(val) else: if key in cfg: raise _E("%r doesn't take multiple values" % (key, )) cfg[key] = val lineno = None missing = set() for req in required: if not cfg[req]: missing.add(req) if missing: raise _E('Missing required keys %r' % (missing, )) # Reformat result a bit so the new format doesn't require code changes all over the place. rename = { 'target workdir': 'target_workdir', 'method packages': 'method_directories', 'source directory': 'source_directory', 'result directory': 'result_directory', 'project directory': 'project_directory', } res = DotDict({rename.get(k, k): v for k, v in cfg.items()}) if 'target_workdir' not in res: res.target_workdir = res.workdirs[0][0] if 'project_directory' not in res: res.project_directory = os.path.dirname(filename) res.workdirs = dict(res.workdirs) if res.target_workdir not in res.workdirs: raise _E('target workdir %r not in defined workdirs %r' % ( res.target_workdir, set(res.workdirs), )) res.interpreters = dict(res.interpreters) res.listen, res.url = fixup_listen(res.project_directory, res.listen) if res.get('urd'): res.urd_listen, res.urd = fixup_listen(res.project_directory, res.urd, True) else: res.urd_listen, res.urd = None, None except _E as e: if lineno is None: prefix = 'Error in %s:\n' % (filename, ) else: prefix = 'Error on line %d of %s:\n' % ( lineno, filename, ) raise UserError(prefix + e.args[0]) return res
def load_config(filename): from accelerator.error import UserError multivalued = {'workdirs', 'method packages', 'interpreters'} required = {'slices', 'workdirs', 'method packages'} known = { 'target workdir', 'listen', 'urd', 'board listen', 'result directory', 'input directory', 'project directory' } | required | multivalued cfg = {key: [] for key in multivalued} def fixpath(fn, realpath=True): # convert relative path to absolute wrt location of config file p = os.path.join(project_directory, fn) if realpath: p = os.path.realpath(p) else: p = os.path.normpath(p) return p class _E(Exception): pass def parse_workdir(val): return val[0], fixpath(val[1]) def parse_interpreter(val): return val[0], fixpath(val[1], False) def check_interpreter(val): if val[0] == 'DEFAULT': raise _E("Don't override DEFAULT interpreter") if not os.path.isfile(val[1]): raise _E('%r does not exist' % (val, )) def check_workdirs(val): name, path = val if name in (v[0] for v in cfg['workdirs']): raise _E('Workdir %s redefined' % (name, )) if path in (v[1] for v in cfg['workdirs']): raise _E('Workdir path %r re-used' % (path, )) parsers = { 'slices': (['count'], int), 'workdirs': (['name', 'path'], parse_workdir), 'interpreters': (['name', 'path'], parse_interpreter), 'listen': (['path or [host]:port'], resolve_listen), 'urd': (['path or [host]:port'], resolve_listen), 'board listen': (['path or [host]:port'], resolve_listen), 'input directory': (['path'], fixpath), 'result directory': (['path'], fixpath), } checkers = dict( interpreter=check_interpreter, workdirs=check_workdirs, ) with open(filename, 'r', encoding='utf-8') as fh: lines = list(enumerate(fh, 1)) def parse(handle): key = None for n, line in lines: lineno[0] = n line_stripped = line.strip() if not line_stripped or line_stripped[0] == '#': continue if line == line.lstrip(): if ':' not in line: raise _E('Expected a ":"') key, val = line.split(':', 1) if key not in known: raise _E('Unknown key %r' % (key, )) else: if not key: raise _E('First line indented') val = line val = shlex.split(interpolate(val), posix=True, comments=True) if val: handle(key, val) def just_project_directory(key, val): if key == 'project directory': if len(val) != 1: raise _E( "%s takes a single value path (maybe you meant to quote it?)" % (key, )) project_directory[0] = val[0] def everything(key, val): if key in parsers: args, p = parsers[key] if len(val) != len(args): if len(args) == 1: raise _E( "%s takes a single value %s (maybe you meant to quote it?)" % (key, args[0])) else: raise _E("%s takes %d values (expected %s, got %r)" % (key, len(args), ' '.join(args), val)) if len(args) == 1: val = val[0] val = p(val) elif len(val) == 1: val = val[0] else: raise _E("%s takes a single value (maybe you meant to quote it?)" % (key, )) if key in checkers: checkers[key](val) if key in multivalued: cfg[key].append(val) else: if key in cfg: raise _E("%r doesn't take multiple values" % (key, )) cfg[key] = val try: project_directory = [os.path.dirname(filename)] lineno = [None] parse(just_project_directory) lineno = [None] project_directory = os.path.realpath(project_directory[0]) parse(everything) lineno = [None] missing = set() for req in required: if not cfg[req]: missing.add(req) if missing: raise _E('Missing required keys %r' % (missing, )) # Reformat result a bit so the new format doesn't require code changes all over the place. rename = { 'target workdir': 'target_workdir', 'method packages': 'method_directories', 'input directory': 'input_directory', 'result directory': 'result_directory', 'project directory': 'project_directory', 'board listen': 'board_listen', } res = DotDict({rename.get(k, k): v for k, v in cfg.items()}) if 'listen' not in res: res.listen = '.socket.dir/server', None if 'target_workdir' not in res: res.target_workdir = res.workdirs[0][0] if 'project_directory' not in res: res.project_directory = os.path.dirname(filename) res.project_directory = os.path.realpath(res.project_directory) res.workdirs = dict(res.workdirs) if res.target_workdir not in res.workdirs: raise _E('target workdir %r not in defined workdirs %r' % ( res.target_workdir, set(res.workdirs), )) res.interpreters = dict(res.interpreters) res.listen, res.url = fixup_listen(res.project_directory, res.listen) if res.get('urd'): res.urd_listen, res.urd = fixup_listen(res.project_directory, res.urd, True) else: res.urd_listen, res.urd = None, None res.board_listen, _ = fixup_listen( res.project_directory, res.get('board_listen', ('.socket.dir/board', None))) except _E as e: if lineno[0] is None: prefix = 'Error in %s:\n' % (filename, ) else: prefix = 'Error on line %d of %s:\n' % ( lineno[0], filename, ) raise UserError(prefix + e.args[0]) return res
def execute_process(workdir, jobid, slices, result_directory, common_directory, source_directory, index=None, workdirs=None, daemon_url=None, subjob_cookie=None, parent_pid=0): WORKDIRS.update(workdirs) g.job = jobid setproctitle('launch') path = os.path.join(workdir, jobid) try: os.chdir(path) except Exception: print("Cannot cd to workdir", path) exit(1) g.params = params = job_params() method_ref = import_module(params.package+'.a_'+params.method) g.sliceno = -1 g.job = CurrentJob(jobid, params, result_directory, source_directory) g.slices = slices g.options = params.options g.datasets = params.datasets g.jobids = params.jobids method_ref.options = params.options method_ref.datasets= params.datasets method_ref.jobids = params.jobids g.daemon_url = daemon_url g.running = 'launch' status._start('%s %s' % (jobid, params.method,), parent_pid) def dummy(): pass prepare_func = getattr(method_ref, 'prepare' , dummy) analysis_func = getattr(method_ref, 'analysis' , dummy) synthesis_func = getattr(method_ref, 'synthesis', dummy) synthesis_needs_analysis = 'analysis_res' in getarglist(synthesis_func) fd2pid, names, masters, slaves = iowrapper.setup(slices, prepare_func is not dummy, analysis_func is not dummy) def switch_output(): fd = slaves.pop() os.dup2(fd, 1) os.dup2(fd, 2) os.close(fd) iowrapper.run_reader(fd2pid, names, masters, slaves) for fd in masters: os.close(fd) # A chain must be finished from the back, so sort on that. sortnum_cache = {} def dw_sortnum(name): if name not in sortnum_cache: dw = dataset._datasetwriters[name] if dw.previous and dw.previous.startswith(jobid + '/'): pname = dw.previous.split('/')[1] num = dw_sortnum(pname) + 1 else: num = 0 sortnum_cache[name] = num return sortnum_cache[name] prof = {} if prepare_func is dummy: prof['prepare'] = 0 # truthish! else: t = time() switch_output() g.running = 'prepare' g.subjob_cookie = subjob_cookie setproctitle(g.running) with status.status(g.running): g.prepare_res = method_ref.prepare(**args_for(method_ref.prepare)) to_finish = [dw.name for dw in dataset._datasetwriters.values() if dw._started] if to_finish: with status.status("Finishing datasets"): for name in sorted(to_finish, key=dw_sortnum): dataset._datasetwriters[name].finish() c_fflush() prof['prepare'] = time() - t switch_output() setproctitle('launch') from accelerator.extras import saved_files if analysis_func is dummy: prof['per_slice'] = [] prof['analysis'] = 0 else: t = time() g.running = 'analysis' g.subjob_cookie = None # subjobs are not allowed from analysis with status.status('Waiting for all slices to finish analysis') as update: g.update_top_status = update prof['per_slice'], files, g.analysis_res = fork_analysis(slices, analysis_func, args_for(analysis_func), synthesis_needs_analysis, slaves) del g.update_top_status prof['analysis'] = time() - t saved_files.update(files) t = time() g.running = 'synthesis' g.subjob_cookie = subjob_cookie setproctitle(g.running) with status.status(g.running): synthesis_res = synthesis_func(**args_for(synthesis_func)) if synthesis_res is not None: blob.save(synthesis_res, temp=False) if dataset._datasetwriters: with status.status("Finishing datasets"): for name in sorted(dataset._datasetwriters, key=dw_sortnum): dataset._datasetwriters[name].finish() if dataset._datasets_written: with open('datasets.txt', 'w', encoding='utf-8') as fh: for name in dataset._datasets_written: fh.write(name) fh.write(u'\n') c_fflush() t = time() - t prof['synthesis'] = t from accelerator.subjobs import _record return None, (prof, saved_files, _record)