def generate_schema(self): with pkio.open_text(self.setup_dir.join('setup_vars')) as f: self.var_names = self.__parse_vars(f) with pkio.open_text(self.setup_dir.join('setup_params')) as f: self.models, self.views = self.__parse_setup(f) with pkio.open_text(self.setup_dir.join('setup_datafiles')) as f: self.datafiles = self.__parse_datafiles(f) return self.__format_schema()
def parse_tfs_file(tfs_file, header_only=False, want_page=-1): mode = 'header' col_names = [] rows = [] current_page = -1 with pkio.open_text(tfs_file) as f: for line in f: if mode == 'header': # header row starts with * if re.search(r'^\*\s', line): col_names = re.split(r'\s+', line.strip()) col_names = col_names[1:] mode = 'data' if header_only: return [x.lower() for x in col_names] elif mode == 'data': # data rows after header, start with blank if re.search(r'^\s+\S', line) and want_page == current_page: data = re.split(r'\s+', line.strip()) rows.append(data) elif want_page >= 0 and re.search(r'^\#segment\s', line): current_page += 1 res = PKDict(map(lambda x: (x.lower(), []), col_names)) for i in range(len(col_names)): name = col_names[i].lower() if name: for row in rows: res[name].append(row[i]) return res
def tosca_info(tosca): # determine the list of available files (from zip if necessary) # compute the tosca length from datafile #TODO(pjm): keep a cache on the tosca model? n = _SIM_DATA.lib_file_name_with_model_field('TOSCA', 'magnetFile', tosca['magnetFile']) if not _SIM_DATA.lib_file_exists(n): return PKDict(error='missing or invalid file: {}'.format( tosca['magnetFile']), ) error = None length = None datafile = _SIM_DATA.lib_file_abspath(n) if is_zip_file(n): with zipfile.ZipFile(str(datafile), 'r') as z: filenames = [] if 'fileNames' not in tosca or not tosca['fileNames']: tosca['fileNames'] = [] for info in z.infolist(): filenames.append(info.filename) if not length and info.filename in tosca['fileNames']: length, error = _tosca_length(tosca, z.read(info).splitlines()) if length: error = None else: filenames = [tosca['magnetFile']] with pkio.open_text(str(datafile)) as f: length, error = _tosca_length(tosca, f) if error: return PKDict(error=error) return PKDict(toscaInfo=PKDict( toscaLength=length, fileList=sorted(filenames) if filenames else None, magnetFile=tosca['magnetFile'], ), )
def _parse_match_summary(run_dir, filename): path = run_dir.join(filename) node_names = '' res = '' with pkio.open_text(str(path)) as f: state = 'search' for line in f: if re.search(r'^MATCH SUMMARY', line): state = 'summary' elif state == 'summary': if re.search(r'^END MATCH SUMMARY', line): state = 'node_names' else: res += line elif state == 'node_names': # MAD-X formats the outpus incorrectly when piped to a file # need to look after the END MATCH for node names #Global constraint: dq1 4 0.00000000E+00 -3.04197881E-12 9.25363506E-24 if len(line) > 28 and re.search( r'^\w.*?\:', line) and line[26] == ' ' and line[27] != ' ': node_names += line if node_names: res = re.sub(r'(Node_Name .*?\n\-+\n)', r'\1' + node_names, res) return res
def sim_frame_beamline3dAnimation(frame_args): res = PKDict( title=' ', points=[], polys=[], colors=[], bounds=_compute_3d_bounds(frame_args.run_dir), ) state = None with pkio.open_text(_OPAL_VTK_FILE) as f: for line in f: if line == '\n': continue if line.startswith('POINTS '): state = 'points' continue if line.startswith('CELLS '): state = 'polys' continue if line.startswith('CELL_TYPES'): state = None continue if line.startswith('COLOR_SCALARS'): state = 'colors' continue if state == 'points' or state == 'colors': for v in line.split(' '): res[state].append(float(v)) elif state == 'polys': for v in line.split(' '): res[state].append(int(v)) return res
def _flash_run_command_and_parse_log_on_error( cls, command, work_dir, log_file, regex, ): p = pkio.py_path(log_file) with pkio.open_text(p.ensure(), mode='r+') as l: try: subprocess.run( command, check=True, cwd=work_dir, stderr=l, stdout=l, ) except subprocess.CalledProcessError as e: l.seek(0) c = l.read() m = [x.group().strip() for x in re.finditer( regex, c, re.MULTILINE, )] if m: r = '\n'.join(m) else: r = c.splitlines()[-1] raise sirepo.util.UserAlert( r, '{}', e )
def _read_data_file(path): col_names = [] rows = [] with pkio.open_text(str(path)) as f: col_names = [] rows = [] mode = '' for line in f: if '---' in line: if mode == 'header': mode = 'data' elif mode == 'data': break if not mode: mode = 'header' continue line = re.sub('\0', '', line) if mode == 'header': col_names = re.split(r'\s+', line.lower()) elif mode == 'data': #TODO(pjm): separate overlapped columns. Instead should explicitly set field dimensions line = re.sub(r'(\d)(\-\d)', r'\1 \2', line) line = re.sub(r'(\.\d{3})(\d+\.)', r'\1 \2', line) rows.append(re.split(r'\s+', line)) return col_names, rows
def background_percent_complete(report, run_dir, is_running): data = simulation_db.read_json( run_dir.join(template_common.INPUT_BASE_NAME)) res = PKDict( percentComplete=0, frameCount=0, ) if report == 'animation': line = template_common.read_last_csv_line( run_dir.join(_SUMMARY_CSV_FILE)) m = re.search(r'^(\d+)', line) if m and int(m.group(1)) > 0: res.frameCount = int((int(m.group(1)) + 1) / 2) res.wavefrontsFrameCount = _counts_for_beamline( res.frameCount, data.models.beamline)[0] total_count = _total_frame_count(data) res.percentComplete = res.frameCount * 100 / total_count return res assert report == 'crystalAnimation' count = 0 path = run_dir.join(_CRYSTAL_CSV_FILE) if path.exists(): with pkio.open_text(str(path)) as f: for line in f: count += 1 # first two lines are axis points if count > 2: plot_count = int((count - 2) / 2) res.frameCount = plot_count res.percentComplete = plot_count * 100 / ( 1 + data.models.crystalSettings.steps / data.models.crystalSettings.plotInterval) return res
def _read_data_file(path, mode='title'): # mode: title -> header -> data col_names = [] rows = [] with pkio.open_text(str(path)) as f: for line in f: if mode == 'title': if not re.search(r'^\@', line): mode = 'header' continue # work-around odd header/value "! optimp.f" int twiss output line = re.sub(r'\!\s', '', line) # remove space from quoted values line = re.sub(r"'(\S*)\s*'", r"'\1'", line) if mode == 'header': # header row starts with '# <letter>' if re.search(r'^\s*#\s+[a-zA-Z]', line): col_names = re.split(r'\s+', line) col_names = [re.sub(r'\W|_', '', x) for x in col_names[1:]] mode = 'data' elif mode == 'data': if re.search(r'^\s*#', line): continue row = re.split(r'\s+', re.sub(r'^\s+', '', line)) rows.append(row) return col_names, rows
def _parse_madx_log(run_dir): path = run_dir.join(MADX_LOG_FILE) if not path.exists(): return '' res = '' with pkio.open_text(str(path)) as f: for line in f: if re.search(r'^\++ (error|warning):', line, re.IGNORECASE): line = re.sub(r'^\++ ', '', line) res += line + "\n" return res
def load_output(filename): with pkio.open_text(filename) as f: t = [x.strip().split() for x in f] i = 0 res = [] while i < len(t): l = t[i] i += 1 if '#segment' in l: p = int(l[3]) res.append(np.asarray(t[i:i + p]).astype(np.float)) i += p return res
def _compute_3d_bounds(run_dir): res = [] p = run_dir.join('data/opal_ElementPositions.txt') with pkio.open_text(p) as f: for line in f: m = re.search(r'^".*?"\s+(\S*?)\s+(\S*?)\s+(\S*?)\s*$', line) if m: res.append([float(v) for v in (m.group(1), m.group(2), m.group(3))]) res = np.array(res) bounds = [] for n in range(3): v = res[:, n] bounds.append([min(v), max(v)]) return bounds
def _data_cell(path, var_name): import csv with pkio.open_text(path) as f: reader = csv.reader(f) d = [f'{var_name} = numpy.array(['] for r in reader: if not _is_data(r): continue d.append(f' {[float(col) for col in r]},') # for legal JSON if len(d) > 1: re.sub(r',$', '', d[-1]) d.append('])') return d
def _parse_silas_log(run_dir): res = '' path = run_dir.join(template_common.RUN_LOG) if not path.exists(): return res with pkio.open_text(str(path)) as f: for line in f: m = re.search(r'^\s*\*+\s+Error:\s+(.*)$', line) if m: err = m.group(1) if re.search('Unable to evaluate function at point', err): return 'Point evaulated outside of mesh boundary. Consider increasing Mesh Density or Boundary Tolerance.' res += err + '\n' if res: return res return 'An unknown error occurred'
def srw_is_valid_file(cls, file_type, path): # special handling for mirror and arbitraryField - scan for first data row and count columns if file_type not in ('mirror', 'arbitraryField'): return True _ARBITRARY_FIELD_COL_COUNT = 3 with pkio.open_text(path) as f: for line in f: if re.search(r'^\s*#', line): continue c = len(line.split()) if c > 0: if file_type == 'arbitraryField': return c == _ARBITRARY_FIELD_COL_COUNT return c != _ARBITRARY_FIELD_COL_COUNT return False
def _parse_opal_log(run_dir): res = '' p = run_dir.join((OPAL_OUTPUT_FILE)) if not p.exists(): return res with pkio.open_text(p) as f: prev_line = '' for line in f: if re.search(r'^Error.*?>', line): line = re.sub(r'^Error.*?>\s*\**\s*', '', line.rstrip()) if re.search(r'1DPROFILE1-DEFAULT', line): continue if line and line != prev_line: res += line + '\n' prev_line = line if res: return res return 'An unknown error occurred'
def parse_tfs_page_info(tfs_file): # returns an array of page info: name, turn, s col_names = parse_tfs_file(tfs_file, header_only=True) turn_idx = col_names.index('turn') s_idx = col_names.index('s') res = [] mode = 'segment' with pkio.open_text(tfs_file) as f: for line in f: if mode == 'segment' and re.search(r'^\#segment\s', line): name = re.split(r'\s+', line.strip())[-1] res.append(PKDict(name=name, )) mode = 'data' elif mode == 'data' and re.search(r'^\s+\S', line): data = re.split(r'\s+', line.strip()) res[-1].update(PKDict( turn=data[turn_idx], s=data[s_idx], )) mode = 'segment' return res
def _tosca_info(data): # determine the list of available files (from zip if necessary) # compute the tosca length from datafile tosca = data['tosca'] #TODO(pjm): keep a cache on the tosca model? datafile = simulation_db.simulation_lib_dir(SIM_TYPE).join( template_common.lib_file_name('TOSCA', 'magnetFile', tosca['magnetFile'])) if not datafile.exists(): return { 'error': 'missing or invalid file: {}'.format(tosca['magnetFile']), } error = None length = None if _is_zip_file(datafile): with zipfile.ZipFile(str(datafile), 'r') as z: filenames = [] if 'fileNames' not in tosca or not tosca['fileNames']: tosca['fileNames'] = [] for info in z.infolist(): filenames.append(info.filename) if not length and info.filename in tosca['fileNames']: length, error = _tosca_length(tosca, z.read(info).splitlines()) if length: error = None else: filenames = [tosca['magnetFile']] with pkio.open_text(str(datafile)) as f: length, error = _tosca_length(tosca, f) if error: return {'error': error} return { 'toscaInfo': { 'toscaLength': length, 'fileList': sorted(filenames) if filenames else None, 'magnetFile': tosca['magnetFile'], }, }
def _grid_evolution_columns(run_dir): try: with pkio.open_text(run_dir.join(_GRID_EVOLUTION_FILE)) as f: return [x for x in re.split('[ ]{2,}', f.readline().strip())] except FileNotFoundError: return []