def generate_schema(self):
with pkio.open_text(self.setup_dir.join('setup_vars')) as f:
self.var_names = self.__parse_vars(f)
with pkio.open_text(self.setup_dir.join('setup_params')) as f:
self.models, self.views = self.__parse_setup(f)
with pkio.open_text(self.setup_dir.join('setup_datafiles')) as f:
self.datafiles = self.__parse_datafiles(f)
return self.__format_schema()
def parse_tfs_file(tfs_file, header_only=False, want_page=-1):
mode = 'header'
col_names = []
rows = []
current_page = -1
with pkio.open_text(tfs_file) as f:
for line in f:
if mode == 'header':
# header row starts with *
if re.search(r'^\*\s', line):
col_names = re.split(r'\s+', line.strip())
col_names = col_names[1:]
mode = 'data'
if header_only:
return [x.lower() for x in col_names]
elif mode == 'data':
# data rows after header, start with blank
if re.search(r'^\s+\S', line) and want_page == current_page:
data = re.split(r'\s+', line.strip())
rows.append(data)
elif want_page >= 0 and re.search(r'^\#segment\s', line):
current_page += 1
res = PKDict(map(lambda x: (x.lower(), []), col_names))
for i in range(len(col_names)):
name = col_names[i].lower()
if name:
for row in rows:
res[name].append(row[i])
return res
def tosca_info(tosca):
# determine the list of available files (from zip if necessary)
# compute the tosca length from datafile
#TODO(pjm): keep a cache on the tosca model?
n = _SIM_DATA.lib_file_name_with_model_field('TOSCA', 'magnetFile',
tosca['magnetFile'])
if not _SIM_DATA.lib_file_exists(n):
return PKDict(error='missing or invalid file: {}'.format(
tosca['magnetFile']), )
error = None
length = None
datafile = _SIM_DATA.lib_file_abspath(n)
if is_zip_file(n):
with zipfile.ZipFile(str(datafile), 'r') as z:
filenames = []
if 'fileNames' not in tosca or not tosca['fileNames']:
tosca['fileNames'] = []
for info in z.infolist():
filenames.append(info.filename)
if not length and info.filename in tosca['fileNames']:
length, error = _tosca_length(tosca,
z.read(info).splitlines())
if length:
error = None
else:
filenames = [tosca['magnetFile']]
with pkio.open_text(str(datafile)) as f:
length, error = _tosca_length(tosca, f)
if error:
return PKDict(error=error)
return PKDict(toscaInfo=PKDict(
toscaLength=length,
fileList=sorted(filenames) if filenames else None,
magnetFile=tosca['magnetFile'],
), )
def _parse_match_summary(run_dir, filename):
path = run_dir.join(filename)
node_names = ''
res = ''
with pkio.open_text(str(path)) as f:
state = 'search'
for line in f:
if re.search(r'^MATCH SUMMARY', line):
state = 'summary'
elif state == 'summary':
if re.search(r'^END MATCH SUMMARY', line):
state = 'node_names'
else:
res += line
elif state == 'node_names':
# MAD-X formats the outpus incorrectly when piped to a file
# need to look after the END MATCH for node names
#Global constraint: dq1 4 0.00000000E+00 -3.04197881E-12 9.25363506E-24
if len(line) > 28 and re.search(
r'^\w.*?\:',
line) and line[26] == ' ' and line[27] != ' ':
node_names += line
if node_names:
res = re.sub(r'(Node_Name .*?\n\-+\n)', r'\1' + node_names, res)
return res
def sim_frame_beamline3dAnimation(frame_args):
res = PKDict(
title=' ',
points=[],
polys=[],
colors=[],
bounds=_compute_3d_bounds(frame_args.run_dir),
)
state = None
with pkio.open_text(_OPAL_VTK_FILE) as f:
for line in f:
if line == '\n':
continue
if line.startswith('POINTS '):
state = 'points'
continue
if line.startswith('CELLS '):
state = 'polys'
continue
if line.startswith('CELL_TYPES'):
state = None
continue
if line.startswith('COLOR_SCALARS'):
state = 'colors'
continue
if state == 'points' or state == 'colors':
for v in line.split(' '):
res[state].append(float(v))
elif state == 'polys':
for v in line.split(' '):
res[state].append(int(v))
return res
def _flash_run_command_and_parse_log_on_error(
cls,
command,
work_dir,
log_file,
regex,
):
p = pkio.py_path(log_file)
with pkio.open_text(p.ensure(), mode='r+') as l:
try:
subprocess.run(
command,
check=True,
cwd=work_dir,
stderr=l,
stdout=l,
)
except subprocess.CalledProcessError as e:
l.seek(0)
c = l.read()
m = [x.group().strip() for x in re.finditer(
regex,
c,
re.MULTILINE,
)]
if m:
r = '\n'.join(m)
else:
r = c.splitlines()[-1]
raise sirepo.util.UserAlert(
r,
'{}',
e
)
def _read_data_file(path):
col_names = []
rows = []
with pkio.open_text(str(path)) as f:
col_names = []
rows = []
mode = ''
for line in f:
if '---' in line:
if mode == 'header':
mode = 'data'
elif mode == 'data':
break
if not mode:
mode = 'header'
continue
line = re.sub('\0', '', line)
if mode == 'header':
col_names = re.split(r'\s+', line.lower())
elif mode == 'data':
#TODO(pjm): separate overlapped columns. Instead should explicitly set field dimensions
line = re.sub(r'(\d)(\-\d)', r'\1 \2', line)
line = re.sub(r'(\.\d{3})(\d+\.)', r'\1 \2', line)
rows.append(re.split(r'\s+', line))
return col_names, rows
def background_percent_complete(report, run_dir, is_running):
data = simulation_db.read_json(
run_dir.join(template_common.INPUT_BASE_NAME))
res = PKDict(
percentComplete=0,
frameCount=0,
)
if report == 'animation':
line = template_common.read_last_csv_line(
run_dir.join(_SUMMARY_CSV_FILE))
m = re.search(r'^(\d+)', line)
if m and int(m.group(1)) > 0:
res.frameCount = int((int(m.group(1)) + 1) / 2)
res.wavefrontsFrameCount = _counts_for_beamline(
res.frameCount, data.models.beamline)[0]
total_count = _total_frame_count(data)
res.percentComplete = res.frameCount * 100 / total_count
return res
assert report == 'crystalAnimation'
count = 0
path = run_dir.join(_CRYSTAL_CSV_FILE)
if path.exists():
with pkio.open_text(str(path)) as f:
for line in f:
count += 1
# first two lines are axis points
if count > 2:
plot_count = int((count - 2) / 2)
res.frameCount = plot_count
res.percentComplete = plot_count * 100 / (
1 + data.models.crystalSettings.steps /
data.models.crystalSettings.plotInterval)
return res
def _read_data_file(path, mode='title'):
# mode: title -> header -> data
col_names = []
rows = []
with pkio.open_text(str(path)) as f:
for line in f:
if mode == 'title':
if not re.search(r'^\@', line):
mode = 'header'
continue
# work-around odd header/value "! optimp.f" int twiss output
line = re.sub(r'\!\s', '', line)
# remove space from quoted values
line = re.sub(r"'(\S*)\s*'", r"'\1'", line)
if mode == 'header':
# header row starts with '# <letter>'
if re.search(r'^\s*#\s+[a-zA-Z]', line):
col_names = re.split(r'\s+', line)
col_names = [re.sub(r'\W|_', '', x) for x in col_names[1:]]
mode = 'data'
elif mode == 'data':
if re.search(r'^\s*#', line):
continue
row = re.split(r'\s+', re.sub(r'^\s+', '', line))
rows.append(row)
return col_names, rows
def _parse_madx_log(run_dir):
path = run_dir.join(MADX_LOG_FILE)
if not path.exists():
return ''
res = ''
with pkio.open_text(str(path)) as f:
for line in f:
if re.search(r'^\++ (error|warning):', line, re.IGNORECASE):
line = re.sub(r'^\++ ', '', line)
res += line + "\n"
return res
def load_output(filename):
with pkio.open_text(filename) as f:
t = [x.strip().split() for x in f]
i = 0
res = []
while i < len(t):
l = t[i]
i += 1
if '#segment' in l:
p = int(l[3])
res.append(np.asarray(t[i:i + p]).astype(np.float))
i += p
return res
def _compute_3d_bounds(run_dir):
res = []
p = run_dir.join('data/opal_ElementPositions.txt')
with pkio.open_text(p) as f:
for line in f:
m = re.search(r'^".*?"\s+(\S*?)\s+(\S*?)\s+(\S*?)\s*$', line)
if m:
res.append([float(v) for v in (m.group(1), m.group(2), m.group(3))])
res = np.array(res)
bounds = []
for n in range(3):
v = res[:, n]
bounds.append([min(v), max(v)])
return bounds
def _data_cell(path, var_name):
import csv
with pkio.open_text(path) as f:
reader = csv.reader(f)
d = [f'{var_name} = numpy.array([']
for r in reader:
if not _is_data(r):
continue
d.append(f' {[float(col) for col in r]},')
# for legal JSON
if len(d) > 1:
re.sub(r',$', '', d[-1])
d.append('])')
return d
def _parse_silas_log(run_dir):
res = ''
path = run_dir.join(template_common.RUN_LOG)
if not path.exists():
return res
with pkio.open_text(str(path)) as f:
for line in f:
m = re.search(r'^\s*\*+\s+Error:\s+(.*)$', line)
if m:
err = m.group(1)
if re.search('Unable to evaluate function at point', err):
return 'Point evaulated outside of mesh boundary. Consider increasing Mesh Density or Boundary Tolerance.'
res += err + '\n'
if res:
return res
return 'An unknown error occurred'
def srw_is_valid_file(cls, file_type, path):
# special handling for mirror and arbitraryField - scan for first data row and count columns
if file_type not in ('mirror', 'arbitraryField'):
return True
_ARBITRARY_FIELD_COL_COUNT = 3
with pkio.open_text(path) as f:
for line in f:
if re.search(r'^\s*#', line):
continue
c = len(line.split())
if c > 0:
if file_type == 'arbitraryField':
return c == _ARBITRARY_FIELD_COL_COUNT
return c != _ARBITRARY_FIELD_COL_COUNT
return False
def _parse_opal_log(run_dir):
res = ''
p = run_dir.join((OPAL_OUTPUT_FILE))
if not p.exists():
return res
with pkio.open_text(p) as f:
prev_line = ''
for line in f:
if re.search(r'^Error.*?>', line):
line = re.sub(r'^Error.*?>\s*\**\s*', '', line.rstrip())
if re.search(r'1DPROFILE1-DEFAULT', line):
continue
if line and line != prev_line:
res += line + '\n'
prev_line = line
if res:
return res
return 'An unknown error occurred'
def parse_tfs_page_info(tfs_file):
# returns an array of page info: name, turn, s
col_names = parse_tfs_file(tfs_file, header_only=True)
turn_idx = col_names.index('turn')
s_idx = col_names.index('s')
res = []
mode = 'segment'
with pkio.open_text(tfs_file) as f:
for line in f:
if mode == 'segment' and re.search(r'^\#segment\s', line):
name = re.split(r'\s+', line.strip())[-1]
res.append(PKDict(name=name, ))
mode = 'data'
elif mode == 'data' and re.search(r'^\s+\S', line):
data = re.split(r'\s+', line.strip())
res[-1].update(PKDict(
turn=data[turn_idx],
s=data[s_idx],
))
mode = 'segment'
return res
def _tosca_info(data):
# determine the list of available files (from zip if necessary)
# compute the tosca length from datafile
tosca = data['tosca']
#TODO(pjm): keep a cache on the tosca model?
datafile = simulation_db.simulation_lib_dir(SIM_TYPE).join(
template_common.lib_file_name('TOSCA', 'magnetFile',
tosca['magnetFile']))
if not datafile.exists():
return {
'error': 'missing or invalid file: {}'.format(tosca['magnetFile']),
}
error = None
length = None
if _is_zip_file(datafile):
with zipfile.ZipFile(str(datafile), 'r') as z:
filenames = []
if 'fileNames' not in tosca or not tosca['fileNames']:
tosca['fileNames'] = []
for info in z.infolist():
filenames.append(info.filename)
if not length and info.filename in tosca['fileNames']:
length, error = _tosca_length(tosca,
z.read(info).splitlines())
if length:
error = None
else:
filenames = [tosca['magnetFile']]
with pkio.open_text(str(datafile)) as f:
length, error = _tosca_length(tosca, f)
if error:
return {'error': error}
return {
'toscaInfo': {
'toscaLength': length,
'fileList': sorted(filenames) if filenames else None,
'magnetFile': tosca['magnetFile'],
},
}
def _grid_evolution_columns(run_dir):
try:
with pkio.open_text(run_dir.join(_GRID_EVOLUTION_FILE)) as f:
return [x for x in re.split('[ ]{2,}', f.readline().strip())]
except FileNotFoundError:
return []
