def import_file(text): data = simulation_db.default_data(_SIM_TYPE) beamline = [] data['models']['beamlines'] = [ { 'name': 'BL1', 'id': 1, 'items': beamline, }, ] current_id = 2 title, elements = zgoubi_parser.parse_file(text, 1) data['models']['simulation']['name'] = title if title else 'zgoubi' ids_and_elements = [[], []] for el in elements: _validate_model(el) if 'name' in el: if el not in ids_and_elements[1]: current_id += 1 ids_and_elements[0].append(current_id) ids_and_elements[1].append(el) beamline.append(ids_and_elements[0][ids_and_elements[1].index(el)]) else: if el['type'] in data['models']: pkdlog('replacing existing {} model', el['type']) data['models'][el['type']] = el for idx in range(len(ids_and_elements[0])): el = ids_and_elements[1][idx] el['_id'] = ids_and_elements[0][idx] data['models']['elements'].append(el) elegant_common.sort_elements_and_beamlines(data) return data
def import_file(text, data=None): models = elegant_lattice_parser.parse_file(text, max_id(data) if data else 0) name_to_id, default_beamline_id = _create_name_map(models) if 'default_beamline_name' in models and models['default_beamline_name'] in name_to_id: default_beamline_id = name_to_id[models['default_beamline_name']] element_names = {} rpn_cache = {} for el in models['elements']: el['type'] = _validate_type(el, element_names) element_names[el['name'].upper()] = el validate_fields(el, rpn_cache, models['rpnVariables']) for bl in models['beamlines']: bl['items'] = _validate_beamline(bl, name_to_id, element_names) if len(models['elements']) == 0 or len(models['beamlines']) == 0: raise IOError('no beamline elements found in file') if not data: data = simulation_db.default_data(elegant_common.SIM_TYPE) data['models']['elements'] = models['elements'] data['models']['beamlines'] = models['beamlines'] data['models']['rpnVariables'] = models['rpnVariables'] elegant_common.sort_elements_and_beamlines(data) if default_beamline_id: data['models']['simulation']['activeBeamlineId'] = default_beamline_id data['models']['simulation']['visualizationBeamlineId'] = default_beamline_id # used by synergia app to get values for rpn expressions data['models']['rpnCache'] = rpn_cache return data
def import_file(text): commands = elegant_command_parser.parse_file(text) if not commands: raise IOError('no commands found in file') _verify_lattice_name(commands) rpn_variables = PKDict() # iterate commands, validate values and set defaults from schema for cmd in commands: cmd_type = cmd['_type'] if not cmd_type in _TYPES: raise IOError('unknown command: {}'.format(cmd_type)) elegant_lattice_importer.validate_fields(cmd, PKDict()) # convert macro variables into rpnVariables n = lattice.LatticeUtil.model_name_for_data(cmd) for field in cmd: el_schema = _SCHEMA.model[n].get(field) if el_schema and el_schema[1] == 'RPNValue': m = re.search(r'^<(\w+)>$', str(cmd[field])) if m: cmd[field] = m.group(1) rpn_variables[cmd[field]] = _SIM_DATA.model_defaults(n).get(field, 0) data = simulation_db.default_data(SIM_TYPE) #TODO(pjm) javascript needs to set bunch, bunchSource, bunchFile values from commands data.models.commands = commands data.models.rpnVariables = [PKDict(name=k, value=v) for k, v in rpn_variables.items()] return data
def import_file(text): data = simulation_db.default_data(_SIM_TYPE) beamline = [] data['models']['beamlines'] = [ { 'name': 'BL1', 'id': 1, 'items': beamline, }, ] current_id = 2 title, elements = zgoubi_parser.parse_file(text, 1) data['models']['simulation']['name'] = title if title else 'zgoubi' ids_and_elements = [[], []] for el in elements: _validate_model(el) if 'name' in el: if el not in ids_and_elements[1]: current_id += 1 ids_and_elements[0].append(current_id) ids_and_elements[1].append(el) beamline.append(ids_and_elements[0][ids_and_elements[1].index(el)]) else: if el['type'] in data['models']: pkdlog('replacing existing {} model', el['type']) data['models'][el['type']] = el for idx in range(len(ids_and_elements[0])): el = ids_and_elements[1][idx] el['_id'] = ids_and_elements[0][idx] data['models']['elements'].append(el) elegant_common.sort_elements_and_beamlines(data) return data
def api_newSimulation(): new_simulation_data = _parse_data_input() sim_type = new_simulation_data['simulationType'] data = simulation_db.default_data(sim_type) data['models']['simulation']['name'] = new_simulation_data['name'] data['models']['simulation']['folder'] = new_simulation_data['folder'] sirepo.template.import_module(sim_type).new_simulation(data, new_simulation_data) return _save_new_and_reply(sim_type, data)
def import_file(req, tmp_dir=None, **kwargs): data = simulation_db.default_data(req.type) data.models.simulation.pkupdate( {k: v for k, v in req.req_data.items() if k in data.models.simulation} ) data.models.simulation.pkupdate(_parse_input_file_arg_str(req.import_file_arguments)) _prep_new_sim(data) return data
def srw_to_shadow(self, models): res = simulation_db.default_data(_SHADOW.sim_type()) self.__simulation_to_shadow(models, res.models) if res.models.simulation.sourceType == 'geometricSource': self.__beam_to_shadow(models, res.models) self.__beamline_to_shadow(models, res.models) _SHADOW.fixup_old_data(res) return res
def _import_elegant_file(text): elegant_data = elegant_lattice_importer.import_file(text) rpn_cache = elegant_data['models']['rpnCache'] data = simulation_db.default_data(SIM_TYPE) element_ids = {} for el in elegant_data['models']['elements']: if el['type'] not in _ELEGANT_NAME_MAP: if 'l' in el: el['name'] += '_{}'.format(el['type']) el['type'] = 'DRIF' else: continue el['name'] = re.sub(r':', '_', el['name']) name = _ELEGANT_NAME_MAP[el['type']] schema = _SCHEMA['model'][name] m = { '_id': el['_id'], 'type': name, } for f in el: v = el[f] if el['type'] in _ELEGANT_FIELD_MAP and f in _ELEGANT_FIELD_MAP[ el['type']]: f = _ELEGANT_FIELD_MAP[el['type']][f] if f in schema: if v in rpn_cache: v = rpn_cache[v] m[f] = v template_common.update_model_defaults(m, name, _SCHEMA) data['models']['elements'].append(m) element_ids[m['_id']] = True beamline_ids = {} for bl in elegant_data['models']['beamlines']: bl['name'] = re.sub(r':', '_', bl['name']) element_ids[bl['id']] = True element_ids[-bl['id']] = True for bl in elegant_data['models']['beamlines']: items = [] for element_id in bl['items']: if element_id in element_ids: items.append(element_id) data['models']['beamlines'].append({ 'id': bl['id'], 'items': items, 'name': bl['name'], }) elegant_sim = elegant_data['models']['simulation'] if 'activeBeamlineId' in elegant_sim: data['models']['simulation']['activeBeamlineId'] = elegant_sim[ 'activeBeamlineId'] data['models']['simulation']['visualizationBeamlineId'] = elegant_sim[ 'activeBeamlineId'] data['models']['elements'] = sorted(data['models']['elements'], key=lambda el: el['type']) data['models']['elements'] = sorted(data['models']['elements'], key=lambda el: (el['type'], el['name'].lower())) return data
def _import_madx_file(text): import synergia data = simulation_db.default_data(SIM_TYPE) reader = synergia.lattice.MadX_reader() reader.parse(text) lattice = _import_main_beamline(reader, data) _import_elements(lattice, data) _import_bunch(lattice, data) return data
def _import_elegant_file(text): try: from sirepo.template import elegant_lattice_importer except AssertionError: assert False, 'The elegant sirepo application is not configured.' elegant_data = elegant_lattice_importer.import_file(text) rpn_cache = elegant_data.models.rpnCache data = simulation_db.default_data(SIM_TYPE) element_ids = {} for el in elegant_data.models.elements: if el.type not in _ELEGANT_NAME_MAP: if 'l' in el: el.name += '_{}'.format(el.type) el.type = 'DRIF' else: continue el.name = re.sub(r':', '_', el.name) name = _ELEGANT_NAME_MAP[el.type] schema = _SCHEMA.model[name] m = PKDict( _id=el._id, type=name, ) for f in el: v = el[f] if el.type in _ELEGANT_FIELD_MAP and f in _ELEGANT_FIELD_MAP[ el.type]: f = _ELEGANT_FIELD_MAP[el.type][f] if f in schema: if v in rpn_cache: v = rpn_cache[v] m[f] = v _SIM_DATA.update_model_defaults(m, name) data.models.elements.append(m) element_ids[m._id] = True beamline_ids = {} for bl in elegant_data.models.beamlines: bl.name = re.sub(r':', '_', bl.name) element_ids[bl.id] = True element_ids[-bl.id] = True for bl in elegant_data.models.beamlines: items = [] for element_id in bl['items']: if element_id in element_ids: items.append(element_id) data.models.beamlines.append( PKDict( id=bl.id, items=items, name=bl.name, )) elegant_sim = elegant_data.models.simulation if 'activeBeamlineId' in elegant_sim: data.models.simulation.activeBeamlineId = elegant_sim.activeBeamlineId data.models.simulation.visualizationBeamlineId = elegant_sim.activeBeamlineId return data
def _convert(self, data): self.result = simulation_db.default_data(self.to_class.sim_type()) self._copy_beamlines(data) self._copy_elements(data) self._copy_code_variables(data) LatticeUtil( self.result, self.to_class.schema(), ).sort_elements_and_beamlines() return self.result
def _convert(name, data, direction): assert name in _FIELD_MAP if direction == 'from': field_map = _FIELD_MAP[name].from_madx from_class = sirepo.sim_data.get_class('madx') to_class = sirepo.sim_data.get_class(name) drift_type = 'DRIFT' else: assert direction == 'to' field_map = _FIELD_MAP[name].to_madx from_class = sirepo.sim_data.get_class(name) to_class = sirepo.sim_data.get_class('madx') drift_type = _FIELD_MAP[name].from_madx.DRIFT[0] res = simulation_db.default_data(to_class.sim_type()) for bl in data.models.beamlines: res.models.beamlines.append( PKDict( name=bl.name, items=bl['items'], id=bl.id, )) for el in data.models.elements: if el.type not in field_map: #TODO(pjm): convert to a sim appropriate drift rather than skipping pkdlog('Unhandled element type: {}', el.type) el.type = drift_type if 'l' not in el: el.l = 0 fields = field_map[el.type] values = to_class.model_defaults(fields[0]) values.name = el.name values.type = fields[0] values._id = el._id for idx in range(1, len(fields)): values[fields[idx]] = el[fields[idx]] # add any non-default values not in map to a comment comment = '' defaults = from_class.model_defaults(el.type) for f in el: if f not in fields and f in defaults and str(el[f]) != str( defaults[f]): v = el[f] if ' ' in str(v): v = '"{}"'.format(v) comment += '{}={} '.format(f, v) if comment: values._comment = '{}: {} {}'.format(from_class.sim_type(), el.type, comment.strip()) res.models.elements.append(values) res.models.rpnVariables = _rpn_variables(to_class, data) for f in ('name', 'visualizationBeamlineId', 'activeBeamlineId'): if f in data.models.simulation: res.models.simulation[f] = data.models.simulation[f] return res
def import_file(text): data = simulation_db.default_data(_SIM_TYPE) #TODO(pjm): need a common way to clean-up/uniquify a simulation name from imported text title, elements = zgoubi_parser.parse_file(text, 1) title = re.sub(r'\s+', ' ', title) title = re.sub(r'^\s+|\s+$', '', title) data['models']['simulation']['name'] = title if title else 'zgoubi' info = _validate_and_dedup_elements(data, elements) _validate_element_names(data, info) elegant_common.sort_elements_and_beamlines(data) return data
def api_newSimulation(): req = http_request.parse_post(template=True, folder=True, name=True) d = simulation_db.default_data(req.type) #TODO(pjm): update fields from schema values across new_simulation_data values d.models.simulation.pkupdate( name=req.name, folder=req.folder, notes=req.req_data.get('notes', ''), ) if hasattr(req.template, 'new_simulation'): req.template.new_simulation(d, req.req_data) return _save_new_and_reply(d)
def parse_file(self, lattice_text): from sirepo import simulation_db self.data = simulation_db.default_data(self.sim_data.sim_type()) self.parser = LineParser(100) self.data.models.rpnVariables = {} self.data.models.sequences = [] # None | sequence | track | match | edit self.container = None self.elements_by_name = PKDict() lines = lattice_text.replace('\r', '').split('\n') self.__parse_lines(lines) return self.data
def import_file(request, lib_dir=None, tmp_dir=None): f = request.files['file'] filename = werkzeug.secure_filename(f.filename) if not pkio.has_file_extension(str(filename), 'zip'): raise RuntimeError('unsupported import filename: {}'.format(filename)) filepath = str(tmp_dir.join(_ZIP_FILE_NAME)) f.save(filepath) data = simulation_db.default_data(SIM_TYPE) data['models']['simulation']['name'] = filename data['models']['simulation'][_TMP_INPUT_FILE_FIELD] = filepath # more processing occurs below in prepare_for_client() after simulation dir is prepared return data
def import_file(request, lib_dir=None, tmp_dir=None): f = request.files['file'] filename = werkzeug.secure_filename(f.filename) if not pkio.has_file_extension(str(filename), 'zip'): raise RuntimeError('unsupported import filename: {}'.format(filename)) filepath = str(tmp_dir.join(_ZIP_FILE_NAME)) f.save(filepath) data = simulation_db.default_data(SIM_TYPE) data['models']['simulation']['name'] = filename data['models']['simulation'][_TMP_INPUT_FILE_FIELD] = filepath # more processing occurs below in prepare_for_client() after simulation dir is prepared return data
def api_newSimulation(): new_simulation_data = _parse_data_input() sim_type = new_simulation_data['simulationType'] data = simulation_db.default_data(sim_type) data['models']['simulation']['name'] = new_simulation_data['name'] data['models']['simulation']['folder'] = new_simulation_data['folder'] if 'notes' in new_simulation_data: data['models']['simulation']['notes'] = new_simulation_data['notes'] template = sirepo.template.import_module(sim_type) if hasattr(template, 'new_simulation'): template.new_simulation(data, new_simulation_data) return _save_new_and_reply(data)
def api_newSimulation(): req = http_request.parse_post(template=True, folder=True, name=True) d = simulation_db.default_data(req.type) d.models.simulation.pkupdate( {k: v for k, v in req.req_data.items() if k in d.models.simulation}) d.models.simulation.pkupdate( name=req.name, folder=req.folder, ) if hasattr(req.template, 'new_simulation'): req.template.new_simulation(d, req.req_data) return _save_new_and_reply(d)
def _import_elegant_file(text): elegant_data = elegant_lattice_importer.import_file(text) rpn_cache = elegant_data['models']['rpnCache'] data = simulation_db.default_data(SIM_TYPE) element_ids = {} for el in elegant_data['models']['elements']: if el['type'] not in _ELEGANT_NAME_MAP: if 'l' in el: el['name'] += '_{}'.format(el['type']) el['type'] = 'DRIF' else: continue el['name'] = re.sub(r':', '_', el['name']) name = _ELEGANT_NAME_MAP[el['type']] schema = _SCHEMA['model'][name] m = { '_id': el['_id'], 'type': name, } for f in el: v = el[f] if el['type'] in _ELEGANT_FIELD_MAP and f in _ELEGANT_FIELD_MAP[el['type']]: f = _ELEGANT_FIELD_MAP[el['type']][f] if f in schema: if v in rpn_cache: v = rpn_cache[v] m[f] = v template_common.update_model_defaults(m, name, _SCHEMA) data['models']['elements'].append(m) element_ids[m['_id']] = True beamline_ids = {} for bl in elegant_data['models']['beamlines']: bl['name'] = re.sub(r':', '_', bl['name']) element_ids[bl['id']] = True element_ids[-bl['id']] = True for bl in elegant_data['models']['beamlines']: items = [] for element_id in bl['items']: if element_id in element_ids: items.append(element_id) data['models']['beamlines'].append({ 'id': bl['id'], 'items': items, 'name': bl['name'], }) elegant_sim = elegant_data['models']['simulation'] if 'activeBeamlineId' in elegant_sim: data['models']['simulation']['activeBeamlineId'] = elegant_sim['activeBeamlineId'] data['models']['simulation']['visualizationBeamlineId'] = elegant_sim['activeBeamlineId'] return data
def import_file(req, tmp_dir=None, **kwargs): if not pkio.has_file_extension(req.filename, 'zip'): raise sirepo.util.UserAlert('unsupported import filename: {}'.format(filename)) #TODO(pjm): writing to simulation lib for now, tmp_dir will get removed after this request filepath = str(simulation_db.simulation_lib_dir(SIM_TYPE).join(_ZIP_FILE_NAME)) pkio.mkdir_parent_only(filepath) with open(filepath, 'wb') as f: f.write(req.file_stream.read()) data = simulation_db.default_data(SIM_TYPE) data['models']['simulation']['name'] = req.filename data['models']['simulation'][_TMP_INPUT_FILE_FIELD] = filepath # more processing occurs in prepare_for_client() via: # import_file => _save_new_and_reply => api_simulationData => prepare_for_client return data
def import_file(text): commands = elegant_command_parser.parse_file(text) if not len(commands): raise IOError('no commands found in file') _verify_lattice_name(commands) # iterate commands, validate values and set defaults from schema for cmd in commands: cmd_type = cmd['_type'] if not cmd_type in _ELEGANT_TYPES: raise IOError('unknown command: {}'.format(cmd_type)) elegant_lattice_importer.validate_fields(cmd, {}, {}) data = simulation_db.default_data(elegant_common.SIM_TYPE) #TODO(pjm) javascript needs to set bunch, bunchSource, bunchFile values from commands data['models']['commands'] = commands return data
def import_file(text): commands = elegant_command_parser.parse_file(text) if not len(commands): raise IOError('no commands found in file') _verify_lattice_name(commands) # iterate commands, validate values and set defaults from schema for cmd in commands: cmd_type = cmd['_type'] if not cmd_type in _ELEGANT_TYPES: raise IOError('unknown command: {}'.format(cmd_type)) elegant_lattice_importer.validate_fields(cmd, {}, {}) data = simulation_db.default_data(elegant_common.SIM_TYPE) #TODO(pjm) javascript needs to set bunch, bunchSource, bunchFile values from commands data['models']['commands'] = commands return data
def srw_to_shadow(self, models): res = simulation_db.default_data(_SHADOW.sim_type()) self.beamline = res.models.beamline self.__simulation_to_shadow(models, res.models) if res.models.simulation.sourceType == 'geometricSource': self.__beam_to_shadow(models, res.models) elif res.models.simulation.sourceType == 'undulator': self.__undulator_to_shadow(models, res.models) elif res.models.simulation.sourceType == 'bendingMagnet': self.__multipole_to_shadow(models, res.models) self.__beamline_to_shadow(models, res.models) if res.models.simulation.sourceType == 'undulator': self.__fix_undulator_gratings(res.models) _SHADOW.fixup_old_data(res) return res
def __remove_default_commands(self): from sirepo import simulation_db size = len( simulation_db.default_data( self.sim_data.sim_type()).models.commands) res = [] names = set(self.data.models.commands[0].name) for i in range(len(self.data.models.commands)): cmd = self.data.models.commands[i] if i >= size: if not cmd.name: cmd.name = self.__unique_name(cmd, names) res.append(cmd) names.add(cmd.name) self.data.models.commands = res
def import_file(text, unit_test_mode=False): if unit_test_mode: global _UNIT_TEST_MODE _UNIT_TEST_MODE = unit_test_mode data = simulation_db.default_data(SIM_TYPE) #TODO(pjm): need a common way to clean-up/uniquify a simulation name from imported text title, elements, unhandled_elements = zgoubi_parser.parse_file(text, 1) title = re.sub(r'\s+', ' ', title) title = re.sub(r'^\s+|\s+$', '', title) data.models.simulation.name = title if title else 'zgoubi' if unhandled_elements: data.models.simulation.warnings = 'Unsupported Zgoubi elements: {}'.format( ', '.join(unhandled_elements)) info = _validate_and_dedup_elements(data, elements) _validate_element_names(data, info) LatticeUtil(data, _SCHEMA).sort_elements_and_beamlines() if 'missingFiles' in info and info.missingFiles: data.error = 'Missing data files' data.missingFiles = info.missingFiles _update_report_parameters(data) return data
def import_file(text, data=None, update_filenames=True): if not data: data = simulation_db.default_data(elegant_common.SIM_TYPE) models = elegant_lattice_parser.parse_file( text, data.models.rpnVariables, lattice.LatticeUtil.max_id(data), ) name_to_id, default_beamline_id = _create_name_map(models) if 'default_beamline_name' in models and models[ 'default_beamline_name'] in name_to_id: default_beamline_id = name_to_id[models['default_beamline_name']] element_names = PKDict() rpn_cache = PKDict() code_var = elegant_code_var(models.rpnVariables) for el in models['elements']: el['type'] = _validate_type(el, element_names) element_names[el['name'].upper()] = el validate_fields(el, rpn_cache, code_var, update_filenames) for bl in models['beamlines']: bl['items'] = _validate_beamline(bl, name_to_id, element_names) if len(models['elements']) == 0 or len(models['beamlines']) == 0: raise IOError('no beamline elements found in file') data['models']['elements'] = models['elements'] data['models']['beamlines'] = models['beamlines'] data['models']['rpnVariables'] = models['rpnVariables'] lattice.LatticeUtil(data, _SCHEMA).sort_elements_and_beamlines() if default_beamline_id: data['models']['simulation']['activeBeamlineId'] = default_beamline_id data['models']['simulation'][ 'visualizationBeamlineId'] = default_beamline_id # used by synergia app to get values for rpn expressions data['models']['rpnCache'] = rpn_cache return data
def import_file(text, data=None): models = elegant_lattice_parser.parse_file(text, max_id(data) if data else 0) name_to_id, default_beamline_id = _create_name_map(models) if 'default_beamline_name' in models and models[ 'default_beamline_name'] in name_to_id: default_beamline_id = name_to_id[models['default_beamline_name']] element_names = {} rpn_cache = {} for el in models['elements']: el['type'] = _validate_type(el, element_names) element_names[el['name'].upper()] = el validate_fields(el, rpn_cache, models['rpnVariables']) for bl in models['beamlines']: bl['items'] = _validate_beamline(bl, name_to_id, element_names) if len(models['elements']) == 0 or len(models['beamlines']) == 0: raise IOError('no beamline elements found in file') if not data: data = simulation_db.default_data(elegant_common.SIM_TYPE) data['models']['elements'] = sorted(models['elements'], key=lambda el: el['type']) data['models']['elements'] = sorted(models['elements'], key=lambda el: (el['type'], el['name'].lower())) data['models']['beamlines'] = sorted(models['beamlines'], key=lambda b: b['name'].lower()) data['models']['rpnVariables'] = models['rpnVariables'] if default_beamline_id: data['models']['simulation']['activeBeamlineId'] = default_beamline_id data['models']['simulation'][ 'visualizationBeamlineId'] = default_beamline_id # used by synergia app to get values for rpn expressions data['models']['rpnCache'] = rpn_cache return data
def fixup_old_data(cls, data): dm = data.models cls._init_models(dm, ( 'dataSource', 'elegantAnimation', 'epochAnimation', 'files', 'latticeSettings', 'mlModel', 'mlModelGraph', 'neuralNet', 'neuralNetLayer', 'partition', 'partitionSelectionReport', 'rfcSettings', )) if 'beamlines' not in dm: from sirepo import simulation_db defaults = simulation_db.default_data(cls.sim_type()).models for m in ('beamlines', 'commands', 'elements', 'rpnCache', 'rpnVariables'): dm[m] = defaults[m]
def _import_mad_file(reader, beamline_names): data = simulation_db.default_data(SIM_TYPE) lattice = _import_main_beamline(reader, data, beamline_names) _import_elements(lattice, data) _import_bunch(lattice, data) return data