def _validate_number(val, sch_field_info): """Ensure the value of a numeric field falls within the supplied limits (if any) Args: val: numeric value to validate sch_field_info ([str]): field info array from schema """ if len(sch_field_info) <= 4: return try: fv = float(val) fmin = float(sch_field_info[4]) # Currently the values in enum arrays at the indices below are sometimes # used for other purposes, so we return rather than fail for non-numeric values except ValueError: return if fv < fmin: raise AssertionError( util.err(sch_field_info, 'numeric value {} out of range', val)) if len(sch_field_info) > 5: try: fmax = float(sch_field_info[5]) except ValueError: return if fv > fmax: raise AssertionError( util.err(sch_field_info, 'numeric value {} out of range', val))
def _validate_number(val, sch_field_info): """Ensure the value of a numeric field falls within the supplied limits (if any) Args: val: numeric value to validate sch_field_info ([str]): field info array from schema """ if len(sch_field_info) <= 4: return try: fv = float(val) fmin = float(sch_field_info[4]) # Currently the values in enum arrays at the indices below are sometimes # used for other purposes, so we return rather than fail for non-numeric values except ValueError: return if fv < fmin: raise AssertionError(util.err(sch_field_info, 'numeric value {} out of range', val)) if len(sch_field_info) > 5: try: fmax = float(sch_field_info[5]) except ValueError: return if fv > fmax: raise AssertionError(util.err(sch_field_info, 'numeric value {} out of range', val))
def _fit_to_equation(x, y, equation, var, params): # TODO: must sanitize input - sympy uses eval # These security measures taken so far: # Whitelist of allowed functions and other symbols as defined in the schema # Variable and parameters must be 1 alphabetic character eq_ops = [ t for t in _tokenize_equation(equation) if t != var and t not in params ] eq_ops_rejected = [ op for op in eq_ops if op not in _SCHEMA.constants.allowedEquationOps ] assert len(eq_ops_rejected) == 0, util.err(eq_ops_rejected, 'operation fobidden') assert _validate_eq_var(var), util.err(var, 'invalid variable name') assert all([_validate_eq_var(p) for p in re.split(r'\s*,\s*', params)]),\ util.err(params, 'invalid parameter name(s)') sym_curve = sympy.sympify(equation) sym_str = var + ' ' + ' '.join(params) syms = sympy.symbols(sym_str) sym_curve_l = sympy.lambdify(syms, sym_curve, 'numpy') p_vals, pcov = scipy.optimize.curve_fit(sym_curve_l, x, y, maxfev=500000) sigma = np.sqrt(np.diagonal(pcov)) p_subs = [] p_subs_min = [] p_subs_max = [] # exclude the symbol of the variable when subbing for sidx, p in enumerate(p_vals, 1): sig = sigma[sidx - 1] p_min = p - 2 * sig p_max = p + 2 * sig s = syms[sidx] p_subs.append((s, p)) p_subs_min.append((s, p_min)) p_subs_max.append((s, p_max)) y_fit = sym_curve.subs(p_subs) y_fit_min = sym_curve.subs(p_subs_min) y_fit_max = sym_curve.subs(p_subs_max) # used for the laTeX label - rounding should take size of uncertainty into account y_fit_rounded = sym_curve.subs(p_subs) y_fit_l = sympy.lambdify(var, y_fit, 'numpy') y_fit_min_l = sympy.lambdify(var, y_fit_min, 'numpy') y_fit_max_l = sympy.lambdify(var, y_fit_max, 'numpy') latex_label = sympy.latex(y_fit_rounded, mode='inline') #TODO(pjm): round rather than truncate? latex_label = re.sub(r'(\.\d{4})\d+', r'\1', latex_label) x_uniform = np.linspace(np.min(x), np.max(x), 100) return x_uniform, y_fit_l(x_uniform), y_fit_min_l(x_uniform), y_fit_max_l( x_uniform), p_vals, sigma, latex_label
def validate(schema): """Validate the schema Validations performed: Values of default data (if any) Existence of dynamic modules Enums keyed by string value Model names containing special characters Method name for API calls with them are valid python function names and not too long Args: schema (PKDict): app schema """ sch_models = schema.model sch_enums = schema.enum sch_ntfy = schema.notifications sch_cookies = schema.cookies for name in sch_enums: for values in sch_enums[name]: if not isinstance(values[0], pkconfig.STRING_TYPES): raise AssertionError( util.err( name, 'enum values must be keyed by a string value: {}', type(values[0]))) for model_name in sch_models: _validate_model_name(model_name) sch_model = sch_models[model_name] for field_name in sch_model: sch_field_info = sch_model[field_name] if len(sch_field_info) <= 2: continue field_default = sch_field_info[2] if field_default == '' or field_default is None: continue _validate_enum(field_default, sch_field_info, sch_enums) _validate_number(field_default, sch_field_info) for n in sch_ntfy: if 'cookie' not in sch_ntfy[n] or sch_ntfy[n].cookie not in sch_cookies: raise AssertionError( util.err(sch_ntfy[n], 'notification must reference a cookie in the schema')) for sc in sch_cookies: _validate_cookie_def(sch_cookies[sc]) for t in schema.dynamicModules: for src in schema.dynamicModules[t]: pkresource.filename(src[1:]) _validate_strings(schema.strings)
def _validate_enum(val, sch_field_info, sch_enums): type = sch_field_info[1] if not type in sch_enums: return if str(val) not in map(lambda enum: str(enum[0]), sch_enums[type]): raise AssertionError( util.err(sch_enums, 'enum value {} not in schema', val))
def api_findByName(simulation_type, application_mode, simulation_name): if cfg.oauth_login: from sirepo import oauth oauth.set_default_state(logged_out_as_anonymous=True) # use the existing named simulation, or copy it from the examples rows = simulation_db.iterate_simulation_datafiles(simulation_type, simulation_db.process_simulation_list, { 'simulation.name': simulation_name, 'simulation.isExample': True, }) if len(rows) == 0: for s in simulation_db.examples(simulation_type): if s['models']['simulation']['name'] == simulation_name: simulation_db.save_new_example(s) rows = simulation_db.iterate_simulation_datafiles(simulation_type, simulation_db.process_simulation_list, { 'simulation.name': simulation_name, }) break else: raise AssertionError(util.err(simulation_name, 'simulation not found with type {}', simulation_type)) return javascript_redirect( uri_router.format_uri( simulation_type, application_mode, rows[0]['simulationId'], simulation_db.get_schema(simulation_type) ) )
def _unnest_subclasses(schema, item, key, subclass_keys): item_schema = schema[item] try: if _SCHEMA_SUPERCLASS_FIELD not in item_schema[key]: return except TypeError: # Ignore non-indexable types return sub_model = item_schema[key] sub_item = sub_model[_SCHEMA_SUPERCLASS_FIELD][1] sub_key = sub_model[_SCHEMA_SUPERCLASS_FIELD][2] assert sub_item in schema, util.err(sub_item, 'No such field in schema') assert sub_item == item, util.err( sub_item, 'Superclass must be in same section of schema {}', item) assert sub_key in item_schema, util.err(sub_key, 'No such superclass') subclass_keys.append(sub_key) _unnest_subclasses(schema, item, sub_key, subclass_keys)
def _merge_subclasses(schema, item): for m in schema[item]: has_super = False s = schema[item][m] try: has_super = _SCHEMA_SUPERCLASS_FIELD in s except TypeError: # Ignore non-indexable types continue if has_super: i = s[_SCHEMA_SUPERCLASS_FIELD] s_item = i[1] s_class = i[2] assert s_item in schema, util.err(s_item, 'No such field in schema') assert s_item == item, util.err(s_item, 'Superclass must be in same section of schema {}', item) assert s_class in schema[s_item], util.err(s_class, 'No such superclass') _merge_dicts(schema[item][s_class], s)
def _validate_model_name(model_name): """Ensure model name contain no special characters Args: model_name (str): name to validate """ if not re.search(r'^[a-z_]\w*$', model_name, re.IGNORECASE): raise AssertionError( util.err(model_name, 'model name must be a Python identifier'))
def _validate_number(val, sch_field_info): if len(sch_field_info) <= 4: return min = sch_field_info[4] try: fv = float(val) fmin = float(min) except ValueError: return if fv < fmin: raise AssertionError( util.err(sch_field_info, 'numeric value {} out of range', val)) if len(sch_field_info) > 5: max = sch_field_info[5] try: fmax = float(max) except ValueError: return if fv > fmax: raise AssertionError( util.err(sch_field_info, 'numeric value {} out of range', val))
def _validate_enum(val, sch_field_info, sch_enums): """Ensure the value of an enum field is one listed in the schema Args: val: enum value to validate sch_field_info ([str]): field info array from schema sch_enums (pkcollections.Dict): enum section of the schema """ type = sch_field_info[1] if not type in sch_enums: return if str(val) not in map(lambda enum: str(enum[0]), sch_enums[type]): raise AssertionError(util.err(sch_enums, 'enum value {} not in schema', val))
def _validate_enum(val, sch_field_info, sch_enums): """Ensure the value of an enum field is one listed in the schema Args: val: enum value to validate sch_field_info ([str]): field info array from schema sch_enums (pkcollections.Dict): enum section of the schema """ type = sch_field_info[1] if not type in sch_enums: return if str(val) not in map(lambda enum: str(enum[0]), sch_enums[type]): raise AssertionError(util.err(sch_enums, 'enum {} value {} not in schema', type, val))
def api_copySimulation(): """Takes the specified simulation and returns a newly named copy with the suffix ( X)""" req = http_request.parse_json() sim_type = req.simulationType name = req.name assert name, util.err(req, 'No name in request') folder = req.folder if 'folder' in req else '/' data = simulation_db.read_simulation_json(sim_type, sid=req.simulationId) data.models.simulation.name = name data.models.simulation.folder = folder data.models.simulation.isExample = False data.models.simulation.outOfSessionSimulationId = '' return _save_new_and_reply(data)
def _validate_schema(schema): """Validate the schema Validations performed: Values of default data (if any) Existence of dynamic modules Enums keyed by string value Args: schema (pkcollections.Dict): app schema """ sch_models = schema.model sch_enums = schema.enum sch_ntfy = schema.notifications sch_cookies = schema.cookies for name in sch_enums: for values in sch_enums[name]: if not isinstance(values[0], pkconfig.STRING_TYPES): raise AssertionError(util.err(name, 'enum values must be keyed by a string value: {}', type(values[0]))) for model_name in sch_models: sch_model = sch_models[model_name] for field_name in sch_model: sch_field_info = sch_model[field_name] if len(sch_field_info) <= 2: continue field_default = sch_field_info[2] if field_default == '' or field_default is None: continue _validate_enum(field_default, sch_field_info, sch_enums) _validate_number(field_default, sch_field_info) for n in sch_ntfy: if 'cookie' not in sch_ntfy[n] or sch_ntfy[n].cookie not in sch_cookies: raise AssertionError(util.err(sch_ntfy[n], 'notification must reference a cookie in the schema')) for sc in sch_cookies: _validate_cookie_def(sch_cookies[sc]) for type in schema.dynamicModules: for src in schema.dynamicModules[type]: pkresource.filename(src[1:])
def _validate_cookie_def(c_def): """Validate the cookie definitions in the schema Validations performed: cannot contain delimiters we use on the client side values must match the valType if provided timeout must be numeric if provided Args: data (pkcollections.Dict): cookie definition object from the schema """ c_delims = '|:;=' c_delim_re = re.compile('[{}]'.format(c_delims)) if c_delim_re.search(str(c_def.name) + str(c_def.value)): raise AssertionError(util.err(c_def, 'cookie name/value cannot include delimiters {}', c_delims)) if 'valType' in c_def: if c_def.valType == 'b': pkconfig.parse_bool(c_def.value) if c_def.valType == 'n': float(c_def.value) if 'timeout' in c_def: float(c_def.timeout)
def validate_name(data, data_files, max_copies): """Validate and if necessary uniquify name Args: data (dict): what to validate data_files(list): simulation files already in the folder """ s = data.models.simulation sim_id = s.simulationId n = s.name f = s.folder starts_with = pkcollections.Dict() for d in data_files: n2 = d.models.simulation.name if n2.startswith(n) and d.models.simulation.simulationId != sim_id: starts_with[n2] = d.models.simulation.simulationId i = 2 n2 = data.models.simulation.name while n2 in starts_with: n2 = '{} {}'.format(data.models.simulation.name, i) i += 1 assert i - 1 <= max_copies, util.err(n, 'Too many copies: {} > {}', i - 1, max_copies) data.models.simulation.name = n2