def validate_schema(self): """ Draft7 formats: https://json-schema.org/understanding-json-schema/reference/string.html for additional format validation refer https://python-jsonschema.readthedocs.io/en/stable/validate/ for ex. enable uri checks: sudo -H pip3 install rfc3987 :return: """ try: import jsonsempai with jsonsempai.imports(): from ocli.ai import recipe_schema # my_path = os.path.abspath(os.path.dirname(__file__)) # path = os.path.join(my_path, "recipe_schema.json") # f = open(path, 'r') # schema = json.load(f) # f.close() schema = recipe_schema.properties.envelope v = Draft7Validator(schema, format_checker=draft7_format_checker) errors = sorted(v.iter_errors(self.recipe), key=lambda e: e.path) if not len(errors): self.log.info(f"recipe {self.file}: syntax is valid") return None for error in errors: self.log.error(f'{error.message} in {list(error.path)}') return 1 except Exception as e: self.log.error(f'Could not perform validation: {e}') return 1
def validate_recipe(self, dry_run): """ Draft7 formats: https://json-schema.org/understanding-json-schema/reference/string.html for additional format validation refer https://python-jsonschema.readthedocs.io/en/stable/validate/ for ex. enable uri checks: sudo -H pip3 install rfc3987 :return: list of errors (empty for valid recipe) """ errors = [] try: with jsonsempai.imports(): from ocli.ai import recipe_head_schema task = self._task schema = recipe_head_schema.properties.envelope template = task.config.get('template') _res = True if template: """check task created from template, validate""" try: tpl = import_module(template) # type: TaskTemplate if not hasattr(tpl, 'validate_schema'): raise ModuleNotFoundError( f"Template is invalid: validate_schema is not defined" ) errors += tpl.validate_schema(schema, self.recipe) except ModuleNotFoundError as e: errors.append(f'Could not import "{template}": {e}') except Exception as e: # log.exception(e) errors.append('Could not perform validation: {e}') return errors
def validate_schema(schema: Dict, recipe: Dict): errors = [] try: with jsonsempai.imports(): from ocli.aikp.cluster import recipe_schema deep_update(schema, recipe_schema.properties.envelope) v = Draft7Validator(schema, format_checker=draft7_format_checker) _errors = sorted(v.iter_errors(recipe), key=lambda e: e.path) errors.extend([ f'Recipe schema "{".".join(e.absolute_path)}" invalid : {e.message}' if e.absolute_path else f"Recipe schema {e.message}" for e in _errors ]) try: _t = numpy.load(os.path.join(recipe['PREDICTOR_DIR'], 'tnorm.npy')) _mlc = max(recipe['learn_channels']) if _t.shape[0] < _mlc: errors.append( f"Predictor is invalid: learning chanel {_mlc} could not be used with tnorm.npy shape [0..{_t.shape[0] - 1}]" ) with open(os.path.join(recipe['PREDICTOR_DIR'], 'gm.pkl'), 'rb') as _f: _t = pickle.load(_f) if recipe['num_clusters'] != _t.n_components: errors.append( f"Predictor is invalid: gm.pkl has {_t.n_components} clusters, \ recipe has {recipe['num_clusters']} clusters ") except Exception as e: errors.append( f"Predictor is invalid: Could not validate tnorm.npy: {e}") except Exception as e: # log.exception(e) errors.append(f'Could not perform validation: {e}') return errors
def test_array_dotting(self): with jsonsempai.imports(): import sempai self.assertEqual('but dotted', sempai.array[0].nested)
def test_obj_in_list_in_list(self): with jsonsempai.imports(): import sempai self.assertTrue(sempai.lots_of_lists[0][0].in_da_list)
def test_set(self): with jsonsempai.imports(): import sempai sempai.one.two.three = 4 self.assertEqual(4, sempai.one.two.three)
def test_location(self): del sys.modules['sempai'] # force the module to be reloaded with jsonsempai.imports(): import sempai self.assertEqual(os.path.join(self.direc, 'sempai.json'), sempai.__file__)
def test_import(self): with jsonsempai.imports(): import sempai self.assertTrue(sempai is not None)
def test_access_nested(self): with jsonsempai.imports(): import sempai self.assertEqual(3, sempai.one.two.three)
def test_import_from_package(self): with jsonsempai.imports(): from python_package import nested self.assertEqual(3, nested.three)
def test_acts_like_dict(self): with jsonsempai.imports(): import sempai self.assertEqual({"three": 3}, sempai.one.two)
def test_access(self): with jsonsempai.imports(): import sempai self.assertEqual(3, sempai.three)
def test_import_package(self): with jsonsempai.imports(): import python_package.nested self.assertEqual(3, python_package.nested.three)
def test_unicode(self): with jsonsempai.imports(): import sempai self.assertEqual(sempai._unicode.chinese, u"汉语/漢語")
def test_del(self): with jsonsempai.imports(): import sempai del sempai.one.two.three self.assertEqual('not at home', sempai.one.two.get('three', 'not at home'))
def test_array(self): with jsonsempai.imports(): import sempai self.assertEqual({"nested": "but dotted"}, sempai.array[0])
def test_array_nested_dotting(self): with jsonsempai.imports(): import sempai self.assertEqual('dotted', sempai.array[1].array[0].and_nested_again)
def test_import_invalid_file(self): with open(os.path.join(self.direc, 'invalid.json'), 'w') as f: f.write('not a valid json file') with jsonsempai.imports(): self.assertRaises(ImportError, __import__, 'invalid')
def test_import_invalid_file(self): with open(os.path.join(self.direc, "invalid.json"), "w") as f: f.write("not a valid json file") with jsonsempai.imports(): self.assertRaises(ImportError, __import__, "invalid")
""" Author balance ============== Version: 0.1.0 Adds balance of given author as score. """ import jsonsempai with jsonsempai.imports(): import mapping from algorithms.utils import create_asset from algorithms.utils import param, pipeable, filter_debug from collections import defaultdict VALUE_BALANCE = """ SELECT received.address as address, received.amount - COALESCE(send.amount, 0) as amount FROM (SELECT receiver as address, sum(cast(amount as decimal)) as amount from persistent_transfer where asset = %(_asset)s and receiver in (SELECT * FROM UNNEST(%(addresses)s)) group by receiver) as received left outer join (SELECT senders as address, sum(cast(amount as decimal)) as amount from persistent_transfer where asset = %(_asset)s and senders in (SELECT * FROM UNNEST(%(addresses)s)) group by senders) as send on received.address = send.address; """ @pipeable @filter_debug @param("asset", required=True) def run(conn_mgr, input, **params): authors = find_all_authors(input["items"]) if not authors:
def test_del(self): with jsonsempai.imports(): import sempai del sempai.one.two.three self.assertEqual("not at home", sempai.one.two.get("three", "not at home"))