def validate_schema(self):
"""
Draft7 formats: https://json-schema.org/understanding-json-schema/reference/string.html
for additional format validation refer https://python-jsonschema.readthedocs.io/en/stable/validate/
for ex. enable uri checks: sudo -H pip3 install rfc3987
:return:
"""
try:
import jsonsempai
with jsonsempai.imports():
from ocli.ai import recipe_schema
# my_path = os.path.abspath(os.path.dirname(__file__))
# path = os.path.join(my_path, "recipe_schema.json")
# f = open(path, 'r')
# schema = json.load(f)
# f.close()
schema = recipe_schema.properties.envelope
v = Draft7Validator(schema, format_checker=draft7_format_checker)
errors = sorted(v.iter_errors(self.recipe), key=lambda e: e.path)
if not len(errors):
self.log.info(f"recipe {self.file}: syntax is valid")
return None
for error in errors:
self.log.error(f'{error.message} in {list(error.path)}')
return 1
except Exception as e:
self.log.error(f'Could not perform validation: {e}')
return 1
def validate_recipe(self, dry_run):
"""
Draft7 formats: https://json-schema.org/understanding-json-schema/reference/string.html
for additional format validation refer https://python-jsonschema.readthedocs.io/en/stable/validate/
for ex. enable uri checks: sudo -H pip3 install rfc3987
:return: list of errors (empty for valid recipe)
"""
errors = []
try:
with jsonsempai.imports():
from ocli.ai import recipe_head_schema
task = self._task
schema = recipe_head_schema.properties.envelope
template = task.config.get('template')
_res = True
if template:
"""check task created from template, validate"""
try:
tpl = import_module(template) # type: TaskTemplate
if not hasattr(tpl, 'validate_schema'):
raise ModuleNotFoundError(
f"Template is invalid: validate_schema is not defined"
)
errors += tpl.validate_schema(schema, self.recipe)
except ModuleNotFoundError as e:
errors.append(f'Could not import "{template}": {e}')
except Exception as e:
# log.exception(e)
errors.append('Could not perform validation: {e}')
return errors
def validate_schema(schema: Dict, recipe: Dict):
errors = []
try:
with jsonsempai.imports():
from ocli.aikp.cluster import recipe_schema
deep_update(schema, recipe_schema.properties.envelope)
v = Draft7Validator(schema, format_checker=draft7_format_checker)
_errors = sorted(v.iter_errors(recipe), key=lambda e: e.path)
errors.extend([
f'Recipe schema "{".".join(e.absolute_path)}" invalid : {e.message}'
if e.absolute_path else f"Recipe schema {e.message}"
for e in _errors
])
try:
_t = numpy.load(os.path.join(recipe['PREDICTOR_DIR'], 'tnorm.npy'))
_mlc = max(recipe['learn_channels'])
if _t.shape[0] < _mlc:
errors.append(
f"Predictor is invalid: learning chanel {_mlc} could not be used with tnorm.npy shape [0..{_t.shape[0] - 1}]"
)
with open(os.path.join(recipe['PREDICTOR_DIR'], 'gm.pkl'),
'rb') as _f:
_t = pickle.load(_f)
if recipe['num_clusters'] != _t.n_components:
errors.append(
f"Predictor is invalid: gm.pkl has {_t.n_components} clusters, \
recipe has {recipe['num_clusters']} clusters ")
except Exception as e:
errors.append(
f"Predictor is invalid: Could not validate tnorm.npy: {e}")
except Exception as e:
# log.exception(e)
errors.append(f'Could not perform validation: {e}')
return errors
def test_array_dotting(self):
with jsonsempai.imports():
import sempai
self.assertEqual('but dotted', sempai.array[0].nested)
def test_obj_in_list_in_list(self):
with jsonsempai.imports():
import sempai
self.assertTrue(sempai.lots_of_lists[0][0].in_da_list)
def test_set(self):
with jsonsempai.imports():
import sempai
sempai.one.two.three = 4
self.assertEqual(4, sempai.one.two.three)
def test_location(self):
del sys.modules['sempai'] # force the module to be reloaded
with jsonsempai.imports():
import sempai
self.assertEqual(os.path.join(self.direc, 'sempai.json'),
sempai.__file__)
def test_import(self):
with jsonsempai.imports():
import sempai
self.assertTrue(sempai is not None)
def test_access_nested(self):
with jsonsempai.imports():
import sempai
self.assertEqual(3, sempai.one.two.three)
def test_set(self):
with jsonsempai.imports():
import sempai
sempai.one.two.three = 4
self.assertEqual(4, sempai.one.two.three)
def test_import_from_package(self):
with jsonsempai.imports():
from python_package import nested
self.assertEqual(3, nested.three)
def test_access_nested(self):
with jsonsempai.imports():
import sempai
self.assertEqual(3, sempai.one.two.three)
def test_acts_like_dict(self):
with jsonsempai.imports():
import sempai
self.assertEqual({"three": 3}, sempai.one.two)
def test_access(self):
with jsonsempai.imports():
import sempai
self.assertEqual(3, sempai.three)
def test_import(self):
with jsonsempai.imports():
import sempai
self.assertTrue(sempai is not None)
def test_import_package(self):
with jsonsempai.imports():
import python_package.nested
self.assertEqual(3, python_package.nested.three)
def test_unicode(self):
with jsonsempai.imports():
import sempai
self.assertEqual(sempai._unicode.chinese, u"汉语/漢語")
def test_del(self):
with jsonsempai.imports():
import sempai
del sempai.one.two.three
self.assertEqual('not at home',
sempai.one.two.get('three', 'not at home'))
def test_import_from_package(self):
with jsonsempai.imports():
from python_package import nested
self.assertEqual(3, nested.three)
def test_location(self):
del sys.modules['sempai'] # force the module to be reloaded
with jsonsempai.imports():
import sempai
self.assertEqual(os.path.join(self.direc, 'sempai.json'),
sempai.__file__)
def test_import_package(self):
with jsonsempai.imports():
import python_package.nested
self.assertEqual(3, python_package.nested.three)
def test_array(self):
with jsonsempai.imports():
import sempai
self.assertEqual({"nested": "but dotted"},
sempai.array[0])
def test_access(self):
with jsonsempai.imports():
import sempai
self.assertEqual(3, sempai.three)
def test_array_dotting(self):
with jsonsempai.imports():
import sempai
self.assertEqual('but dotted', sempai.array[0].nested)
def test_acts_like_dict(self):
with jsonsempai.imports():
import sempai
self.assertEqual({"three": 3}, sempai.one.two)
def test_array_nested_dotting(self):
with jsonsempai.imports():
import sempai
self.assertEqual('dotted', sempai.array[1].array[0].and_nested_again)
def test_del(self):
with jsonsempai.imports():
import sempai
del sempai.one.two.three
self.assertEqual('not at home',
sempai.one.two.get('three', 'not at home'))
def test_obj_in_list_in_list(self):
with jsonsempai.imports():
import sempai
self.assertTrue(sempai.lots_of_lists[0][0].in_da_list)
def test_array(self):
with jsonsempai.imports():
import sempai
self.assertEqual({"nested": "but dotted"},
sempai.array[0])
def test_import_invalid_file(self):
with open(os.path.join(self.direc, 'invalid.json'), 'w') as f:
f.write('not a valid json file')
with jsonsempai.imports():
self.assertRaises(ImportError, __import__, 'invalid')
def test_array_nested_dotting(self):
with jsonsempai.imports():
import sempai
self.assertEqual('dotted', sempai.array[1].array[0].and_nested_again)
def test_import_invalid_file(self):
with open(os.path.join(self.direc, "invalid.json"), "w") as f:
f.write("not a valid json file")
with jsonsempai.imports():
self.assertRaises(ImportError, __import__, "invalid")
"""
Author balance
==============
Version: 0.1.0
Adds balance of given author as score.
"""
import jsonsempai
with jsonsempai.imports():
import mapping
from algorithms.utils import create_asset
from algorithms.utils import param, pipeable, filter_debug
from collections import defaultdict
VALUE_BALANCE = """
SELECT received.address as address, received.amount - COALESCE(send.amount, 0) as amount
FROM (SELECT receiver as address, sum(cast(amount as decimal)) as amount from persistent_transfer where asset = %(_asset)s and receiver in (SELECT * FROM UNNEST(%(addresses)s)) group by receiver) as received
left outer join (SELECT senders as address, sum(cast(amount as decimal)) as amount from persistent_transfer where asset = %(_asset)s and senders in (SELECT * FROM UNNEST(%(addresses)s)) group by senders) as send
on received.address = send.address;
"""
@pipeable
@filter_debug
@param("asset", required=True)
def run(conn_mgr, input, **params):
authors = find_all_authors(input["items"])
if not authors:
def test_unicode(self):
with jsonsempai.imports():
import sempai
self.assertEqual(sempai._unicode.chinese, u"汉语/漢語")
def test_import_invalid_file(self):
with open(os.path.join(self.direc, 'invalid.json'), 'w') as f:
f.write('not a valid json file')
with jsonsempai.imports():
self.assertRaises(ImportError, __import__, 'invalid')
def test_del(self):
with jsonsempai.imports():
import sempai
del sempai.one.two.three
self.assertEqual("not at home", sempai.one.two.get("three", "not at home"))
