def test_const_dict_of_dict(self):
int_dict = ConstDict(int, int)
int_dict_2 = ConstDict(int_dict,int_dict)
d = int_dict({1:2})
d2 = int_dict({1:2,3:4})
big = int_dict_2({d:d2})
self.assertTrue(d in big)
self.assertTrue(d2 not in big)
self.assertTrue(big[d] == d2)
def test_const_dict_lookup(self):
for type_to_use, vals in [
(int, list(range(20))),
(bytes, [b'1', b'2', b'3', b'4', b'5'])
]:
t = ConstDict(type_to_use, type_to_use)
for _ in range(10):
ks = list(vals)
vs = list(vals)
numpy.random.shuffle(ks)
numpy.random.shuffle(vs)
py_d = {}
for i in range(len(ks)):
py_d[ks[i]] = vs[i]
typed_d = t(py_d)
for k in py_d:
self.assertEqual(py_d[k], typed_d[k])
last_k = None
for k in typed_d:
assert last_k is None or k > last_k, (k,last_k)
last_k = k
def test_create_function_with_kwargs_and_star_args_and_defaults(self):
@Function
def f(
x: int,
y=30,
z: None = None,
*args: TupleOf(float),
**kwargs: ConstDict(str, float)
) -> int:
return x + 1
self.assertEqual(len(f.overloads), 1)
o = f.overloads[0]
self.assertEqual(len(o.args), 5)
self.assertEqual([a.name for a in o.args],
['x', 'y', 'z', 'args', 'kwargs'])
self.assertEqual(
[a.typeFilter for a in o.args],
[Int64, None, NoneType,
TupleOf(float),
ConstDict(str, float)])
self.assertEqual([a.defaultValue for a in o.args],
[None, (30, ), (None, ), None, None])
self.assertEqual([a.isStarArg for a in o.args],
[False, False, False, True, False])
self.assertEqual([a.isKwarg for a in o.args],
[False, False, False, False, True])
def test_const_dict_of_tuple(self):
K = NamedTuple(a=OneOf(float, int), b=OneOf(float, int))
someKs = [K(a=0,b=0), K(a=1), K(a=10), K(b=10), K()]
T = ConstDict(K, K)
indexDict = {}
x = T()
numpy.random.seed(42)
for _ in range(100):
i1 = numpy.random.choice(len(someKs))
i2 = numpy.random.choice(len(someKs))
add = numpy.random.choice([False, True])
if add:
indexDict[i1] = i2
x = x + {someKs[i1]: someKs[i2]}
else:
if i1 in indexDict:
del indexDict[i1]
x = x - (someKs[i1],)
self.assertEqual(x, T({someKs[i]:someKs[v] for i,v in indexDict.items()}))
for k in x:
self.assertTrue(k in x)
x[k]
class LoginPlugin:
name = Indexed(str)
# auth plugin
login_plugin_factory = object # factory for LoginPluginInterface objects
auth_plugins = TupleOf(OneOf(None, AuthPluginBase))
codebase = OneOf(None, service_schema.Codebase)
config = ConstDict(str, str)
def test_class_in_forward(self):
class C(Class):
x = Member(int)
Fwd = Forward("Forward")
Fwd = Fwd.define(OneOf(None, C, TupleOf(Fwd), ListOf(Fwd), ConstDict(str, Fwd)))
Fwd(C())
def test_const_dict_lookup_time(self):
int_dict = ConstDict(int, int)
d = int_dict({k:k for k in range(1000000)})
for k in range(1000000):
self.assertTrue(k in d)
self.assertTrue(d[k] == k)
def test_dict_to_oneof(self):
t = ConstDict(str,OneOf("A","B","ABCDEF"))
a = t({'a':'A','b':'ABCDEF'})
self.assertEqual(a['a'], "A")
self.assertEqual(a['b'], "ABCDEF")
self.assertEqual(a, deserialize(t,serialize(t,a)))
def f(
x: int,
y=30,
z: None = None,
*args: TupleOf(float),
**kwargs: ConstDict(str, float)
) -> int:
return x + 1
def test_const_dict_int_perf(self):
t = ConstDict(int,int)
t0 = time.time()
for i in range(100000):
t({k:k+1 for k in range(10)})
elapsed = time.time() - t0
print("Took ", elapsed, " to do 1mm")
self.check_expected_performance(elapsed)
class State:
instance_type = Indexed(str)
booted = int
desired = int
spot_desired = int
spot_booted = int
observedLimit = OneOf(None, int) # maximum observed limit count
capacityConstrained = bool
spotPrices = ConstDict(str, float)
def test_const_dict_str_perf(self):
t = ConstDict(str,str)
t0 = time.time()
for i in range(100000):
t({str(k): str(k+1) for k in range(10)})
elapsed = time.time() - t0
print("Took ", elapsed, " to do 1mm")
self.check_expected_performance(elapsed)
def loop(x: ConstDict(int, int)):
res = 0
i = 0
while i < len(x):
j = 0
while j < len(x):
res = res + x[j] + x[i]
j = j + 1
i = i + 1
return res
def test_dict_hash_perf(self):
str_dict = ConstDict(str, str)
s = str_dict({'a' * 1000000: 'b' * 1000000})
t0 = time.time()
for k in range(1000000):
hash(s)
elapsed = time.time() - t0
print(elapsed, " to do 1mm")
self.check_expected_performance(elapsed)
def test_const_dict(self):
T = ConstDict(int, int)
self.assertEqual(
serialize(T, T({
1: 2,
33: 44
})),
BEGIN_COMPOUND(0) + VARINT(0) + unsignedVarint(2) + # for the size
VARINT(0) + signedVarint(1) + VARINT(0) + signedVarint(2) +
VARINT(0) + signedVarint(33) + VARINT(0) + signedVarint(44) +
END_COMPOUND())
def test_const_dict(self):
t = ConstDict(str,str)
self.assertEqual(len(t()), 0)
self.assertEqual(len(t({})), 0)
self.assertEqual(len(t({'a':'b'})), 1)
self.assertEqual(t({'a':'b'})['a'], 'b')
self.assertEqual(t({'a':'b','b':'c'})['b'], 'c')
self.assertTrue("a" in deserialize(t,serialize(t, t({'a':'b'}))))
self.assertTrue("a" in deserialize(t,serialize(t, t({'a':'b','b':'c'}))))
self.assertTrue("a" in deserialize(t,serialize(t, t({'a':'b','b':'c','c':'d'}))))
self.assertTrue("a" in deserialize(t,serialize(t, t({'a':'b','b':'c','c':'d','d':'e'}))))
self.assertTrue("c" in deserialize(t,serialize(t, t({'a':'b','b':'c','c':'d','def':'e'}))))
self.assertTrue("def" in deserialize(t,serialize(t, t({'a':'b','b':'c','c':'d','def':'e'}))))
def test_refcounts_of_objects_across_boundary(self):
class Object:
pass
_ = Object()
A = Alternative("A", X={'x': int}, Y={'y': int})
for instance in [
TupleOf(int)((1, 2, 3)),
ListOf(int)((1, 2, 3)),
# Dict(int,int)({1:2,3:4}),
ConstDict(int, int)({1: 2, 3: 4}),
AClass(),
# anObject,
A.X(x=10)
]:
self.refcountsTest(instance)
def test_serialize_primitive_compound_types(self):
class A:
pass
B = Alternative("B", X={'a': A})
ts = SerializationContext({'A': A, 'B': B})
for t in [ ConstDict(int, float),
NamedTuple(x=int, y=str),
TupleOf(bool),
Tuple(int, int, bool),
OneOf(int, float),
OneOf(1, 2, 3, "hi", b"goodbye"),
TupleOf(NamedTuple(x=int)),
TupleOf(object),
TupleOf(A),
TupleOf(B)
]:
self.assertIs(ping_pong(t, ts), t)
class TaskStatus:
task = Indexed(Task)
parentStatus = OneOf(None, task_schema.TaskStatus)
resourceScope = Indexed(OneOf(None, ResourceScope))
state = Indexed(
OneOf("Unassigned", "Assigned", "Working", "Sleeping",
"WaitForSubtasks", "DoneCalculating", "Collected"))
wakeup_timestamp = OneOf(None, float)
subtasks = OneOf(None, ConstDict(str, task_schema.TaskStatus))
subtasks_completed = int
times_failed = int
worker = Indexed(OneOf(None, task_schema.TaskWorker))
@revisionConflictRetry
def finish(self, db, result, elapsed=0.0):
with db.transaction():
self._finish(result, elapsed)
def _finish(self, result, elapsed=0.0):
self.task.result = result
self.task.time_elapsed += elapsed
self.worker = None
self.state = "DoneCalculating"
def test_dictionary_addition_and_subtraction(self):
someDicts = [{i:choice([1,2,3,4,5]) for i in range(choice([4,6,10,20]))} for _ in range(20)]
intDict = ConstDict(int,int)
for d1 in someDicts:
for d2 in someDicts:
addResult = dict(d1)
addResult.update(d2)
self.assertEqual(intDict(d1) + intDict(d2), intDict(addResult))
res = intDict(addResult)
while len(res):
toRemove = []
for i in range(choice(list(range(len(res))))+1):
key = choice(list(addResult))
del addResult[key]
toRemove.append(key)
res = res - toRemove
self.assertEqual(res, intDict(addResult))
class Codebase:
hash = Indexed(str)
# filename (at root of project import) to contents
files = ConstDict(str, service_schema.File)
@staticmethod
def createFromRootlevelPath(rootPath):
return Codebase.createFromCodebase(
TypedPythonCodebase.FromRootlevelPath(rootPath))
@staticmethod
def createFromCodebase(codebase: TypedPythonCodebase):
return Codebase.createFromFiles(codebase.filesToContents)
@staticmethod
def createFromFiles(files):
assert files
files = {
k: File.create(v) if not isinstance(v, File) else v
for k, v in files.items()
}
hashval = sha_hash(files).hexdigest
c = Codebase.lookupAny(hash=hashval)
if c:
return c
return Codebase(hash=hashval, files=files)
def instantiate(self, module_name=None, codebase_dir_override=None):
"""Instantiate a codebase on disk and load it."""
with _codebase_lock:
if codebase_dir_override is None:
assert _codebase_instantiation_dir is not None
codebase_dir_override = _codebase_instantiation_dir
if self.hash not in _codebase_cache:
try:
if not os.path.exists(codebase_dir_override):
os.makedirs(codebase_dir_override)
except Exception as e:
logging.getLogger(__name__).warn(
"Exception trying to make directory '%s'",
codebase_dir_override)
logging.getLogger(__name__).warn("Exception: %s", e)
disk_path = os.path.join(codebase_dir_override, self.hash)
# preload the files, since they're lazy.
object_database.current_transaction().db().requestLazyObjects(
set(self.files.values()))
fileContents = {
fpath: file.contents
for fpath, file in self.files.items()
}
_codebase_cache[self.hash] = TypedPythonCodebase.Instantiate(
fileContents, disk_path)
if module_name is None:
return _codebase_cache[self.hash]
return _codebase_cache[self.hash].getModuleByName(module_name)
return TaskStatusResult.Finished(result=self.f(taskContext.db))
class FunctionTask(TaskExecutor):
"""A simple task that just runs a single function."""
def __init__(self, f):
self.f = f
def instantiate(self):
return RunningFunctionTask(self.f)
TaskStatusResult = Alternative(
'TaskStatusResult',
Finished={'result': object},
Subtasks={'subtasks': ConstDict(str, TaskExecutor)},
SleepUntil={'wakeup_timestamp': float})
TaskResult = Alternative("TaskResult",
Result={'result': object},
Error={'error': str},
Failure={})
@task_schema.define
class Task:
service = Indexed(service_schema.Service)
service_and_finished = Index('service', 'finished')
resourceScope = Indexed(OneOf(None, ResourceScope))
executor = TaskExecutor
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import inspect
from object_database.object import DatabaseObject, Index, Indexed
from types import FunctionType
from typed_python import ConstDict, NamedTuple, Tuple, TupleOf
TypeDefinition = NamedTuple(fields=TupleOf(str), indices=TupleOf(str))
SchemaDefinition = ConstDict(str, TypeDefinition)
def SubscribeLazilyByDefault(t):
t.__object_database_lazy_subscription__ = True
return t
class Schema:
"""A collection of types that can be used to access data in a database."""
def __init__(self, name):
self._name = name
# Map: typename:str -> cls(DatabaseObject)
self._types = {}
self._supportingTypes = {}
# class -> indexname -> fun(object->value)
_heartbeatInterval = [5.0]
def setHeartbeatInterval(newInterval):
_heartbeatInterval[0] = newInterval
def getHeartbeatInterval():
return _heartbeatInterval[0]
ClientToServer = Alternative(
"ClientToServer",
TransactionData={
"writes": ConstDict(ObjectFieldId, OneOf(None, bytes)),
"set_adds": ConstDict(IndexId, TupleOf(ObjectId)),
"set_removes": ConstDict(IndexId, TupleOf(ObjectId)),
"key_versions": TupleOf(ObjectFieldId),
"index_versions": TupleOf(IndexId),
"transaction_guid": int
},
CompleteTransaction={
"as_of_version": int,
"transaction_guid": int
},
Heartbeat={},
DefineSchema={ 'name': str, 'definition': SchemaDefinition },
LoadLazyObject={ 'schema': str, 'typename': str, 'identity': ObjectId },
Subscribe={
'schema': str,
def raising(type, msg):
raise type(msg)
Display = lambda: Display
Display = Alternative(
"Display",
Displays={
'displays': TupleOf(Display),
'title': str
},
Plot={
'args': TupleOf(object),
'kwargs': ConstDict(str, object),
'title': str
},
Object={
'object': object,
'title': str
}, # show an arbitrary python object (should be small - a class or module)
Print={
'str': str,
'title': str
}, # show a message from the code.
titled=lambda self, title: Display.Displays(displays=self.displays,
title=title)
if self.matches.Displays else Display.Object(object=self.object,
title=title)
if self.matches.Object else Display.Print(str=self.str, title=title)
def test_dictionary_subtraction_basic(self):
intDict = ConstDict(int,int)
self.assertEqual(intDict({1:2}) - (1,), intDict({}))
self.assertEqual(intDict({1:2, 3:4}) - (1,), intDict({3:4}))
self.assertEqual(intDict({1:2, 3:4}) - (3,), intDict({1:2}))
# See the License for the specific language governing permissions and
# limitations under the License.
from typed_python import Function, ConstDict, String
import typed_python._types as _types
from nativepython.runtime import Runtime
import unittest
import time
def Compiled(f):
f = Function(f)
return Runtime.singleton().compile(f)
dictTypes = [ConstDict(str, str), ConstDict(int, str), ConstDict(int, int)]
def makeSomeValues(dtype, count=10):
res = dtype()
for i in range(count):
if res.KeyType is String:
k = str(i)
else:
k = i
if res.ValueType is String:
v = str(i)
else:
v = i
def test_const_dict(self):
self.assertEqual(ConstDict(str, int).KeyType, String())
self.assertEqual(ConstDict(str, int).ValueType, Int64())
def test_recursive_forwards(self):
Value = Forward("Value")
Value.define(OneOf(
None,
ConstDict(str, Value)
))
class ThingWithDicts:
x = ConstDict(str, bytes)