async def gen_update_object(self, obj_id, data):
param_check(obj_id, UUID, 'obj_id')
param_check(data, dict, 'data')
old_object = await self.gen_object(obj_id)
for key, val in data.items():
old_object[key] = val
_kv_shard_replace_object(self.conn(), obj_id, data)
async def gen_insert_object(self, type_id, data):
param_check(type_id, int, 'type_id')
param_check(data, dict, 'data')
new_id = uuid4()
shard = self.get_shard_from_obj_id(new_id)
await shard.gen_insert_object(new_id, type_id, data)
for edge_definition in self._edge_dict.values():
attr = edge_definition.from_id_attr()
if not (attr in data) or not data[attr]:
continue
from_id = data[attr]
from_id_shard = self.get_shard_from_obj_id(from_id)
await from_id_shard.gen_insert_edge(edge_definition, from_id,
new_id, {})
for index in self._index_dict.values():
if index.indexed_type_id() != type_id:
continue
attr = index.indexed_attr()
if not (attr in data) or not data[attr]:
continue
indexed_value = data[attr]
indexed_shard = self.get_shard_from_obj_id(new_id)
await indexed_shard.gen_insert_index_entry(index, indexed_value,
new_id)
return new_id
async def gen_edges(self,
edge_definition,
from_id,
after=None,
first=None):
param_check(from_id, UUID, 'from_id')
return _kv_shard_get_edges(self.conn(), edge_definition.edge_id(),
from_id, after, first)
async def create_worksheet_instance(context, input_object):
param_check(context, PentContext, 'context')
param_check(input_object, CreateWorksheetInstanceInput, 'input_object')
index_name = 'report_record_number_index'
record_number = input_object.data()['report_record_number']
report = await pent_from_index(context, Report, index_name, record_number)
if report:
input_object.report_id(report.obj_id())
return await create_pent(context, WorksheetInstance, input_object)
async def create_report(context, input_object):
param_check(context, PentContext, 'context')
param_check(input_object, ReportCsvRow, 'input_object')
index_name = 'provider_number_index'
provider_number = input_object.data()['prvdr_num']
provider = await pent_from_index(context, Provider, index_name,
provider_number)
if provider:
input_object.provider_id(provider.obj_id())
return await create_pent(context, Report, input_object)
async def gen_insert_objects(self, type_id, datas):
param_check(datas, list, 'datas')
if len(self._shards) > 1:
raise Exception('shards > 1 currently not supported')
shard = self._shards[0]
new_ids = []
for _ in range(0, len(datas)):
new_ids.append(uuid4())
await shard.gen_insert_objects(new_ids, type_id, datas)
return new_ids
def __init__(self,
*,
graphql_type=None,
python_type=None,
is_nullable,
is_list=False,
list_type=None):
param_check(is_nullable, bool, 'is_nullable')
param_check(is_list, bool, 'is_list')
self._graphql_type = graphql_type
self._python_type = python_type
self._is_nullable = is_nullable
self._is_list = is_list
self._list_type = list_type
async def gen_update_object(self, obj_id, data):
param_check(obj_id, UUID, 'obj_id')
param_check(data, dict, 'data')
if not obj_id in self._objects:
# raise exception?
raise Exception('obj_id not found')
obj = self._objects[obj_id]
for key, val in data.items():
obj[key] = val
obj['updated'] = datetime.now()
self._objects[obj_id] = obj
return obj
def __init__(self, *, index_name, indexed_type_id, indexed_attr,
sql_type_of_index):
param_check(index_name, str, 'index_name')
param_check(indexed_type_id, int, 'indexed_type_id')
param_check(indexed_attr, str, 'indexed_attr')
param_check(sql_type_of_index, str, 'sql_type_of_index')
self._index_name = index_name
self._indexed_type_id = indexed_type_id
self._indexed_attr = indexed_attr
self._sql_type_of_index = sql_type_of_index
async def gen_edges(self,
edge_definition,
from_id,
after=None,
first=None):
param_check(from_id, UUID, 'from_id')
edge_name = edge_definition.edge_name()
if edge_name not in self._all_edges:
self._all_edges[edge_name] = {}
edges = self._all_edges[edge_name].get(from_id, [])
if after:
index = self.get_after_index(edges, after)
edges = edges[index:]
if first:
edges = edges[0:first]
return edges
def execute_test_graphql(query, pent_context, graphql_schema):
param_check(query, str, 'query')
loop = asyncio.new_event_loop()
result = graphql(graphql_schema,
query,
executor=AsyncioExecutor(loop=loop),
context_value=pent_context)
if result.errors:
error = result.errors[0]
print_error('GRAPHQL ERROR')
print_error(error)
orig = error.original_error
print_error('ORIGINAL ERROR')
print_error(orig)
trace = orig.__traceback__
print_error(''.join(traceback.format_tb(trace)))
raise error
return result
async def gen_insert_edge(self,
edge_definition,
from_id,
to_id,
data=None):
param_check(from_id, UUID, 'from_id')
param_check(to_id, UUID, 'to_id')
if data is None:
data = {}
param_check(data, dict, 'data')
edge_name = edge_definition.edge_name()
if edge_name not in self._all_edges:
self._all_edges[edge_name] = OrderedDict()
now = datetime.now()
edge_entry = {
'edge_id': edge_definition.edge_id(),
'from_id': from_id,
'to_id': to_id,
'data': data,
'created': now,
'updated': now,
}
safe_append_to_dict_of_list(self._all_edges[edge_name], from_id,
edge_entry)
def __init__(self, *, index_name, indexed_type_id, indexed_attr):
param_check(indexed_attr, str, 'indexed_attr')
param_check(indexed_type_id, int, 'indexed_type_id')
param_check(index_name, str, 'index_name')
self._indexed_attr = indexed_attr
self._index_name = index_name
self._indexed_type_id = indexed_type_id
def __init__(self, context, obj_id, data):
param_check(context, PentContext, 'context')
param_check(obj_id, UUID, 'obj_id')
param_check(data, dict, 'dict')
self._context = context
self._obj_id = obj_id
self._data = data
def check_insert_object_vars(self, new_id, type_id, data):
param_check(new_id, UUID, 'new_id')
param_check(type_id, int, 'type_id')
param_check(data, dict, 'data')
if 'obj_id' in data:
raise ValueError('Cannot specify obj_id')
if 'type_id' in data:
raise ValueError('Cannot specify type_id')
async def gen_insert_edge(self,
edge_definition,
from_id,
to_id,
data=None):
# print('inserting edge. name: %s from_id: %s to_id %s' % (edge_definition.edge_name(), from_id, to_id))
param_check(from_id, UUID, 'from_id')
param_check(to_id, UUID, 'to_id')
if data is None:
data = {}
param_check(data, dict, 'data')
_kv_shard_insert_edge(self.conn(), edge_definition.edge_id(), from_id,
to_id, data)
def __init__(self, *, shards, edges, indexes):
param_check(shards, list, 'shards')
param_check(edges, list, 'edges')
param_check(indexes, list, 'indexes')
self._shards = shards
# shard => shard_id
self._shard_lookup = dict(zip(self._shards, range(0, len(shards))))
# index_name => index
self._index_dict = dict(
zip([index.index_name() for index in indexes], indexes))
# edge_name => edge
self._edge_dict = dict(zip([edge.edge_name() for edge in edges],
edges))
def create_kvetch_index_table_sql(index_column, index_sql_type, target_column, index_name):
param_check(index_column, str, 'index_column')
param_check(target_column, str, 'target_column')
param_check(index_name, str, 'index_name')
# something is up here. the two indexing keys (not updated) should be unique
return """CREATE TABLE IF NOT EXISTS %s (
row_id INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
%s %s NOT NULL,
%s BINARY(16) NOT NULL,
created DATETIME NOT NULL,
KEY (%s, %s),
KEY (%s, %s),
KEY (created)
) ENGINE=InnoDB;
""" % (index_name, index_column, index_sql_type, target_column,
index_column, target_column, target_column, index_column)
async def gen_objects(self, ids):
param_check(ids, list, 'ids')
if not ids:
raise ValueError('ids must have at least 1 element')
return {obj_id: self._objects.get(obj_id) for obj_id in ids}
async def gen_delete_object(self, obj_id):
param_check(obj_id, UUID, 'obj_id')
shard = self.get_shard_from_obj_id(obj_id)
return await shard.gen_delete_object(obj_id)
async def gen_update_object(self, obj_id, data):
param_check(obj_id, UUID, 'obj_id')
param_check(data, dict, 'data')
shard = self.get_shard_from_obj_id(obj_id)
return await shard.gen_update_object(obj_id, data)
def get_shard_id_from_obj_id(self, obj_id):
# do something less stupid like consistent hashing
# excellent description here http://michaelnielsen.org/blog/consistent-hashing/
param_check(obj_id, UUID, 'obj_id')
return int(obj_id) % len(self._shards)
def get_shard_from_obj_id(self, obj_id):
param_check(obj_id, UUID, 'obj_id')
shard_id = self.get_shard_id_from_obj_id(obj_id)
return self._shards[shard_id]
def get_index(self, index_name):
param_check(index_name, str, 'index_name')
return self._index_dict[index_name]
def sync_kv_get_edge_ids(shard, edge_def, from_id, after=None, first=None):
param_check(shard, KvetchShard, 'shard')
return execute_gen(shard.gen_edge_ids(edge_def, from_id, after, first))
def sync_kv_delete_object(shard, obj_id):
param_check(shard, KvetchShard, 'shard')
return execute_gen(shard.gen_delete_object(obj_id))
async def create_provider(context, input_object):
param_check(context, PentContext, 'context')
param_check(input_object, ProviderCsvRow, 'input_object')
return await create_pent(context, Provider, input_object)
def get_pent_context(kvetch):
param_check(kvetch, Kvetch, 'kvetch')
return PentContext(
kvetch=kvetch,
config=get_hcris_config(),
)
async def gen_delete_object(self, obj_id):
param_check(obj_id, UUID, 'obj_id')
if not obj_id in self._objects:
# raise exception?
raise Exception('id not found')
del self._objects[obj_id]
async def gen_object(self, obj_id):
param_check(obj_id, UUID, 'obj_id')
return self._objects.get(obj_id)
