async def gen_update_object(self, obj_id, data): param_check(obj_id, UUID, 'obj_id') param_check(data, dict, 'data') old_object = await self.gen_object(obj_id) for key, val in data.items(): old_object[key] = val _kv_shard_replace_object(self.conn(), obj_id, data)
async def gen_insert_object(self, type_id, data): param_check(type_id, int, 'type_id') param_check(data, dict, 'data') new_id = uuid4() shard = self.get_shard_from_obj_id(new_id) await shard.gen_insert_object(new_id, type_id, data) for edge_definition in self._edge_dict.values(): attr = edge_definition.from_id_attr() if not (attr in data) or not data[attr]: continue from_id = data[attr] from_id_shard = self.get_shard_from_obj_id(from_id) await from_id_shard.gen_insert_edge(edge_definition, from_id, new_id, {}) for index in self._index_dict.values(): if index.indexed_type_id() != type_id: continue attr = index.indexed_attr() if not (attr in data) or not data[attr]: continue indexed_value = data[attr] indexed_shard = self.get_shard_from_obj_id(new_id) await indexed_shard.gen_insert_index_entry(index, indexed_value, new_id) return new_id
async def gen_edges(self, edge_definition, from_id, after=None, first=None): param_check(from_id, UUID, 'from_id') return _kv_shard_get_edges(self.conn(), edge_definition.edge_id(), from_id, after, first)
async def create_worksheet_instance(context, input_object): param_check(context, PentContext, 'context') param_check(input_object, CreateWorksheetInstanceInput, 'input_object') index_name = 'report_record_number_index' record_number = input_object.data()['report_record_number'] report = await pent_from_index(context, Report, index_name, record_number) if report: input_object.report_id(report.obj_id()) return await create_pent(context, WorksheetInstance, input_object)
async def create_report(context, input_object): param_check(context, PentContext, 'context') param_check(input_object, ReportCsvRow, 'input_object') index_name = 'provider_number_index' provider_number = input_object.data()['prvdr_num'] provider = await pent_from_index(context, Provider, index_name, provider_number) if provider: input_object.provider_id(provider.obj_id()) return await create_pent(context, Report, input_object)
async def gen_insert_objects(self, type_id, datas): param_check(datas, list, 'datas') if len(self._shards) > 1: raise Exception('shards > 1 currently not supported') shard = self._shards[0] new_ids = [] for _ in range(0, len(datas)): new_ids.append(uuid4()) await shard.gen_insert_objects(new_ids, type_id, datas) return new_ids
def __init__(self, *, graphql_type=None, python_type=None, is_nullable, is_list=False, list_type=None): param_check(is_nullable, bool, 'is_nullable') param_check(is_list, bool, 'is_list') self._graphql_type = graphql_type self._python_type = python_type self._is_nullable = is_nullable self._is_list = is_list self._list_type = list_type
async def gen_update_object(self, obj_id, data): param_check(obj_id, UUID, 'obj_id') param_check(data, dict, 'data') if not obj_id in self._objects: # raise exception? raise Exception('obj_id not found') obj = self._objects[obj_id] for key, val in data.items(): obj[key] = val obj['updated'] = datetime.now() self._objects[obj_id] = obj return obj
def __init__(self, *, index_name, indexed_type_id, indexed_attr, sql_type_of_index): param_check(index_name, str, 'index_name') param_check(indexed_type_id, int, 'indexed_type_id') param_check(indexed_attr, str, 'indexed_attr') param_check(sql_type_of_index, str, 'sql_type_of_index') self._index_name = index_name self._indexed_type_id = indexed_type_id self._indexed_attr = indexed_attr self._sql_type_of_index = sql_type_of_index
async def gen_edges(self, edge_definition, from_id, after=None, first=None): param_check(from_id, UUID, 'from_id') edge_name = edge_definition.edge_name() if edge_name not in self._all_edges: self._all_edges[edge_name] = {} edges = self._all_edges[edge_name].get(from_id, []) if after: index = self.get_after_index(edges, after) edges = edges[index:] if first: edges = edges[0:first] return edges
def execute_test_graphql(query, pent_context, graphql_schema): param_check(query, str, 'query') loop = asyncio.new_event_loop() result = graphql(graphql_schema, query, executor=AsyncioExecutor(loop=loop), context_value=pent_context) if result.errors: error = result.errors[0] print_error('GRAPHQL ERROR') print_error(error) orig = error.original_error print_error('ORIGINAL ERROR') print_error(orig) trace = orig.__traceback__ print_error(''.join(traceback.format_tb(trace))) raise error return result
async def gen_insert_edge(self, edge_definition, from_id, to_id, data=None): param_check(from_id, UUID, 'from_id') param_check(to_id, UUID, 'to_id') if data is None: data = {} param_check(data, dict, 'data') edge_name = edge_definition.edge_name() if edge_name not in self._all_edges: self._all_edges[edge_name] = OrderedDict() now = datetime.now() edge_entry = { 'edge_id': edge_definition.edge_id(), 'from_id': from_id, 'to_id': to_id, 'data': data, 'created': now, 'updated': now, } safe_append_to_dict_of_list(self._all_edges[edge_name], from_id, edge_entry)
def __init__(self, *, index_name, indexed_type_id, indexed_attr): param_check(indexed_attr, str, 'indexed_attr') param_check(indexed_type_id, int, 'indexed_type_id') param_check(index_name, str, 'index_name') self._indexed_attr = indexed_attr self._index_name = index_name self._indexed_type_id = indexed_type_id
def __init__(self, context, obj_id, data): param_check(context, PentContext, 'context') param_check(obj_id, UUID, 'obj_id') param_check(data, dict, 'dict') self._context = context self._obj_id = obj_id self._data = data
def check_insert_object_vars(self, new_id, type_id, data): param_check(new_id, UUID, 'new_id') param_check(type_id, int, 'type_id') param_check(data, dict, 'data') if 'obj_id' in data: raise ValueError('Cannot specify obj_id') if 'type_id' in data: raise ValueError('Cannot specify type_id')
async def gen_insert_edge(self, edge_definition, from_id, to_id, data=None): # print('inserting edge. name: %s from_id: %s to_id %s' % (edge_definition.edge_name(), from_id, to_id)) param_check(from_id, UUID, 'from_id') param_check(to_id, UUID, 'to_id') if data is None: data = {} param_check(data, dict, 'data') _kv_shard_insert_edge(self.conn(), edge_definition.edge_id(), from_id, to_id, data)
def __init__(self, *, shards, edges, indexes): param_check(shards, list, 'shards') param_check(edges, list, 'edges') param_check(indexes, list, 'indexes') self._shards = shards # shard => shard_id self._shard_lookup = dict(zip(self._shards, range(0, len(shards)))) # index_name => index self._index_dict = dict( zip([index.index_name() for index in indexes], indexes)) # edge_name => edge self._edge_dict = dict(zip([edge.edge_name() for edge in edges], edges))
def create_kvetch_index_table_sql(index_column, index_sql_type, target_column, index_name): param_check(index_column, str, 'index_column') param_check(target_column, str, 'target_column') param_check(index_name, str, 'index_name') # something is up here. the two indexing keys (not updated) should be unique return """CREATE TABLE IF NOT EXISTS %s ( row_id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, %s %s NOT NULL, %s BINARY(16) NOT NULL, created DATETIME NOT NULL, KEY (%s, %s), KEY (%s, %s), KEY (created) ) ENGINE=InnoDB; """ % (index_name, index_column, index_sql_type, target_column, index_column, target_column, target_column, index_column)
async def gen_objects(self, ids): param_check(ids, list, 'ids') if not ids: raise ValueError('ids must have at least 1 element') return {obj_id: self._objects.get(obj_id) for obj_id in ids}
async def gen_delete_object(self, obj_id): param_check(obj_id, UUID, 'obj_id') shard = self.get_shard_from_obj_id(obj_id) return await shard.gen_delete_object(obj_id)
async def gen_update_object(self, obj_id, data): param_check(obj_id, UUID, 'obj_id') param_check(data, dict, 'data') shard = self.get_shard_from_obj_id(obj_id) return await shard.gen_update_object(obj_id, data)
def get_shard_id_from_obj_id(self, obj_id): # do something less stupid like consistent hashing # excellent description here http://michaelnielsen.org/blog/consistent-hashing/ param_check(obj_id, UUID, 'obj_id') return int(obj_id) % len(self._shards)
def get_shard_from_obj_id(self, obj_id): param_check(obj_id, UUID, 'obj_id') shard_id = self.get_shard_id_from_obj_id(obj_id) return self._shards[shard_id]
def get_index(self, index_name): param_check(index_name, str, 'index_name') return self._index_dict[index_name]
def sync_kv_get_edge_ids(shard, edge_def, from_id, after=None, first=None): param_check(shard, KvetchShard, 'shard') return execute_gen(shard.gen_edge_ids(edge_def, from_id, after, first))
def sync_kv_delete_object(shard, obj_id): param_check(shard, KvetchShard, 'shard') return execute_gen(shard.gen_delete_object(obj_id))
async def create_provider(context, input_object): param_check(context, PentContext, 'context') param_check(input_object, ProviderCsvRow, 'input_object') return await create_pent(context, Provider, input_object)
def get_pent_context(kvetch): param_check(kvetch, Kvetch, 'kvetch') return PentContext( kvetch=kvetch, config=get_hcris_config(), )
async def gen_delete_object(self, obj_id): param_check(obj_id, UUID, 'obj_id') if not obj_id in self._objects: # raise exception? raise Exception('id not found') del self._objects[obj_id]
async def gen_object(self, obj_id): param_check(obj_id, UUID, 'obj_id') return self._objects.get(obj_id)