def test_construction(self): db_path = self.cfg['cache-directory'] + '/object_cache.db' test_table = [{ 'constructor': { 'path': db_path, 'workspace_url': self.cfg['workspace-url'] } }] for test in test_table: try: ObjectCache(**test['constructor']) self.assertTrue(True) except ValueError as err: self.assertTrue(False) test_table = [{ 'input': { 'constructor': {} } }, { 'input': { 'constructor': { 'workspace_url': 'x' } } }, { 'input': { 'constructor': { 'path': 'x', } } }, { 'input': { 'constructor': { 'workspace_url': 123, 'path': 'x', } } }, { 'input': { 'constructor': { 'workspace_url': 'x', 'path': 123, } } }] for test in test_table: try: ObjectCache(**test['input']['constructor']) self.assertTrue(False) except ValueError as err: self.assertTrue(True)
def test_initialize(self): db_path = self.cfg['cache-directory'] + '/object_cache.db' test_table = [{ 'constructor': { 'path': db_path, 'workspace_url': self.cfg['workspace-url'] } }] for test in test_table: try: app_cache = ObjectCache(**test['constructor']) app_cache.initialize() self.assertTrue(True) except ValueError as err: self.assertTrue(False)
def delete_narrative(self, obji=None): if (obji is None): raise ValueError('"wsi" is required') rpc = GenericClient(module='Workspace', url=self.config['services']['Workspace'], token=self.token) wsi = WorkspaceIdentity(id=obji.workspace_id()) rpc.call_func('delete_workspace', [wsi.make_wsi()]) permissions_cache = PermissionsCache( workspace_url=self.config['services']['Workspace'], path=self.config['caches']['workspace']['path'], username=self.username, # just use the token for now... token=self.token) # permissions_cache.remove(wsi) # TODO: object_cache = ObjectCache( workspace_url=self.config['services']['Workspace'], path=self.config['caches']['object']['path'], token=self.token) # object_cache.remove(obji) pass
def test_caching(self): db_path = self.cfg['cache-directory'] + '/object_cache.db' test_table = [{ 'constructor': { 'path': db_path, 'workspace_url': self.cfg['workspace-url'] }, 'get': [[[34742, 1, None], [34599, 1, None]]] }] for test in test_table: try: object_cache = ObjectCache(**test['constructor']) object_cache.initialize() items = object_cache.get_items(*test['get']) self.assertTrue(True) except ValueError as err: self.assertTrue(False)
def __init__(self, config): #BEGIN_CONSTRUCTOR self.config = config config, err = Validation.validate_config(config) if err: raise ValueError(err) self.call_config = config def setwal(db): db.cursor().execute("pragma journal_mode=wal") # custom auto checkpoint interval (use zero to disable) db.wal_autocheckpoint(0) apsw.connection_hooks.append(setwal) # TODO: move into Model? user_profile_cache = UserProfileCache( path=config['caches']['userprofile']['path'], user_profile_url=config['services']['UserProfile']) user_profile_cache.initialize() # The app cache can be populated upon load. app_cache = AppCache( path=config['caches']['app']['path'], narrative_method_store_url=config['services']['NarrativeMethodStore'] ) app_cache.initialize() object_cache = ObjectCache( path=config['caches']['object']['path'], workspace_url=config['services']['Workspace'] ) object_cache.initialize() workspace_cache = PermissionsCache( path=config['caches']['workspace']['path'], workspace_url=config['services']['Workspace'] ) workspace_cache.initialize() #END_CONSTRUCTOR pass
def test_add_items(self): db_path = self.cfg['cache-directory'] + '/object_cache.db' test_table = [{ 'constructor': { 'path': db_path, 'workspace_url': self.cfg['workspace-url'] }, 'add_items': [[[1, 2, 3, 'hi'], [4, 5, 6, 'hello']]] }] for test in test_table: try: object_cache = ObjectCache(**test['constructor']) object_cache.initialize() object_cache.add_items(*test['add_items']) all = object_cache.get_all_items() # TODO: test that the items are present and the only ones. self.assertTrue(True) except ValueError as err: print('ERROR!', err) self.assertTrue(False)
def test_get2(self): db_path = self.cfg['cache-directory'] + '/object_cache.db' now = int(round(time.time() * 1000)) now2 = now + 10000 test_table = [{ 'constructor': { 'path': db_path, 'workspace_url': self.cfg['workspace-url'], 'token': self.token }, 'get1': [[[34742, 1, now], [34599, 1, now]]], 'get2': [[[34742, 1, now2], [34599, 1, 1532558773000]]] }] for test in test_table: try: object_cache = ObjectCache(**test['constructor']) object_cache.initialize() items = object_cache.get(*test['get1']) items2 = object_cache.get(*test['get2']) self.assertTrue(True) except ValueError as err: self.assertTrue(False)
def list_all_narratives(self): # current_username = ctx['user_id'] stats = [] then = time.time() # WORKSPACES print('getting workspaces...') narrative_workspaces = sorted(self.fetch_narrative_workspaces(), key=lambda x: x.get('id')) now = time.time() print('got %s in %s' % (len(narrative_workspaces), now - then)) stats.append(['list_workspace', now - then]) then = now # NB - all workpace/narrative related data below must be # eventually formed into a list with the same order as the # narrative workspaces. # PERMISSION print('getting permissions...') # TODO: we need to ensure that at the very least the items which are # returned are returned in the same order requested. This will allow us # to weave the results back together in the end. workspace_cache = PermissionsCache( workspace_url=self.config['services']['Workspace'], path=self.config['caches']['workspace']['path'], username=self.username, # just use the token for now... token=self.token) workspaces_to_get = [ws['id'] for ws in narrative_workspaces] narrative_workspaces_perms = workspace_cache.get(workspaces_to_get) now = time.time() print('...got %s, %s in %s' % (len(workspaces_to_get), len(narrative_workspaces_perms), now - then)) stats.append(['get_permissions', now - then]) then = now # PROFILE PER USER print('getting profiles...') # Get all profiles for all users in the permissions list. It is returned with the # response so that the caller can map usernames to profiles. users = set() for perms in narrative_workspaces_perms: for username, _perm in perms: users.add(username) # We end up with a map of username -> profile # Transform profiles into map of user to record. # TODO: parse down profile record, also define in the spec # The profile record itself is simplified down to just what # we need profiles_cache = UserProfileCache( user_profile_url=self.config['services']['UserProfile'], path=self.config['caches']['userprofile']['path']) profiles = profiles_cache.get(list(users)) profiles_map = dict() # for (username, profile) in itertools.izip(users, profiles): # profiles_map[username] = profile for [username, profile] in profiles: profiles_map[username] = profile now = time.time() print('...got %s in %s' % (len(profiles), now - then)) stats.append(['user_profiles', now - then]) then = now # NARRATIVE OBJECT # based on the WS lookup table, lookup the narratives # narrative_objects, _missing, _missed = self.objectInfo.get_object_info_for_workspaces( # narrative_workspaces, clients['Workspace']) object_cache = ObjectCache( workspace_url=self.config['services']['Workspace'], path=self.config['caches']['object']['path'], token=self.token) # object_cache.start() # object_cache = object_cache.newInstance(token=ctx['token']) # convert to the cache format, which includes the mod date as timestamp # NB: includes object id, because the key should have enough info to fetch # the object from storage as well as form a unique key to_get = [(ws['id'], int(ws['metadata']['narrative']), ws['modDateMs']) for ws in narrative_workspaces] # to_get = [(ws['id'], # int(ws['metadata']['narrative'])) # for ws in narrative_workspaces] # TODO: split up into batches of 1000 # if len(to_get) >= 1000: # to_get = to_get[1:1000] print('getting objects...') narrative_objects = object_cache.get(to_get) print('done') now = time.time() print('...got %s in %s' % (len(narrative_objects), now - then)) stats.append(['narrative_objects', now - then]) then = now # APPS # Gather all apps in this narrative, # Get the apps # Profile a map of apps for a given tag print('parsing apps...') (narrative_apps, narrative_cell_types, apps, elapsed) = self.parse_apps(narrative_objects) print('...done') stats.append(['app_gets', elapsed]) now = time.time() stats.append(['parse_apps', now - then]) then = now # TODO: permissions, user profiles for permissions, apps print('assembling...') # Now weave together the sources into a single narrative narratives = [] for (ws, obj, perms, cell_stats, napps) in zip(narrative_workspaces, narrative_objects, narrative_workspaces_perms, narrative_cell_types, narrative_apps): if obj is None: continue narratives.append({ 'workspaceId': ws['id'], 'objectId': obj['id'], 'objectVersion': obj['version'], 'owner': ws['owner'], 'permission': ws['user_permission'], 'isPublic': ws['isPublic'], 'isNarratorial': ws['isNarratorial'], 'title': ws['metadata']['narrative_nice_name'], 'modifiedTime': ws['modDateMs'], 'savedTime': obj['saveDateMs'], 'savedBy': obj['saved_by'], 'permissions': perms, 'cellTypes': cell_stats, 'apps': napps }) now = time.time() stats.append(['finalize', now - then]) then = now result = { 'narratives': narratives, 'profiles': profiles_map, 'apps': apps } return result, stats