def unshare_narrative(self, wsi=None, users=None):
if (wsi is None):
raise ValueError('"wsi" is required')
if (users is None):
raise ValueError('"users" is required')
# First do the actual unsharing in the workspace.
rpc = GenericClient(module='Workspace',
url=self.config['services']['Workspace'],
token=self.token)
rpc.call_func('set_permissions', [{
'id': wsi.id(),
'new_permission': 'n',
'users': users
}])
# Then ensure that the cache for this workspace is
# refreshed.
workspace_cache = PermissionsCache(
workspace_url=self.config['services']['Workspace'],
path=self.config['caches']['workspace']['path'],
username=self.username, # just use the token for now...
token=self.token)
workspace_cache.refresh_items([wsi.id()])
pass
def delete_narrative(self, obji=None):
if (obji is None):
raise ValueError('"wsi" is required')
rpc = GenericClient(module='Workspace',
url=self.config['services']['Workspace'],
token=self.token)
wsi = WorkspaceIdentity(id=obji.workspace_id())
rpc.call_func('delete_workspace', [wsi.make_wsi()])
permissions_cache = PermissionsCache(
workspace_url=self.config['services']['Workspace'],
path=self.config['caches']['workspace']['path'],
username=self.username, # just use the token for now...
token=self.token)
# permissions_cache.remove(wsi)
# TODO:
object_cache = ObjectCache(
workspace_url=self.config['services']['Workspace'],
path=self.config['caches']['object']['path'],
token=self.token)
# object_cache.remove(obji)
pass
def test_initialize(self):
db_path = self.cfg['cache-directory'] + '/object_cache.db'
test_table = [{
'constructor': {
'path': db_path,
'workspace_url': self.cfg['workspace-url'],
'token': self.token,
'username': self.username
}
}]
for test in test_table:
try:
cache = PermissionsCache(**test['constructor'])
cache.initialize()
self.assertTrue(True)
except ValueError as err:
self.assertTrue(False)
def __init__(self, config):
#BEGIN_CONSTRUCTOR
self.config = config
config, err = Validation.validate_config(config)
if err:
raise ValueError(err)
self.call_config = config
def setwal(db):
db.cursor().execute("pragma journal_mode=wal")
# custom auto checkpoint interval (use zero to disable)
db.wal_autocheckpoint(0)
apsw.connection_hooks.append(setwal)
# TODO: move into Model?
user_profile_cache = UserProfileCache(
path=config['caches']['userprofile']['path'],
user_profile_url=config['services']['UserProfile'])
user_profile_cache.initialize()
# The app cache can be populated upon load.
app_cache = AppCache(
path=config['caches']['app']['path'],
narrative_method_store_url=config['services']['NarrativeMethodStore']
)
app_cache.initialize()
object_cache = ObjectCache(
path=config['caches']['object']['path'],
workspace_url=config['services']['Workspace']
)
object_cache.initialize()
workspace_cache = PermissionsCache(
path=config['caches']['workspace']['path'],
workspace_url=config['services']['Workspace']
)
workspace_cache.initialize()
#END_CONSTRUCTOR
pass
def test_fetch_items(self):
db_path = self.cfg['cache-directory'] + '/object_cache.db'
test_table = [{
'constructor': {
'path': db_path,
'workspace_url': self.cfg['workspace-url'],
'token': self.token,
'username': self.username
},
'fetch_items': [[[34742, 'eapearson'], [34599, 'eapearson']]]
}]
for test in test_table:
try:
cache = PermissionsCache(**test['constructor'])
cache.initialize()
items = cache.fetch_items(*test['fetch_items'])
self.assertEqual(len(items), 2)
except ValueError as err:
print('ERROR!', err)
self.assertTrue(False)
def test_refresh(self):
db_path = self.cfg['cache-directory'] + '/object_cache.db'
test_table = [{
'constructor': {
'path': db_path,
'workspace_url': self.cfg['workspace-url'],
'token': self.token,
'username': self.username
},
'add_items': [[[34742, 'eapearson', 'hi'],
[34599, 'eapearson', 'hello']]],
'refresh_items': [[34742, 34599]]
}]
for test in test_table:
try:
cache = PermissionsCache(**test['constructor'])
cache.initialize()
cache.add_items(*test['add_items'])
cache.refresh_items(*test['refresh_items'])
items = cache.get_all_items()
self.assertGreater(len(items), 0)
self.assertTrue(True)
except ValueError as err:
self.assertTrue(False)
def test_add_items_perf(self):
db_path = self.cfg['cache-directory'] + '/object_cache.db'
# test_table = [
# {
# 'constructor': {
# 'path': db_path,
# 'workspace_url': self.cfg['workspace-url'],
# 'token': self.token,
# 'username': self.username
# },
# 'add_items': [[
# [1, 'me', 'hi'],
# [4, 'you', 'hello']
# ]]
# }
# ]
constructor = {
'path': db_path,
'workspace_url': self.cfg['workspace-url'],
'token': self.token,
'username': self.username
}
data = {'field1': 'value1', 'field2': 'value2', 'field3': 1234}
test_data = ([i, 'eapearson', json.dumps(data)]
for i in xrange(0, 1000))
# for test_data in test_table:
try:
cache = PermissionsCache(**constructor)
cache.initialize()
start = time.time()
print('START')
# print(test_data)
cache.add_items(test_data)
elapsed = time.time() - start
print('END %s' % (elapsed / 1000))
all = cache.get_all_items()
self.assertGreater(len(all), 0)
# TODO: test that the items are present and the only ones.
self.assertTrue(True)
except ValueError as err:
print('ERROR!', err)
self.assertTrue(False)
def test_add_items(self):
db_path = self.cfg['cache-directory'] + '/object_cache.db'
test_table = [{
'constructor': {
'path': db_path,
'workspace_url': self.cfg['workspace-url'],
'token': self.token,
'username': self.username
},
'add_items': [[[1, 'me', 'hi'], [4, 'you', 'hello']]]
}]
for test in test_table:
try:
cache = PermissionsCache(**test['constructor'])
cache.initialize()
cache.add_items(*test['add_items'])
all = cache.get_all_items()
self.assertGreater(len(all), 0)
# TODO: test that the items are present and the only ones.
self.assertTrue(True)
except ValueError as err:
print('ERROR!', err)
self.assertTrue(False)
def test_get(self):
db_path = self.cfg['cache-directory'] + '/object_cache.db'
test_table = [{
'constructor': {
'path': db_path,
'workspace_url': self.cfg['workspace-url'],
'token': self.token,
'username': self.username
},
'get': [[34742, 34599]]
}]
for test in test_table:
try:
cache = PermissionsCache(**test['constructor'])
cache.initialize()
items = cache.get(*test['get'])
self.assertEqual(len(items), 2)
items = cache.get(*test['get'])
self.assertEqual(len(items), 2)
self.assertTrue(True)
except ValueError as err:
self.assertTrue(False)
def list_all_narratives(self):
# current_username = ctx['user_id']
stats = []
then = time.time()
# WORKSPACES
print('getting workspaces...')
narrative_workspaces = sorted(self.fetch_narrative_workspaces(),
key=lambda x: x.get('id'))
now = time.time()
print('got %s in %s' % (len(narrative_workspaces), now - then))
stats.append(['list_workspace', now - then])
then = now
# NB - all workpace/narrative related data below must be
# eventually formed into a list with the same order as the
# narrative workspaces.
# PERMISSION
print('getting permissions...')
# TODO: we need to ensure that at the very least the items which are
# returned are returned in the same order requested. This will allow us
# to weave the results back together in the end.
workspace_cache = PermissionsCache(
workspace_url=self.config['services']['Workspace'],
path=self.config['caches']['workspace']['path'],
username=self.username, # just use the token for now...
token=self.token)
workspaces_to_get = [ws['id'] for ws in narrative_workspaces]
narrative_workspaces_perms = workspace_cache.get(workspaces_to_get)
now = time.time()
print('...got %s, %s in %s' %
(len(workspaces_to_get), len(narrative_workspaces_perms),
now - then))
stats.append(['get_permissions', now - then])
then = now
# PROFILE PER USER
print('getting profiles...')
# Get all profiles for all users in the permissions list. It is returned with the
# response so that the caller can map usernames to profiles.
users = set()
for perms in narrative_workspaces_perms:
for username, _perm in perms:
users.add(username)
# We end up with a map of username -> profile
# Transform profiles into map of user to record.
# TODO: parse down profile record, also define in the spec
# The profile record itself is simplified down to just what
# we need
profiles_cache = UserProfileCache(
user_profile_url=self.config['services']['UserProfile'],
path=self.config['caches']['userprofile']['path'])
profiles = profiles_cache.get(list(users))
profiles_map = dict()
# for (username, profile) in itertools.izip(users, profiles):
# profiles_map[username] = profile
for [username, profile] in profiles:
profiles_map[username] = profile
now = time.time()
print('...got %s in %s' % (len(profiles), now - then))
stats.append(['user_profiles', now - then])
then = now
# NARRATIVE OBJECT
# based on the WS lookup table, lookup the narratives
# narrative_objects, _missing, _missed = self.objectInfo.get_object_info_for_workspaces(
# narrative_workspaces, clients['Workspace'])
object_cache = ObjectCache(
workspace_url=self.config['services']['Workspace'],
path=self.config['caches']['object']['path'],
token=self.token)
# object_cache.start()
# object_cache = object_cache.newInstance(token=ctx['token'])
# convert to the cache format, which includes the mod date as timestamp
# NB: includes object id, because the key should have enough info to fetch
# the object from storage as well as form a unique key
to_get = [(ws['id'], int(ws['metadata']['narrative']), ws['modDateMs'])
for ws in narrative_workspaces]
# to_get = [(ws['id'],
# int(ws['metadata']['narrative']))
# for ws in narrative_workspaces]
# TODO: split up into batches of 1000
# if len(to_get) >= 1000:
# to_get = to_get[1:1000]
print('getting objects...')
narrative_objects = object_cache.get(to_get)
print('done')
now = time.time()
print('...got %s in %s' % (len(narrative_objects), now - then))
stats.append(['narrative_objects', now - then])
then = now
# APPS
# Gather all apps in this narrative,
# Get the apps
# Profile a map of apps for a given tag
print('parsing apps...')
(narrative_apps, narrative_cell_types, apps,
elapsed) = self.parse_apps(narrative_objects)
print('...done')
stats.append(['app_gets', elapsed])
now = time.time()
stats.append(['parse_apps', now - then])
then = now
# TODO: permissions, user profiles for permissions, apps
print('assembling...')
# Now weave together the sources into a single narrative
narratives = []
for (ws, obj, perms, cell_stats,
napps) in zip(narrative_workspaces, narrative_objects,
narrative_workspaces_perms, narrative_cell_types,
narrative_apps):
if obj is None:
continue
narratives.append({
'workspaceId': ws['id'],
'objectId': obj['id'],
'objectVersion': obj['version'],
'owner': ws['owner'],
'permission': ws['user_permission'],
'isPublic': ws['isPublic'],
'isNarratorial': ws['isNarratorial'],
'title': ws['metadata']['narrative_nice_name'],
'modifiedTime': ws['modDateMs'],
'savedTime': obj['saveDateMs'],
'savedBy': obj['saved_by'],
'permissions': perms,
'cellTypes': cell_stats,
'apps': napps
})
now = time.time()
stats.append(['finalize', now - then])
then = now
result = {
'narratives': narratives,
'profiles': profiles_map,
'apps': apps
}
return result, stats
def test_construction(self):
db_path = self.cfg['cache-directory'] + '/object_cache.db'
test_table = [{
'constructor': {
'path': db_path,
'workspace_url': self.cfg['workspace-url'],
'token': self.token,
'username': self.username
}
}]
for test in test_table:
try:
PermissionsCache(**test['constructor'])
self.assertTrue(True)
except ValueError as err:
self.assertTrue(False)
test_table = [
{
'input': {
'constructor': {}
},
'expect': {
'exception': True
}
},
{
'input': {
'constructor': {
# 'path': 'x',
'workspace_url': 'x',
'token': 'x',
'username': '******'
}
},
'expect': {
'exception': True
}
},
{
'input': {
'constructor': {
'path': 'x',
# 'workspace_url': 'x',
'token': 'x',
'username': '******'
}
},
'expect': {
'exception': True
}
},
{
'input': {
'constructor': {
'path': 'x',
'workspace_url': 'x',
# 'token': 'x',
'username': '******'
}
},
'expect': {
'exception': False
}
},
{
'input': {
'constructor': {
'path': 'x',
'workspace_url': 'x',
'token': 'x',
# 'username': '******'
}
},
'expect': {
'exception': False
}
},
{
'input': {
'constructor': {
'path': 123,
'workspace_url': 'x',
'token': 'x',
'username': '******'
}
},
'expect': {
'exception': True
}
},
{
'input': {
'constructor': {
'path': 'x',
'workspace_url': 123,
'token': 'x',
'username': '******'
}
},
'expect': {
'exception': True
}
},
{
'input': {
'constructor': {
'path': 'x',
'workspace_url': 'x',
'token': 123,
'username': '******'
}
},
'expect': {
'exception': True
}
},
{
'input': {
'constructor': {
'path': 'x',
'workspace_url': 'x',
'token': 'x',
'username': 123
}
},
'expect': {
'exception': True
}
}
]
for test in test_table:
try:
PermissionsCache(**test['input']['constructor'])
self.assertFalse(test['expect']['exception'])
except ValueError as err:
self.assertTrue(test['expect']['exception'])