def __init__(self, config=None, token=None, username=None):
self.config = config
self.token = token
self.username = username
self.narrative_workspaces = None
self.app_cache = AppCache(
narrative_method_store_url=self.config['services']
['NarrativeMethodStore'],
path=self.config['caches']['app']['path'])
def test_app_cache(self):
db_path = self.cfg['cache-directory'] + '/app_cache.db'
test_table = [{
'constructor': {
'path':
db_path,
'narrative_method_store_url':
self.cfg['narrative-method-store-url']
}
}]
for test in test_table:
try:
AppCache(**test['constructor'])
self.assertTrue(True)
except ValueError as err:
self.assertTrue(False)
test_table = [{
'input': {
'constructor': {}
}
}, {
'input': {
'constructor': {
'narrative_method_store_url': 'x'
}
}
}, {
'input': {
'constructor': {
'path': 'x',
}
}
}, {
'input': {
'constructor': {
'narrative_method_store_url': 123,
'path': 'x',
}
}
}, {
'input': {
'constructor': {
'narrative_method_store_url': 'x',
'path': 123,
}
}
}]
for test in test_table:
try:
AppCache(**test['input']['constructor'])
self.assertTrue(False)
except ValueError as err:
self.assertTrue(True)
def __init__(self, config):
#BEGIN_CONSTRUCTOR
self.config = config
config, err = Validation.validate_config(config)
if err:
raise ValueError(err)
self.call_config = config
def setwal(db):
db.cursor().execute("pragma journal_mode=wal")
# custom auto checkpoint interval (use zero to disable)
db.wal_autocheckpoint(0)
apsw.connection_hooks.append(setwal)
# TODO: move into Model?
user_profile_cache = UserProfileCache(
path=config['caches']['userprofile']['path'],
user_profile_url=config['services']['UserProfile'])
user_profile_cache.initialize()
# The app cache can be populated upon load.
app_cache = AppCache(
path=config['caches']['app']['path'],
narrative_method_store_url=config['services']['NarrativeMethodStore']
)
app_cache.initialize()
object_cache = ObjectCache(
path=config['caches']['object']['path'],
workspace_url=config['services']['Workspace']
)
object_cache.initialize()
workspace_cache = PermissionsCache(
path=config['caches']['workspace']['path'],
workspace_url=config['services']['Workspace']
)
workspace_cache.initialize()
#END_CONSTRUCTOR
pass
def test_initialize(self):
db_path = self.cfg['cache-directory'] + '/app_cache.db'
test_table = [{
'constructor': {
'path':
db_path,
'narrative_method_store_url':
self.cfg['narrative-method-store-url']
}
}]
for test in test_table:
try:
app_cache = AppCache(**test['constructor'])
app_cache.initialize()
self.assertTrue(True)
except ValueError as err:
self.assertTrue(False)
def test_get(self):
db_path = self.cfg['cache-directory'] + '/app_cache.db'
test_table = [{
'constructor': {
'path':
db_path,
'narrative_method_store_url':
self.cfg['narrative-method-store-url']
},
'input': ['ug_kb_ea_utils/run_Fastq_Multx'],
'expect': {
'found': True,
}
}, {
'constructor': {
'path':
db_path,
'narrative_method_store_url':
self.cfg['narrative-method-store-url']
},
'input': ['does_not_exist'],
'expect': {
'found': False,
}
}]
for test in test_table:
try:
app_cache = AppCache(**test['constructor'])
app_cache.initialize()
app_cache.load_all()
app, found = app_cache.get(*test['input'])
if (test['expect']['found']):
self.assertTrue(found)
self.assertIsNotNone(app)
else:
self.assertFalse(found)
except ValueError as err:
self.assertTrue(False)
class Model(object):
def __init__(self, config=None, token=None, username=None):
self.config = config
self.token = token
self.username = username
self.narrative_workspaces = None
self.app_cache = AppCache(
narrative_method_store_url=self.config['services']
['NarrativeMethodStore'],
path=self.config['caches']['app']['path'])
def fetch_narrative_workspaces(self):
rpc = GenericClient(module='Workspace',
url=self.config['services']['Workspace'],
token=self.token)
[ws_list] = rpc.call_func('list_workspace_info', [{
'meta': {
'is_temporary': 'false'
}
}])
# This will filter out invalid narratives - we've already filter for
# "probably valid narrative workspace" above.
# This gives us a list of workspace info but transformed to a
# dictionary.
self.narrative_workspaces = list(
map(
lambda nar_ws: ServiceUtils.ws_info_to_object(nar_ws),
filter(
lambda ws_info: self.is_valid_narrative_workspace(ws_info),
ws_list)))
return self.narrative_workspaces
def is_valid_narrative_workspace(self, ws_info):
metadata = ws_info[8]
if 'narrative' in metadata:
if (metadata['narrative'].isdigit()
and int(metadata['narrative']) > 0):
return True
return False
def parse_apps(self, narrative_objects):
narrative_apps = []
narrative_cell_types = []
query_time = 0
apps_to_get = {}
# pass 1: object stats and collect app ids.
for obj_info in narrative_objects:
# collect summary cell info per narrative.
cell_types = {'app': 0, 'code': 0, 'markdown': 0}
apps = {}
if obj_info is not None:
# print('obj info...', obj_info)
for key in obj_info['metadata'].keys():
key_parts = key.split('.')
# umm...
# if key_parts[0] == 'method' or key_parts[0] == 'app':
# parsed_id = ServiceUtils.parse_app_key(key_parts[1])
# if parsed_id is None:
# continue
# app = {
# # 'key': key_parts[1],
# 'id': parsed_id,
# 'count': int(obj_info['metadata'][key]),
# 'obsolete': True
# }
if key_parts[0] == 'method':
parsed_id = ServiceUtils.parse_app_key(key_parts[1])
if parsed_id is None:
continue
app_ref = parsed_id['shortRef']
apps_to_get[app_ref] = True
apps[app_ref] = apps.get(app_ref, 0) + 1
cell_types['app'] += 1
elif key_parts[0] == 'ipython' or key_parts[0] == 'jupyter':
if key_parts[1] not in cell_types:
cell_types[key_parts[1]] = 0
cell_types[key_parts[1]] += int(
obj_info['metadata'][key])
# condense apps into just one instance per module.id
# apps_map = dict()
# for app in apps:
# ref = app['id']['shortRef']
# if ref not in apps_map:
# apps_map[ref] = app
# else:
# apps_map[ref]['count'] += app['count']
# final_apps = [v for _, v in apps_map.iteritems()]
# final_apps = [v for app_id, count in apps.iteritems()]
narrative_apps.append(apps)
narrative_cell_types.append(cell_types)
# get apps
fetched_apps = self.app_cache.get_items(list(apps_to_get.keys()))
# make dict of fetched apps
apps_db = {}
for app_id, tag, app_info, lookup_app_id in fetched_apps:
app_id_parts = lookup_app_id.split('/')
if len(app_id_parts) == 2:
module = app_id_parts[0]
name = app_id_parts[1]
else:
module = None
name = app_id_parts[0]
app = {
'id': lookup_app_id,
'module': module,
'name': name,
# 'appInfo': app_info
}
# print('app info??', app_ref, app_info)
if app_id is None:
app['notFound'] = True
app['title'] = app['name']
app['subtitle'] = None
app['iconURL'] = None
else:
app['notFound'] = False
# the function name is not actually provided in the
# app info. Weird.
app['version'] = app_info['ver']
app_icon_url = None
if 'icon' in app_info:
app_icon_url = app_info['icon']['url']
else:
app_icon_url = None
app['title'] = app_info['name']
app['subtitle'] = app_info['subtitle']
app['iconURL'] = app_icon_url
apps_db[app_id] = app
return narrative_apps, narrative_cell_types, apps_db, query_time
# def parse_apps(self, narrative_objects):
# narrative_apps = []
# narrative_cell_types = []
# query_time = 0
# for obj_info in narrative_objects:
# # collect summary cell info per narrative.
# cell_types = {
# 'app': 0,
# 'code': 0,
# 'markdown': 0
# }
# apps = []
# if obj_info is not None:
# # print('obj info...', obj_info)
# for key in obj_info['metadata'].keys():
# key_parts = key.split('.')
# # umm...
# # if key_parts[0] == 'method' or key_parts[0] == 'app':
# # parsed_id = ServiceUtils.parse_app_key(key_parts[1])
# # if parsed_id is None:
# # continue
# # app = {
# # # 'key': key_parts[1],
# # 'id': parsed_id,
# # 'count': int(obj_info['metadata'][key]),
# # 'obsolete': True
# # }
# if key_parts[0] == 'method':
# parsed_id = ServiceUtils.parse_app_key(key_parts[1])
# if parsed_id is None:
# continue
# app_ref = parsed_id['shortRef']
# start = time.time()
# tag, app_info = self.app_cache.get(app_ref)
# query_time += (time.time() - start)
# # 'key': key_parts[1],
# app = {
# 'id': parsed_id,
# 'count': int(obj_info['metadata'][key]),
# 'tag': tag
# }
# # print('app info??', app_ref, app_info)
# if app_info is None:
# app['notFound'] = True
# else:
# app_icon_url = None
# # if 'info' not in app_spec:
# # raise ValueError('"info" key not found in app spec ' + app_ref)
# if 'icon' in app_info:
# app_icon_url = app_info['icon']['url']
# else:
# app_icon_url = None
# app['title'] = app_info['name']
# app['subtitle'] = app_info['subtitle']
# app['iconUrl'] = app_icon_url
# apps.append(app)
# cell_types['app'] += 1
# elif key_parts[0] == 'ipython' or key_parts[0] == 'jupyter':
# if key_parts[1] not in cell_types:
# cell_types[key_parts[1]] = 0
# cell_types[key_parts[1]] += int(obj_info['metadata'][key])
# # condense apps into just one instance per module.id
# apps_map = dict()
# for app in apps:
# ref = app['id']['shortRef']
# if ref not in apps_map:
# apps_map[ref] = app
# else:
# apps_map[ref]['count'] += app['count']
# final_apps = [v for _, v in apps_map.iteritems()]
# narrative_apps.append(final_apps)
# narrative_cell_types.append(cell_types)
# return narrative_apps, narrative_cell_types, query_time
def list_all_narratives(self):
# current_username = ctx['user_id']
stats = []
then = time.time()
# WORKSPACES
print('getting workspaces...')
narrative_workspaces = sorted(self.fetch_narrative_workspaces(),
key=lambda x: x.get('id'))
now = time.time()
print('got %s in %s' % (len(narrative_workspaces), now - then))
stats.append(['list_workspace', now - then])
then = now
# NB - all workpace/narrative related data below must be
# eventually formed into a list with the same order as the
# narrative workspaces.
# PERMISSION
print('getting permissions...')
# TODO: we need to ensure that at the very least the items which are
# returned are returned in the same order requested. This will allow us
# to weave the results back together in the end.
workspace_cache = PermissionsCache(
workspace_url=self.config['services']['Workspace'],
path=self.config['caches']['workspace']['path'],
username=self.username, # just use the token for now...
token=self.token)
workspaces_to_get = [ws['id'] for ws in narrative_workspaces]
narrative_workspaces_perms = workspace_cache.get(workspaces_to_get)
now = time.time()
print('...got %s, %s in %s' %
(len(workspaces_to_get), len(narrative_workspaces_perms),
now - then))
stats.append(['get_permissions', now - then])
then = now
# PROFILE PER USER
print('getting profiles...')
# Get all profiles for all users in the permissions list. It is returned with the
# response so that the caller can map usernames to profiles.
users = set()
for perms in narrative_workspaces_perms:
for username, _perm in perms:
users.add(username)
# We end up with a map of username -> profile
# Transform profiles into map of user to record.
# TODO: parse down profile record, also define in the spec
# The profile record itself is simplified down to just what
# we need
profiles_cache = UserProfileCache(
user_profile_url=self.config['services']['UserProfile'],
path=self.config['caches']['userprofile']['path'])
profiles = profiles_cache.get(list(users))
profiles_map = dict()
# for (username, profile) in itertools.izip(users, profiles):
# profiles_map[username] = profile
for [username, profile] in profiles:
profiles_map[username] = profile
now = time.time()
print('...got %s in %s' % (len(profiles), now - then))
stats.append(['user_profiles', now - then])
then = now
# NARRATIVE OBJECT
# based on the WS lookup table, lookup the narratives
# narrative_objects, _missing, _missed = self.objectInfo.get_object_info_for_workspaces(
# narrative_workspaces, clients['Workspace'])
object_cache = ObjectCache(
workspace_url=self.config['services']['Workspace'],
path=self.config['caches']['object']['path'],
token=self.token)
# object_cache.start()
# object_cache = object_cache.newInstance(token=ctx['token'])
# convert to the cache format, which includes the mod date as timestamp
# NB: includes object id, because the key should have enough info to fetch
# the object from storage as well as form a unique key
to_get = [(ws['id'], int(ws['metadata']['narrative']), ws['modDateMs'])
for ws in narrative_workspaces]
# to_get = [(ws['id'],
# int(ws['metadata']['narrative']))
# for ws in narrative_workspaces]
# TODO: split up into batches of 1000
# if len(to_get) >= 1000:
# to_get = to_get[1:1000]
print('getting objects...')
narrative_objects = object_cache.get(to_get)
print('done')
now = time.time()
print('...got %s in %s' % (len(narrative_objects), now - then))
stats.append(['narrative_objects', now - then])
then = now
# APPS
# Gather all apps in this narrative,
# Get the apps
# Profile a map of apps for a given tag
print('parsing apps...')
(narrative_apps, narrative_cell_types, apps,
elapsed) = self.parse_apps(narrative_objects)
print('...done')
stats.append(['app_gets', elapsed])
now = time.time()
stats.append(['parse_apps', now - then])
then = now
# TODO: permissions, user profiles for permissions, apps
print('assembling...')
# Now weave together the sources into a single narrative
narratives = []
for (ws, obj, perms, cell_stats,
napps) in zip(narrative_workspaces, narrative_objects,
narrative_workspaces_perms, narrative_cell_types,
narrative_apps):
if obj is None:
continue
narratives.append({
'workspaceId': ws['id'],
'objectId': obj['id'],
'objectVersion': obj['version'],
'owner': ws['owner'],
'permission': ws['user_permission'],
'isPublic': ws['isPublic'],
'isNarratorial': ws['isNarratorial'],
'title': ws['metadata']['narrative_nice_name'],
'modifiedTime': ws['modDateMs'],
'savedTime': obj['saveDateMs'],
'savedBy': obj['saved_by'],
'permissions': perms,
'cellTypes': cell_stats,
'apps': napps
})
now = time.time()
stats.append(['finalize', now - then])
then = now
result = {
'narratives': narratives,
'profiles': profiles_map,
'apps': apps
}
return result, stats
def create_narrative(self, name=None, title=None):
url = self.config['services']['ServiceWizard']
narrative_service = DynamicServiceClient(url=url,
module='NarrativeService',
token=self.token)
try:
[result
], error = narrative_service.call_func('create_new_narrative',
[{
'title': title
}])
ws = result['workspaceInfo']
obj = result['narrativeInfo']
if error:
return None, error
is_narratorial = True if ws['metadata'].get(
'narratorial') == '1' else False
is_public = ws['globalread'] == 'r'
mod_date_ms = ServiceUtils.iso8601ToMillisSinceEpoch(ws['moddate'])
narrative = {
'workspaceId': ws['id'],
'objectId': obj['id'],
'objectVersion': obj['version'],
'owner': ws['owner'],
'permission': ws['user_permission'],
'isPublic': is_public,
'isNarratorial': is_narratorial,
'title': ws['metadata']['narrative_nice_name'],
'modifiedTime': ws['modDateMs'],
'savedTime': obj['saveDateMs'],
'savedBy': obj['saved_by'],
'permissions': None,
'cellTypes': None,
'apps': None
}
return narrative, None
except Exception as err:
return None, err.message
# Now if the title was provided, set that and save it.
# Otherwise just set the basic metadata.
def delete_narrative(self, obji=None):
if (obji is None):
raise ValueError('"wsi" is required')
rpc = GenericClient(module='Workspace',
url=self.config['services']['Workspace'],
token=self.token)
wsi = WorkspaceIdentity(id=obji.workspace_id())
rpc.call_func('delete_workspace', [wsi.make_wsi()])
permissions_cache = PermissionsCache(
workspace_url=self.config['services']['Workspace'],
path=self.config['caches']['workspace']['path'],
username=self.username, # just use the token for now...
token=self.token)
# permissions_cache.remove(wsi)
# TODO:
object_cache = ObjectCache(
workspace_url=self.config['services']['Workspace'],
path=self.config['caches']['object']['path'],
token=self.token)
# object_cache.remove(obji)
pass
def share_narrative(self, wsi=None, users=None, permission=None):
if (wsi is None):
raise ValueError('"wsi" is required')
if (users is None):
raise ValueError('"users" is required')
if (permission is None):
raise ValueError('"permission" is required')
rpc = GenericClient(module='Workspace',
url=self.config['services']['Workspace'],
token=self.token)
rpc.call_func('set_permissions', [{
'workspace': wsi.workspace(),
'id': wsi.id(),
'new_permission': permission,
'users': users
}])
# Then ensure that the cache for this workspace is
# refreshed.
workspace_cache = PermissionsCache(
workspace_url=self.config['services']['Workspace'],
path=self.config['caches']['workspace']['path'],
username=self.username, # just use the token for now...
token=self.token)
# workspace_cache.start()
workspace_cache.refresh_items([wsi.id()])
pass
def unshare_narrative(self, wsi=None, users=None):
if (wsi is None):
raise ValueError('"wsi" is required')
if (users is None):
raise ValueError('"users" is required')
# First do the actual unsharing in the workspace.
rpc = GenericClient(module='Workspace',
url=self.config['services']['Workspace'],
token=self.token)
rpc.call_func('set_permissions', [{
'id': wsi.id(),
'new_permission': 'n',
'users': users
}])
# Then ensure that the cache for this workspace is
# refreshed.
workspace_cache = PermissionsCache(
workspace_url=self.config['services']['Workspace'],
path=self.config['caches']['workspace']['path'],
username=self.username, # just use the token for now...
token=self.token)
workspace_cache.refresh_items([wsi.id()])
pass
def share_narrative_global(self, wsi=None):
if (wsi is None):
raise ValueError('"wsi" is required')
rpc = GenericClient(module='Workspace',
url=self.config['services']['Workspace'],
token=self.token)
rpc.call_func('set_global_permission', [{
'workspace': wsi.workspace(),
'id': wsi.id(),
'new_permission': 'r',
}])
# Note: the global permissions are on the workspace info, which is
# fetched fresh for every narrative listing, so nothing to uncache.
pass
def unshare_narrative_global(self, wsi=None):
if (wsi is None):
raise ValueError('"wsi" is required')
rpc = GenericClient(module='Workspace',
url=self.config['services']['Workspace'],
token=self.token)
rpc.call_func('set_global_permission', [{
'id': wsi.id(),
'new_permission': 'n',
}])
# Note: the global permissions are on the workspace info, which is
# fetched fresh for every narrative listing, so nothing to uncache.
pass
def test_load(self):
db_path = self.cfg['cache-directory'] + '/app_cache.db'
test_table = [{
'constructor': {
'path':
db_path,
'narrative_method_store_url':
self.cfg['narrative-method-store-url']
},
'load_for_tag': {
'input': ['dev']
},
'expect': {
'success': True,
'exception': False
}
}, {
'constructor': {
'path':
db_path,
'narrative_method_store_url':
self.cfg['narrative-method-store-url']
},
'load_for_tag': {
'input': ['beta']
},
'expect': {
'success': True,
'exception': False
}
}, {
'constructor': {
'path':
db_path,
'narrative_method_store_url':
self.cfg['narrative-method-store-url']
},
'load_for_tag': {
'input': ['release']
},
'expect': {
'success': True,
'exception': False
}
}, {
'constructor': {
'path':
db_path,
'narrative_method_store_url':
self.cfg['narrative-method-store-url']
},
'load_for_tag': {
'input': ['foo']
},
'expect': {
'success': False,
'exception': True,
'exceptionClass': ServiceError,
'error': {
'message': 'Repo-tag [foo] is not supported',
'code': -32500,
'name': 'JSONRPCError'
}
}
}]
for test in test_table:
try:
app_cache = AppCache(**test['constructor'])
app_cache.initialize()
app_cache.load_for_tag(*test['load_for_tag']['input'])
self.assertTrue(test['expect']['success'])
except Exception as err:
# print('ERR', err)
self.assertTrue(test['expect']['exception'])
self.assertIsInstance(err, test['expect']['exceptionClass'])
self.assertIsInstance(str(err), str)
# print('ERR code', err.args[1])
# print(test['expect']['error'])
for key, value in enumerate(test['expect']['error']):
self.assertEquals(getattr(err, key), value)