def createCache(obj, formatted, modelType, user):
obj = MODELS()[modelType]().load(obj['_id'], force=True)
if "cached" in obj:
oc = obj.get("oldCache", [])
obj["oldCache"] = (oc if oc is not None else []).append(obj["cached"])
if modelType in NONES:
print("No modelType!")
print(obj)
if formatted is None:
print("formatting failed!")
print(obj)
obj["cached"] = json_util.dumps({
**formatted, "prov:generatedAtTime":
xsdNow()
})
return (MODELS()[modelType]().save(obj, validate=False))
def smartImport(IRI, user=None, refreshCache=False, modelType=None):
from girderformindlogger.constants import MODELS
from girderformindlogger.utility.jsonld_expander import loadCache, \
reprolibCanonize
MODELS = MODELS()
mt1 = "screen" if modelType in [None, "external JSON-LD document"
] else modelType
model, modelType = MODELS[mt1]().getFromUrl(IRI,
user=user,
refreshCache=refreshCache,
thread=False)
return ((modelType, loadCache(model.get('cached',
model)), reprolibCanonize(IRI)))
def createCache(obj, formatted, modelType, user=None):
obj = MODELS()[modelType]().load(obj['_id'], force=True)
if modelType in NONES:
print("No modelType!")
print(obj)
if formatted is None:
print("formatting failed!")
print(obj)
if obj.get('cached'):
cache_id = obj['cached']
CacheModel().updateCache(cache_id,
MODELS()[modelType]().name, obj['_id'],
modelType, formatted)
else:
saved = CacheModel().insertCache(MODELS()[modelType]().name,
obj['_id'], modelType, formatted)
obj['cached'] = saved['_id']
MODELS()[modelType]().update({'_id': ObjectId(obj['_id'])},
{'$set': {
'cached': obj['cached']
}}, False)
return obj
def getHistoryDataFromItemIRIs(self, protocolId, IRIGroup):
from girderformindlogger.models.item import Item as ItemModel
from girderformindlogger.utility import jsonld_expander
protocol = self.load(protocolId, force=True)
items = {}
activities = {}
itemReferences = {}
result = {
'items': items,
'activities': activities,
'itemReferences': itemReferences
}
if 'historyId' not in protocol.get('meta', {}):
return result
historyFolder = FolderModel().load(protocol['meta']['historyId'],
force=True)
if 'referenceId' not in historyFolder.get('meta', {}):
return result
referencesFolder = FolderModel().load(
historyFolder['meta']['referenceId'], force=True)
itemModel = ItemModel()
for IRI in IRIGroup:
reference = itemModel.findOne({
'folderId': referencesFolder['_id'],
'meta.identifier': IRI
})
if not reference:
continue
history = reference['meta']['history']
for version in IRIGroup[IRI]:
if version not in itemReferences:
itemReferences[version] = {}
inserted = False
for i in range(0, len(history)):
if self.compareVersions(version,
history[i]['version']) <= 0:
if not history[i].get('reference', None):
continue
if history[i]['reference'] not in items:
(modelType,
referenceId) = history[i]['reference'].split('/')
model = MODELS()[modelType]().findOne(
{'_id': ObjectId(referenceId)})
items[history[i]
['reference']] = jsonld_expander.loadCache(
model['cached'])
activityId = str(model['meta']['activityId'])
if activityId not in activities:
activities[
activityId] = jsonld_expander.loadCache(
FolderModel().load(
activityId, force=True)['cached'])
if history[i]['reference']:
itemReferences[version][IRI] = history[i][
'reference']
inserted = True
break
if not inserted:
itemReferences[version][
IRI] = None # this is same as latest version
return result
def importAndCompareModelType(model, url, user, modelType):
import threading
from girderformindlogger.utility import firstLower
if model is None:
return (None, None)
mt = model.get('@type', '')
mt = mt[0] if isinstance(mt, list) else mt
atType = mt.split('/')[-1].split(':')[-1]
modelType = firstLower(atType) if len(atType) else modelType
modelType = 'screen' if modelType.lower(
) == 'field' else 'protocol' if modelType.lower(
) == 'activityset' else modelType
changedModel = ((atType != modelType and len(atType))
or (" " in modelType))
modelType = firstLower(atType) if changedModel else modelType
modelType = 'screen' if modelType.lower(
) == 'field' else 'protocol' if modelType.lower(
) == 'activityset' else modelType
modelClass = MODELS()[modelType]()
prefName = modelClass.preferredName(model)
cachedDocObj = {}
model = expand(url)
print("Loaded {}".format(": ".join([modelType, prefName])))
docCollection = getModelCollection(modelType)
if modelClass.name in ['folder', 'item']:
docFolder = FolderModel().createFolder(
name=prefName,
parent=docCollection,
parentType='collection',
public=True,
creator=user,
allowRename=True,
reuseExisting=(modelType != 'applet'))
if modelClass.name == 'folder':
newModel = modelClass.setMetadata(
docFolder,
{modelType: {
**model, 'schema:url': url,
'url': url
}})
elif modelClass.name == 'item':
newModel = modelClass.setMetadata(
modelClass.createItem(name=prefName if prefName else str(
len(
list(FolderModel().childItems(FolderModel().load(
docFolder, level=None, user=user, force=True)))) +
1),
creator=user,
folder=docFolder,
reuseExisting=True),
{modelType: {
**model, 'schema:url': url,
'url': url
}})
formatted = _fixUpFormat(
formatLdObject(newModel,
mesoPrefix=modelType,
user=user,
refreshCache=True))
createCache(newModel, formatted, modelType, user)
return (formatted, modelType)
def formatLdObject(obj,
mesoPrefix='folder',
user=None,
keepUndefined=False,
dropErrors=False,
refreshCache=False,
responseDates=False):
"""
Function to take a compacted JSON-LD Object within a Girder for Mindlogger
database and return an exapanded JSON-LD Object including an _id.
:param obj: Compacted JSON-LD Object
:type obj: dict or list
:param mesoPrefix: Girder for Mindlogger entity type, defaults to 'folder'
if not provided
:type mesoPrefix: str
:param user: User making the call
:type user: User
:param keepUndefined: Keep undefined properties
:type keepUndefined: bool
:param dropErrors: Return `None` instead of raising an error for illegal
JSON-LD definitions.
:type dropErrors: bool
:param refreshCache: Refresh from Dereferencing URLs?
:type refreshCache: bool
:param responseDates: Include list of ISO date strings of responses
:type responseDates: bool
:returns: Expanded JSON-LD Object (dict or list)
"""
from girderformindlogger.models import pluralize
refreshCache = False if refreshCache is None else refreshCache
try:
if obj is None:
return (None)
if isinstance(obj, dict):
oc = obj.get("cached")
if all([not refreshCache, oc is not None]):
return (loadCache(oc))
if 'meta' not in obj.keys():
return (_fixUpFormat(obj))
mesoPrefix = camelCase(mesoPrefix)
if type(obj) == list:
return (_fixUpFormat([
formatLdObject(o,
mesoPrefix,
refreshCache=refreshCache,
user=user) for o in obj if o is not None
]))
if not type(obj) == dict and not dropErrors:
raise TypeError("JSON-LD must be an Object or Array.")
newObj = obj.get('meta', obj)
newObj = newObj.get(mesoPrefix, newObj)
if not obj.get('loadedFromSingleFile', False):
newObj = expand(newObj, keepUndefined=keepUndefined)
if type(newObj) == list and len(newObj) == 1:
try:
newObj = newObj[0]
except:
raise ValidationException(str(newObj))
if type(newObj) != dict:
newObj = {}
objID = str(obj.get('_id', 'undefined'))
if objID == 'undefined':
raise ResourcePathNotFound('unable to load object')
newObj['_id'] = "/".join([snake_case(mesoPrefix), objID])
if mesoPrefix == 'applet':
protocolUrl = obj.get('meta', {}).get('protocol', obj).get(
'http://schema.org/url',
obj.get('meta', {}).get('protocol', obj).get('url'))
# get protocol data from id
protocol = None
protocolId = obj.get('meta', {}).get('protocol',
{}).get('_id',
'').split('/')[-1]
if protocolId:
cache = ProtocolModel().getCache(protocolId)
if cache:
protocol = loadCache(cache)
if protocolUrl is not None and not protocol:
# get protocol from url
protocol = ProtocolModel().getFromUrl(
protocolUrl,
'protocol',
user,
thread=False,
refreshCache=refreshCache)[0]
# format protocol data
protocol = formatLdObject(protocol,
'protocol',
user,
refreshCache=refreshCache)
applet = {}
applet['activities'] = protocol.pop('activities', {})
applet['items'] = protocol.pop('items', {})
applet['protocol'] = {
key: protocol.get('protocol', protocol.get('activitySet',
{})).pop(key)
for key in
['@type', '_id', 'http://schema.org/url', 'schema:url', 'url']
if key in list(protocol.get('protocol', {}).keys())
}
applet['applet'] = {
**protocol.pop('protocol', {}),
**obj.get('meta', {}).get(mesoPrefix, {}), '_id':
"/".join([snake_case(mesoPrefix), objID]),
'url':
"#".join(
[obj.get('meta', {}).get('protocol', {}).get("url", "")])
}
if 'appletName' in obj and obj['appletName']:
suffix = obj['appletName'].split('/')[-1]
inserted = False
candidates = ['prefLabel', 'altLabel']
for candidate in candidates:
for key in applet['applet']:
if not inserted and str(key).endswith(
candidate) and len(
applet['applet'][key]) and len(
applet['applet'][key][0].get(
'@value', '')):
if len(suffix):
applet['applet'][key][0]['@value'] += (' ' +
suffix)
if not len(applet['applet']
['url']): # for development
applet['applet'][key][0][
'@value'] += ' ( single-file )'
AppletModel().update({'_id': obj['_id']}, {
'$set': {
'displayName':
applet['applet'][key][0]['@value']
}
})
inserted = True
createCache(obj, applet, 'applet', user)
if responseDates:
try:
applet["applet"]["responseDates"] = responseDateList(
obj.get('_id'), user.get('_id'), user)
except:
applet["applet"]["responseDates"] = []
return (applet)
elif mesoPrefix == 'protocol':
protocol = {'protocol': newObj, 'activities': {}, "items": {}}
if obj.get('loadedFromSingleFile', False):
activities = list(ActivityModel().find({
'meta.protocolId':
'{}/{}'.format(MODELS()['protocol']().name, obj['_id'])
}))
items = list(ScreenModel().find({
'meta.protocolId':
'{}/{}'.format(MODELS()['protocol']().name, obj['_id'])
}))
for activity in activities:
formatted = formatLdObject(activity, 'activity', user)
protocol['activities'][formatted['@id']] = formatted
for item in items:
formatted = formatLdObject(item, 'screen', user)
protocol['items'][formatted['@id']] = formatted
else:
try:
protocol = componentImport(newObj,
deepcopy(protocol),
user,
refreshCache=refreshCache)
except:
print("636")
protocol = componentImport(newObj,
deepcopy(protocol),
user,
refreshCache=True)
newActivities = protocol.get('activities', {}).keys()
newItems = protocol.get('items', {}).keys()
while (any([len(newActivities), len(newItems)])):
activitiesNow = set(protocol.get('activities', {}).keys())
itemsNow = set(protocol.get('items', {}).keys())
for activityURL in newActivities:
activity = protocol['activities'][activityURL]
activity = activity.get('meta',
{}).get('activity', activity)
try:
protocol = componentImport(
deepcopy(activity),
deepcopy(protocol),
user,
refreshCache=refreshCache)
except:
print("670")
protocol = componentImport(deepcopy(activity),
deepcopy(protocol),
user,
refreshCache=True)
for itemURL in newItems:
activity = protocol['items'][itemURL]
activity = activity.get('meta',
{}).get('screen', activity)
try:
protocol = componentImport(
deepcopy(activity),
deepcopy(protocol),
user,
refreshCache=refreshCache)
except:
print("691")
protocol = componentImport(deepcopy(activity),
deepcopy(protocol),
user,
refreshCache=True)
newActivities = list(
set(protocol.get('activities', {}).keys()) -
activitiesNow)
newItems = list(
set(protocol.get('items', {}).keys()) - itemsNow)
formatted = _fixUpFormat(protocol)
createCache(obj, formatted, 'protocol')
return formatted
else:
return (_fixUpFormat(newObj))
except:
if refreshCache == False:
return (_fixUpFormat(
formatLdObject(obj,
mesoPrefix,
user,
keepUndefined,
dropErrors,
refreshCache=True,
responseDates=responseDates)))
import sys, traceback
print(sys.exc_info())
print(traceback.print_tb(sys.exc_info()[2]))
def loadFromSingleFile(document, user):
if 'protocol' not in document or 'data' not in document['protocol']:
raise ValidationException(
'should contain protocol field in the json file.', )
if 'activities' not in document['protocol']:
raise ValidationException(
'should contain activities field in the json file.', )
contexts = document.get('contexts', {})
protocol = {'protocol': {}, 'activity': {}, 'screen': {}}
expandedProtocol = expandObj(contexts, document['protocol']['data'])
protocol['protocol'][expandedProtocol['@id']] = {
'expanded': expandedProtocol
}
protocolId = None
for activity in document['protocol']['activities'].values():
expandedActivity = expandObj(contexts, activity['data'])
protocol['activity'][expandedActivity['@id']] = {
'parentKey': 'protocol',
'parentId': expandedProtocol['@id'],
'expanded': expandedActivity
}
if 'items' not in activity:
raise ValidationException(
'should contain at least one item in each activity.', )
for item in activity['items'].values():
expandedItem = expandObj(contexts, item)
protocol['screen'][expandedItem['@id']] = {
'parentKey': 'activity',
'parentId': expandedActivity['@id'],
'expanded': expandedItem
}
for modelType in ['protocol', 'activity', 'screen']:
modelClass = MODELS()[modelType]()
docCollection = getModelCollection(modelType)
for model in protocol[modelType].values():
prefName = modelClass.preferredName(model['expanded'])
if modelClass.name in ['folder', 'item']:
docFolder = FolderModel().createFolder(
name=prefName,
parent=docCollection,
parentType='collection',
public=True,
creator=user,
allowRename=True,
reuseExisting=(modelType != 'applet'))
metadata = {modelType: model['expanded']}
tmp = model
while tmp.get('parentId', None):
key = tmp['parentKey']
tmp = protocol[key][tmp['parentId']]
metadata['{}Id'.format(key)] = '{}/{}'.format(
MODELS()[key]().name, tmp['_id'])
if modelClass.name == 'folder':
newModel = modelClass.setMetadata(docFolder, metadata)
elif modelClass.name == 'item':
newModel = modelClass.setMetadata(
modelClass.createItem(
name=prefName if prefName else str(
len(
list(FolderModel().childItems(
FolderModel().load(docFolder,
level=None,
user=user,
force=True)))) + 1),
creator=user,
folder=docFolder,
reuseExisting=True), metadata)
# we don't need this in the future because we will only using single-file-format
modelClass.update({'_id': newModel['_id']},
{'$set': {
'loadedFromSingleFile': True
}})
if modelType != 'protocol':
formatted = _fixUpFormat(
formatLdObject(newModel,
mesoPrefix=modelType,
user=user,
refreshCache=False))
createCache(newModel, formatted, modelType, user)
model['_id'] = newModel['_id']
if modelType == 'protocol':
protocolId = newModel['_id']
return formatLdObject(ProtocolModel().load(protocolId, force=True),
mesoPrefix='protocol',
user=user,
refreshCache=False)