def setDataSourceDataStore(dataSource,
dataUrlPath,
localPath,
dataLocationStore=None,
dataUrl=None):
#
# Get DataLocationStore
#
if not dataLocationStore:
setCurrentStore(dataSource.root, 'DataLocationStore')
dataLocationStore = dataSource.root.currentDataLocationStore
#
# Get (or create) DataUrl
#
# TODO should this search function go elsewhere?
if not dataUrl:
for tmpDataUrl in dataLocationStore.dataUrls:
if tmpDataUrl.url.dataLocation == dataUrlPath:
dataUrl = tmpDataUrl
if not dataUrl:
dataUrlPath = uniIo.normalisePath(dataUrlPath)
dataUrl = dataLocationStore.newDataUrl(url=Implementation.Url(
path=dataUrlPath))
#
# Create a BlockedBinaryMatrix. TODO: could be other classes that are set up this way - rename func and make general,, pass in class?
#
localPath = uniIo.normalisePath(localPath)
blockedBinaryMatrix = dataLocationStore.newBlockedBinaryMatrix(
path=localPath, dataUrl=dataUrl)
"""
TODO Set here as well, or do this later after returning object:
blockSizes Int 0..* Block sizes in dimension order
complexStoredBy ComplexStorage 1..1 The ordering of real and imaginary parts of hypercomplex numbers in the data matrix. See ComplexStorage type for details
hasBlockPadding Boolean 1..1 Are data padded to fill all blocks completely? Alternatively incomplete blocks store only the actual data.
headerSize Int 1..1 Header size in bytes
isBigEndian Boolean 1..1 Are data big-endian (alternative little-endian).
isComplex Boolean 0..* Are numbers complex (if True) or real/integer (if False).
nByte Int 1..1 Number of bytes per number
numPoints Int 0..* number of points for each matrix dimension - also defines dimensionality of matrix. The number of points is the same for real or complex data, in the sense that n complex points require 2n real numbers for storage.
numRecords Int 1..1 Number of matrix records in file. All other information in the object describes a single record.
numberType NumberType 1..1 Type of numbers held in matrix
"""
dataSource.dataStore = blockedBinaryMatrix
return blockedBinaryMatrix
def configParameter(keyWord):
"""Get configuration parameter corresponding to 'keyWord'.
NBNB TBD this should go to a configuration file to get its data
The format of this file is still up for grabs.
"""
rootDir = uniIo.normalisePath(uniIo.os.getcwd())
if keyWord == 'repositories':
# list of (name,urlPath) pairs
return []
elif keyWord == 'packageLocators':
# list of (packageName, repositoryName) Triplets
# If the packageName appears more than once, the named repositories
# are added in the order given
return [
('ccp.molecule.ChemComp', 'userData'),
('ccp.molecule.ChemComp', 'refData'),
('ccp.molecule.ChemCompCharge', 'refData'),
('ccp.molecule.ChemCompCoord', 'refData'),
('ccp.molecule.ChemCompLabel', 'userData'),
('ccp.molecule.ChemCompLabel', 'refData'),
('ccp.molecule.ChemElement', 'refData'),
('ccp.molecule.StereoChemistry', 'refData'),
('ccp.nmr.NmrExpPrototype', 'userData'),
('ccp.nmr.NmrExpPrototype', 'refData'),
('ccp.nmr.NmrReference', 'refData'),
('ccpnmr.AnalysisProfile', 'generalData'),
('ccpnmr.AnalysisProfile', 'refData'),
]
else:
return None
def fixImplementation(topObj, delayDataDict):
""" Add generalData repository, packageLocator for AnalysisProfile with
repositories link, and set
NB when this is called PAckageLocators and Repositories
have already had their child links and crosslinks set.
"""
from memops.universal import Io as uniIo
from memops.api.Implementation import Url
import os.path
emptyDict = {}
emptyList = []
doGet = delayDataDict.get
urlPath = uniIo.normalisePath(os.path.expanduser('~/.ccpn/data'))
repositories = doGet(topObj).get('repositories')
for refData in repositories:
if refData.name == 'refData':
break
else:
raise ApiError("refData repository not found")
genData = topObj.newRepository(name='generalData', url=Url(path=urlPath))
topObj.__dict__['repositories']['generalData'] = genData
profileLocator = topObj.newPackageLocator(targetName='ccpnmr.AnalysisProfile',
repositories=(genData, refData))
topObj.__dict__['packageLocators']['ccpnmr.AnalysisProfile'] = profileLocator
def getDataStoringFromFilepath(memopsRoot,
fullFilePath,
preferDataUrls=None,
dataLocationStore=None,
keepDirectories=1):
# make absolute,, normalised path
fullFilePath = uniIo.normalisePath(fullFilePath, makeAbsolute=True)
dataUrl, filePath = findDataStoringFromFilepath(memopsRoot, fullFilePath,
preferDataUrls,
dataLocationStore,
keepDirectories)
if dataUrl is None:
urlPath = uniIo.normalisePath((fullFilePath[:-len(filePath)]))
dataLocationStore = memopsRoot.currentDataLocationStore
dataUrl = dataLocationStore.newDataUrl(url=Implementation.Url(
path=urlPath))
dataUrl.name = 'auto-%s' % dataUrl.serial
#
return (dataUrl, filePath)
def resetPackageLocators(project):
carbDataPath = uniIo.normalisePath(
os.path.join(os.path.abspath('..'), 'data'))
print 'Location of local carbohydrate ChemComps: [%s]' % carbDataPath
carbDataUrl = Implementation.Url(path=carbDataPath)
carbDataRepos = project.newRepository(name='carbData', url=carbDataUrl)
chemPackLoc = project.findFirstPackageLocator(
targetName='ccp.molecule.ChemComp')
chemPackLoc.addRepository(carbDataRepos)
def changeDataStoreUrl(dataStore, newPath):
""" Change the url for this dataStore, so at the end we have
dataStore.dataUrl.url.path = newPath. This changes all dataUrls
with the same old path if the old path does not exist and the
new one does.
"""
newPath = uniIo.normalisePath(newPath, makeAbsolute=True)
oldDataUrl = dataStore.dataUrl
oldUrl = oldDataUrl.url
oldPath = oldUrl.dataLocation
oldExists = os.path.exists(oldPath)
if newPath != oldPath:
dataLocationStore = dataStore.dataLocationStore
newUrl = Implementation.Url(
path=newPath) # TBD: should use oldUrl.clone(path=newPath)
# first check if have a dataUrl with this path
newDataUrl = dataLocationStore.findFirstDataUrl(url=newUrl)
if not newDataUrl:
# if old path exists and there is more than one dataStore with
# this dataUrl then create new one
dataUrlStores = dataLocationStore.findAllDataStores(
dataUrl=oldDataUrl)
if oldExists and len(dataUrlStores) > 1:
newDataUrl = dataLocationStore.newDataUrl(name=oldDataUrl.name,
url=newUrl)
# if have found or have created newDataUrl then set dataStore to point to it
# else just change url of oldDataUrl (which could affect other dataStores)
if newDataUrl:
dataStore.dataUrl = newDataUrl
else:
oldDataUrl.url = newUrl
# if old path does not exist and new path exists then change urls of
# all data urls which have old path to new path (there might be none)
if not oldExists:
newExists = os.path.exists(newPath)
if newExists:
for dataUrl in dataLocationStore.dataUrls:
if dataUrl.url == oldUrl:
dataUrl.url = newUrl
def loadProject(parent, path, projectName=None):
path = uniIo.normalisePath(path)
askdir = lambda title, prompt, initial_value: askDir(
title, prompt, initial_value, parent=parent, extra_dismiss_text='Skip')
askfile = lambda title, prompt, initial_value: askFile(
title, prompt, initial_value, parent=parent, extra_dismiss_text='Skip')
project = genIo.loadProject(path,
showWarning=showWarning,
askDir=askdir,
askFile=askfile)
# now check dataStores
# delete those that are not used
# and otherwise check path to see if exists
dataStores = []
for dataLocationStore in project.dataLocationStores:
for dataStore in dataLocationStore.dataStores:
if isinstance(dataStore,
NumericMatrix) and not dataStore.nmrDataSources:
print 'deleting dataStore %s with path %s' % (
dataStore, dataStore.fullPath)
dataStore.delete()
elif isinstance(
dataStore,
MimeTypeDataStore) and not dataStore.nmrDataSourceImages:
print 'deleting dataStore %s with path %s' % (
dataStore, dataStore.fullPath)
dataStore.delete()
else:
dataStores.append(dataStore)
badDataStores = [
dataStore for dataStore in dataStores
if not os.path.exists(dataStore.fullPath)
]
if badDataStores:
popup = DataLocationPopup(parent, project, modal=True)
popup.destroy()
return project
def majorUpgradeToCurrent(oldRoot,
oldVersionStr,
newName=None,
newDir=None,
oldTags=None,
doSave=True):
"""
Do major upgrade of oldRoot tree to current version.
NB oldRoot tree must be all loaded in memory
"""
if oldVersionStr == currentVersionStr:
return oldRoot
from memops.api import Implementation
# make new project and move old directory if necessary
oldName = oldRoot.name
if newName is None:
newName = oldName
newRoot = Implementation.MemopsRoot(name=newName)
if newDir is None:
newPath = getDataDir(oldRoot, oldVersionStr)
else:
newPath = uniIo.normalisePath((os.path.join(newDir, newName)),
makeAbsolute=True)
newRoot.packageLocator.findFirstRepository().url = (Implementation.Url(
path=newPath))
newRoot.findFirstRepository(name='backup').url = (Implementation.Url(
path=newPath + '_backup'))
upgrader = DataUpgrader(oldVersionStr, newRoot, oldRoot, oldTags=oldTags)
# make upgrade
return upgrader.majorUpgrade(doSave=doSave)
def findDataStoringFromFilepath(project,
fullFilePath,
preferDataUrls=None,
dataLocationStore=None,
keepDirectories=1):
""" Get DataUrl and relative filePath from normalised absolute filePath
Uses heuristics to select compatible DataUrl from existing ones.
sisterObjects is a collection of objects with a dataStore link -
DataUrls in use for sisterObjects are given preference in the
heuristics.
uses dataLocationStore or current dataLocationStore
If no compatible DataUrl is found the routine returns dataUrl None
and the file name plus the lowest keepDirectories directories
as the filePath
"""
# NB fullFilePath *must* be absolute her for code to work properly
#
if not os.path.isabs(fullFilePath):
raise ApiError(
"findDataStoringFromFilepath called with non-absolute file name %s"
% fullFilePath)
# get DataLocationStore
if not dataLocationStore:
setCurrentStore(project, 'DataLocationStore')
dataLocationStore = project.currentDataLocationStore
# get DataUrl that match fullFilePath
dataUrls = []
for dataUrl in dataLocationStore.dataUrls:
dirPath = uniIo.normalisePath(dataUrl.url.path)
if fullFilePath.startswith(dirPath):
lenPath = len(dirPath)
ss = fullFilePath
while len(ss) > lenPath:
ss, junk = uniIo.splitPath(ss)
if ss == dirPath:
# DataUrl path matches file path
dataUrls.append(dataUrl)
# process result
if dataUrls:
if preferDataUrls:
# look for DataUrls that are in use with related objects
ll = [x for x in dataUrls if x in preferDataUrls]
if ll:
dataUrls = ll
if len(dataUrls) == 1:
# only one DataUrl - use it
dataUrl = dataUrls[0]
else:
# use DataUrl with longest path
ll = [(len(dataUrl.url.path), dataUrl) for dataUrl in dataUrls]
ll.sort()
dataUrl = ll[-1][1]
# get filePath
ss = uniIo.joinPath(dataUrl.url.path,
'') # removes file separator from end
filePath = fullFilePath[len(ss) + 1:]
else:
dataUrl = None
ll = []
ss = fullFilePath
for dummy in range(keepDirectories + 1):
ss, name = os.path.split(ss)
ll.append(name)
ll.reverse()
filePath = uniIo.joinPath(*ll)
#
return (dataUrl, filePath)
if __name__ == '__main__':
# Load the old project - need to specify the correct directory name.
project = loadProject('topObjectTest')
print 'Project: [%s]\n' % project
# Find the right repository that contains the project data.
projectRepos = project.findFirstRepository(name='userData')
# Specify a new location to save the project in.
newPath = uniIo.normalisePath(
os.path.join(os.path.abspath('.'), 'newDir', 'newSubDir'))
# Make a new Url pointing to the new location and link it to the
# 'userData' repository.
projectRepos.url = Url(path=newPath)
# Print the location of all project repositories.
print 'Repositories:\n%s' % ('\n'.join(
repos.name + ': ' + repos.url.path
for repos in project.sortedRepositories()))
carbDataPath = uniIo.normalisePath(
os.path.join(os.path.abspath('..'), 'data'))
return True
if __name__ == '__main__':
from memops.format.xml import Util as xmlUtil
from memops.format.xml import XmlIO
from memops.universal import Io as uniIo
from memops.api import Implementation
import Tkinter
# standard project name
projectName = 'refIsotopeEditor'
# get or make project
repDir = uniIo.joinPath(uniIo.normalisePath(uniIo.os.getcwd()),
projectName)
projectFile = xmlUtil.getProjectFile(repDir, projectName=projectName)
if os.path.isfile(projectFile):
project = XmlIO.loadProjectFile(projectFile)
else:
project = Implementation.MemopsRoot(name=projectName)
# reorder to make sure data are saved in refData
labelingSchemeLocator = project.findFirstPackageLocator(
targetName='ccp.molecule.ChemCompLabel')
repositories = list(labelingSchemeLocator.repositories)
for rr in repositories:
if rr.name == 'refData':
refRepository = rr
break
def loadProject(parent, path, projectName=None):
path = uniIo.normalisePath(path)
askdir = lambda title, prompt, initial_value: askDir(
title, prompt, initial_value, parent=parent, extra_dismiss_text='Skip')
askfile = lambda title, prompt, initial_value: askFile(
title, prompt, initial_value, parent=parent, extra_dismiss_text='Skip')
project = genIo.loadProject(path,
showWarning=showWarning,
askDir=askdir,
askFile=askfile)
# now check dataStores
# delete those that are not used
# and otherwise check path to see if exists
dataStores = []
for dataLocationStore in project.dataLocationStores:
for dataStore in dataLocationStore.dataStores:
if isinstance(dataStore,
NumericMatrix) and not dataStore.nmrDataSources:
print 'deleting dataStore %s with path %s' % (
dataStore, dataStore.fullPath)
dataStore.delete()
elif isinstance(
dataStore,
MimeTypeDataStore) and not dataStore.nmrDataSourceImages:
print 'deleting dataStore %s with path %s' % (
dataStore, dataStore.fullPath)
dataStore.delete()
else:
dataStores.append(dataStore)
badDataStores = [
dataStore for dataStore in dataStores
if not os.path.exists(dataStore.fullPath)
]
if badDataStores:
# find DataUrls involved
dataUrls = set(dataStore.dataUrl for dataStore in badDataStores)
startDir = project.packageLocator.findFirstRepository(
).url.dataLocation
for dataUrl in dataUrls:
if not dataUrl.dataStores.difference(badDataStores):
# all DataStores for this DataUrl are bad
# we can make changes without affecting 'good' DataStores
# Look for an obvious place the data may have moved to
dataStores = dataUrl.sortedDataStores()
fullPaths = [dataStore.fullPath for dataStore in dataStores]
baseDir, newPaths = uniIo.suggestFileLocations(
fullPaths, startDir=startDir)
if baseDir is not None:
# We have a file location that fits all missing files.
# Change dataStores to use it
print 'WARNING, resetting data locations to: \n%s\n' % baseDir
ccpGenIo.changeDataStoreUrl(dataStores[0], baseDir)
for ii, dataStore in enumerate(dataStores):
dataStore.path = newPaths[ii]
if [
dataStore for dataStore in dataStores
if not os.path.exists(dataStore.fullPath)
]:
popup = DataLocationPopup(parent, project, modal=True)
popup.destroy()
return project
