def dbToAbsPath(self, sDbPath):
sDbPath = pathNorm(sDbPath)
sLibPath = self.absPath()
sLibDmsPath = pathNorm(self.dbPath())
#print '^' + sLibDmsPath, sLibPath, sDbPath
sAbsPath = re.sub('^' + sLibDmsPath, sLibPath, sDbPath)
if sDbPath == sAbsPath:
raise ValueError("{} could not convert damas path to absolute: '{}'"
.format(self, sDbPath))
return sAbsPath
def scan(proj, sDbPath):
sDbPath = pathNorm(sDbPath)
library = proj.libraryFromDbPath(sDbPath)
sCurSite = proj.getCurrentSite()
dbNodeDct = proj._db.findNodes({"file":{"$regex":"^{}".format(addEndSlash(sDbPath))},
"source_size":{"$exists":True}}, asDict=True)
sortKey = lambda n: n._data["#parent"] + "_" + n.name
dbVersIter = (n for n in dbNodeDct.itervalues() if ("#parent" in n._data))
dbVersList = sorted(dbVersIter, key=sortKey, reverse=True)
grpIter = groupby(dbVersList, key=lambda n: n._data["#parent"])
#for k, g in grpIter:
# vn = next(g)
# hn = dbNodeDct.get(vn.getField("#parent"))
# if int(vn.version) != int(hn.version):
# print vn.version, hn.version, hn.file
dbNodeList = tuple(next(g) for _, g in grpIter)
numNodes = len(dbNodeList)
foundList = numNodes * [None]
errorList = []
for i, n in enumerate(dbNodeList):
print "checking {}/{}: {}".format(i + 1, numNodes, n.file)
rcEntry = library.entryFromDbPath(n.file, dbNode=False, assertLibrary=False)
if not rcEntry:
continue
rcEntry._cacheDbNode(n)
if rcEntry.fileSize != rcEntry.sourceSize:
foundList[i] = rcEntry
continue
try:
n = dbNodeDct[n.getField("#parent")]
except KeyError as e:
logMsg(e.message, warning=True)
errorList.append((rcEntry, e))
continue
rcEntry = library.entryFromDbPath(n.file, dbNode=False, assertLibrary=False)
if not rcEntry:
sOrigin = n.origin
if sOrigin and (sOrigin in n._data):
if ("synced_" + sOrigin in n._data):
rcEntry = library.entryFromDbPath(n.file, weak=True, dbNode=False, assertLibrary=False)
errorList.append((rcEntry, EnvironmentError("missing")))
continue
rcEntry._cacheDbNode(n)
if rcEntry.fileSize != rcEntry.sourceSize:
foundList[i] = rcEntry
return foundList, errorList, sDbPath
def createTree(self, pathData, rootPath=""):
pathItems = tuple(((pathNorm(d.pop("path")), d) for d in pathData))
tree = OrderedTree.fromPaths(t[0] for t in pathItems)
self.createItems(self, tree, dict(pathItems), rootPath=rootPath)
self.resizeAllColumns()
def libraryFromPath(self, sAbsPath, space="", anyUser=False):
sAbsPath = pathNorm(sAbsPath)
if space:
rcLibIter = (l for l in self.loadedLibraries.itervalues()
if (l.space == space))
else:
rcLibIter = self.loadedLibraries.itervalues()
for rcLib in rcLibIter:
if rcLib.contains(sAbsPath):
return rcLib
if (space != "public") and anyUser:
for sSpace, sLibSection in self._iterLibrariesSpaceAndSection(space="private"):
sRawLibPath = self.getPath(sSpace, sLibSection, resVars=False)
parseRes = pathParse(sRawLibPath, sAbsPath)
if parseRes and parseRes.named:
sOwner = parseRes.named["user_name"]
return self.getLibrary(sSpace, sLibSection, owner=sOwner, tokens=parseRes.named,
dbNode=False, weak=True, remember=True)
return None
def belowPack(p):
p = pathNorm(p)
if os.environ["IN_SEB_MODE"]:
return True if _belowPack(p) else _belowOldPack(p)
else:
return _belowPack(p)
def assertPack(p, dirStat=None):
if not dirStat:
dirStat = os.stat(pathNorm(p))
if isPack(p, fail=True, dirStat=dirStat):
return dirStat
return None
def createDirsAndFiles(self, sSpace="public", log=True, **kwargs):
bDryRun = kwargs.pop("dryRun", True)
sEntityName = self.name
cls = self.__class__
cls.assertNameParts(cls.getNameParts(sEntityName))
sTemplatePath = self.getTemplatePath()
if not sTemplatePath:
raise EnvironmentError("{} has NO template configured.".format(self))
return []
sDestPathList = []
bCheckIfExists = True
sEntityDirPath = self.getPath(sSpace)
if not osp.exists(sEntityDirPath):
bCheckIfExists = False
if not bDryRun:
os.makedirs(sEntityDirPath)
sDestPathList.append(sEntityDirPath)
sSrcPathIter = iterPaths(sTemplatePath, ignoreFiles=ignorePatterns("*.db", ".*"))
for sSrcPath in sSrcPathIter:
sDestPath = (sSrcPath.replace(sTemplatePath, sEntityDirPath)
.format(**vars(self)))
bExists = osp.exists(sDestPath) if bCheckIfExists else False
if not bExists:
sDestPathList.append(sDestPath)
if not bDryRun:
if sDestPath.endswith("/"):
os.makedirs(pathNorm(sDestPath))
else:
sDirPath = osp.dirname(sDestPath)
if not osp.exists(sDirPath):
os.makedirs(sDirPath)
copyFile(sSrcPath, sDestPath, dry_run=bDryRun)
if log and sDestPathList:
sAction = "Creating" if not bDryRun else "Missing"
sMsg = '\n{} {} paths for "{}":'.format(sAction, sSpace.upper(), sEntityName)
sMsg += "\n " + "\n ".join(sDestPathList)
print sMsg
return sDestPathList
def isEditableResource(self, sAbsPath, assertion=False):
p = pathNorm(sAbsPath)
sFileName = osp.basename(p)
if not (sFileName and osp.splitext(p)[1]):
if assertion:
raise EnvironmentError("Invalid filename: '{}'".format(sFileName))
return False
bPatternOk = False
sPatrnList = self.getVar("project", "editable_file_patterns", ())
for sPatrn in sPatrnList:
if fnmatch(sFileName, sPatrn):
bPatternOk = True
break
if not bPatternOk:
if assertion:
raise EnvironmentError("Not in project's editable files: {}."
.format(" ".join(sPatrnList)))
return False
drcEntry = self.entryFromPath(sAbsPath, dbNode=False)
if drcEntry and drcEntry.isFile():
if drcEntry.parentDir().freeToPublish():
return True
ctxData = self.contextFromPath(p)
sSection = ctxData.get("section")
sRcName = ctxData.get("resource")
if not sRcName:
if assertion:
raise EnvironmentError("Not a configured resource.")
return False
bEditable = self.getRcParam(sSection, sRcName, "editable", default=True)
if not bEditable:
if assertion:
raise EnvironmentError("Resource configured as non-editable: '{}.{}' ."
.format(sSection, sRcName))
return False
return True
def writePackContent(sPackDirPath, dirStat=None):
sPackDirPath = pathNorm(sPackDirPath)
if not dirStat:
dirStat = os.stat(sPackDirPath)
sJsonPath = mkPackFilePath(sPackDirPath)
iMtime = 0
if not osp.exists(sJsonPath):
iMtime = dirStat.st_mtime
iAtime = dirStat.st_atime
try:
open(sJsonPath, 'a+b').close() # create json file so it is listed by parseDirContent()
dirContent = parseDirContent(sPackDirPath)
jsonWrite(sJsonPath, dirContent, sort_keys=True)
finally:
if iMtime:
os.utime(sPackDirPath, (iAtime, iMtime))
return dirContent
def isPack(p, fail=False, dirStat=None):
p = pathNorm(p)
if os.environ["IN_SEB_MODE"]:
bPackPath = True if _isPack(p) else _isOldPack(p)
else:
bPackPath = _isPack(p)
if not bPackPath:
if fail:
sMsg = ("Directory NOT a package (should start with 'pkg_' or 'lyr_'): '{}'."
.format(osp.basename(p)))
raise EnvironmentError(sMsg)
else:
return False
if dirStat and not isDirStat(dirStat):
if fail:
raise EnvironmentError("Package path NOT a directory: '{}'".format(p))
else:
return False
return True
def createDirsAndFiles(self, sSpace="public", **kwargs):
bDryRun = kwargs.pop("dryRun", True)
bCheckDb = kwargs.pop("checkDb", True)
bLog = kwargs.pop("log", True)
bLoadDbNodes = kwargs.pop("loadDbNodes", True)
sEntityName = self.name
library = self.getLibrary(sSpace)
cls = self.__class__
cls.assertNameParts(cls.getNameParts(sEntityName))
sTemplatePath = self.getTemplatePath()
if not sTemplatePath:
raise EnvironmentError("{} has NO template configured.".format(self))
sDestPathList = []
bNewEntity = False
sEntityDirPath = self.getPath(sSpace)
entityDir = library._weakDir(sEntityDirPath, dbNode=False)
if not osp.exists(sEntityDirPath):
dbNode = entityDir.loadDbNode()
if dbNode:
sMsg = """{}'s directory already exists in DB: \n{}"""
raise RuntimeError(sMsg.format(self, dbNode.dataRepr()))
bNewEntity = True
if not bDryRun:
os.makedirs(sEntityDirPath)
entityDir.refresh(simple=True, dbNode=False)
sDestPathList.append(sEntityDirPath)
if bCheckDb and bLoadDbNodes:
entityDir.loadChildDbNodes(recursive=True, noVersions=True)
ruledDirDct = OrderedDict()
newDirList = []
sSrcPathIter = iterPaths(sTemplatePath, ignoreFiles=ignorePatterns("*.db", ".*"))
for sSrcPath in sSrcPathIter:
sDestPath = pathRedir(sSrcPath, sTemplatePath, sEntityDirPath).format(**vars(self))
bExists = False if bNewEntity else osp.exists(sDestPath)
if bExists:
continue
bIsDir = (sSrcPath.endswith("/") or sDestPath.endswith("/"))
if bCheckDb:
if bIsDir:
sEntryType = "Directory"
drcEntry = library._weakDir(sDestPath, dbNode=False)
else:
sEntryType = "File"
drcEntry = library._weakFile(sDestPath, dbNode=False)
dbNode = drcEntry.loadDbNode(fromDb=False)
if dbNode:
print "{} already exists in DB: '{}'".format(sEntryType,
drcEntry.dbPath())
continue
sDestPathList.append(sDestPath)
bDirCreated = True
if bIsDir:
sDirPath = pathNorm(sDestPath)
if not bDryRun:
os.makedirs(sDirPath)
else:
sDirPath = osp.dirname(sDestPath)
if not osp.exists(sDirPath):
if not bDryRun:
os.makedirs(sDirPath)
else:
bDirCreated = False
if not bDryRun:
copyFile(sSrcPath, sDestPath, dry_run=bDryRun)
if not bDryRun:
drcDir = library.getEntry(sDirPath, dbNode=False)
else:
drcDir = library._weakDir(sDirPath, dbNode=False)
if drcDir not in ruledDirDct:
sSyncRuleList = []
if bDirCreated:
sSyncRuleList = drcDir.getParam("default_sync_rules", None)
newDirList.append(drcDir)
else:
dbNode = drcDir.loadDbNode(fromDb=False)
if dbNode:
drcDir.refresh(simple=True, dbNode=False)
sSyncRuleList = drcDir.syncRules
if sSyncRuleList:
ruledDirDct[drcDir] = sSyncRuleList
newDbNodeList = []
for rcDir in newDirList:
initData = rcDir.getDbNodeInitData(syncRules=False)
newDbNodeList.append(initData)
if newDbNodeList and (not bDryRun):
newDbNodeList = self.project._db.createNodes(newDbNodeList)
for dbnode in newDbNodeList:
library._addDbNodeToCache(dbnode)
for rcDir in newDirList:
rcDir.refresh(simple=True, dbNode=False)
for drcDir, sSyncRuleList in reversed(ruledDirDct.items()):
#print drcDir, "default_sync_rules:", sSyncRuleList
if not bDryRun:
drcDir.setSyncRules(sSyncRuleList, applyRules=True, refresh=False)
if bLog and sDestPathList:
sSep = "\n "
sAction = "Created" if not bDryRun else "Missing"
sMsg = '\n{} {} paths for "{}":'.format(sAction, sSpace.upper(), sEntityName)
sMsg += sSep + sSep.join(sDestPathList)
print sMsg, "\n"
return sDestPathList
def getEntry(self, pathOrInfo, weak=False, drcType=None, dbNode=True):
logMsg(log="all")
"""
weak means that we do not check if the path exists.
"""
fileInfo = None
sRelPath = ""
if isinstance(pathOrInfo, QFileInfo):
sAbsPath = pathOrInfo.absoluteFilePath()
fileInfo = pathOrInfo
elif isinstance(pathOrInfo, basestring):
sAbsPath = pathNorm(pathOrInfo)
if not osp.isabs(sAbsPath):
sRelPath = sAbsPath
sAbsPath = self.relToAbsPath(sRelPath)
else:
raise TypeError("argument 'pathOrInfo' must be of type <QFileInfo> \
or <basestring>. Got {}.".format(type(pathOrInfo)))
# entries are cached using their relative path the the library they belong to.
if not sRelPath:
sRelPath = self.absToRelPath(sAbsPath) if osp.isabs(sAbsPath) else sAbsPath
# try to get an already loaded entry...
drcEntry = self._cachedEntries.get(normCase(sRelPath))
if drcEntry:
drcEntry.loadData(drcEntry._qfileinfo, dbNode=dbNode)
if weak:
return drcEntry
else:
return drcEntry if drcEntry.exists() else None
if not weak:
sIgnorePatterns = ("*.db", ".*")
sName = osp.basename(sAbsPath)
for sPattern in sIgnorePatterns:
if fnmatch(sName, sPattern):
return None
if not fileInfo:
fileInfo = toQFileInfo(sAbsPath)
cls = self.__class__
# no cached entry found so let's instance a new one
if weak:
if drcType:
rcCls = drcType
else:
rcCls = cls.classDir if sAbsPath.endswith('/') else cls.classFile
else:
if fileInfo.isDir():
rcCls = cls.classDir
elif fileInfo.isFile():
rcCls = cls.classFile
else:
return None
entry = rcCls(self, fileInfo, dbNode=dbNode)
return entry
def dbToRelPath(self, sDbPath):
sDbPath = pathNorm(sDbPath)
sRelPath = pathRelativeTo(sDbPath, pathNorm(self.dbPath()))
return sRelPath
def launch_old(entityType="", dryRun=True, project="", dialogParent=None):
global TREE_ITEM_DCT
TREE_ITEM_DCT = {}
app = qtGuiApp()
if not app:
app = QtGui.QApplication(sys.argv)
sProject = os.environ["DAVOS_INIT_PROJECT"] if not project else project
proj = DamProject(sProject)
print sProject.center(80, "-")
dlg = QuickTreeDialog(dialogParent)
treeWdg = dlg.treeWidget
treeWdg.setHeaderLabels(("Entity Name", "Infos"))
dlg.show()
missingPathItems = listMissingPathItems(proj, entityType)
badEntityItems = []
for damEntity, sMissingPaths in missingPathItems:
if isinstance(damEntity, basestring):
badEntityItems.append((damEntity, sMissingPaths))
continue
drcLib = proj.getLibrary("public", damEntity.libraryName)
sLibPath = drcLib.absPath()
sEntityTitle = damEntity.sgEntityType + 's'
sEntityPath = damEntity.getPath("public")
sEntityPath = re.sub("^" + sLibPath, sEntityTitle, sEntityPath)
sEntityPathDirs = pathSplitDirs(sEntityPath)
for sAbsPath in sMissingPaths:
sTreePath = re.sub("^" + sLibPath, sEntityTitle, sAbsPath)
sParentPath, sFilename = osp.split(pathNorm(sTreePath))
parentItem = TREE_ITEM_DCT.get(sParentPath)
if not parentItem:
sDirList = pathSplitDirs(sParentPath)
curParentItem = treeWdg
for i, sDirName in enumerate(sDirList):
if i == 0:
sItemPath = sDirName
else:
sItemPath = pathJoin(*sDirList[:i + 1])
item = TREE_ITEM_DCT.get(sItemPath)
if not item:
flags = None
if sItemPath.startswith(sEntityPath):
if len(pathSplitDirs(sItemPath)) > len(sEntityPathDirs):
flags = Qt.NoItemFlags
userData = None
if sItemPath == sEntityPath:
userData = damEntity
item = loadTreeItem(curParentItem, sItemPath, [sDirName],
flags=flags, userData=userData)
curParentItem = item
parentItem = curParentItem
flags = None
if sTreePath.startswith(sEntityPath):
if len(pathSplitDirs(sTreePath)) > len(sEntityPathDirs):
flags = Qt.NoItemFlags
userData = None
if sTreePath == sEntityPath:
userData = damEntity
loadTreeItem(parentItem, sTreePath, [sFilename], flags=flags, userData=userData)
if badEntityItems:
errorsItem = loadTreeItem(None, "Errors", ["ERRORS"])
treeWdg.insertTopLevelItem(0, errorsItem)
for sEntityName, sError in badEntityItems:
loadTreeItem(errorsItem, sEntityName, [sEntityName, sError],
checkable=False)
for i in xrange(treeWdg.topLevelItemCount()):
treeWdg.topLevelItem(i).setExpanded(True)
while True:
bOk = dlg.exec_()
if not bOk:
return
bApply = False
flags = (QTreeWidgetItemIterator.Checked | QTreeWidgetItemIterator.Enabled)
treeIter = QTreeWidgetItemIterator(treeWdg, flags)
damEntities = tuple(it.value().data(0, Qt.UserRole) for it in treeIter)
damAssets = tuple(e for e in damEntities if isinstance(e, DamAsset))
damShots = tuple(e for e in damEntities if isinstance(e, DamShot))
if damAssets or damShots:
sMsg = "Create directories and files for:\n"
if damAssets:
sMsg += "\n - {} Assets".format(len(damAssets))
if damShots:
sMsg += "\n - {} Shots".format(len(damShots))
sConfirm = confirmDialog(title="WARNING !",
message=sMsg,
button=("Yes", "No"),
defaultButton="No",
cancelButton="No",
dismissString="No",
icon="warning",
)
if sConfirm == "Yes":
bApply = True
break
if bApply:
for damEntity in damEntities:
if not damEntity:
continue
damEntity.createDirsAndFiles(dryRun=dryRun)
def getPath(self, sSpace, sSection, pathVar="", tokens=None, default="NoEntry", **kwargs):
bResVars = kwargs.get("resVars", True)
bResEnvs = kwargs.get("resEnvs", True)
sRcPath = ""
if sSpace:
sRcPath = self.getVar(sSection, sSpace + "_path", default=default,
resVars=bResVars)
if not sRcPath:
return sRcPath
if pathVar:
try:
sRcPath = pathJoin(sRcPath, self.getVar(sSection, pathVar))
except AttributeError:
if default != "NoEntry":
return default
raise
if bResEnvs:
sRcPath = pathResolve(sRcPath)
sFieldSet = set()
if bResVars:
# resolve vars from config
sFieldSet = set(findFmtFields(sRcPath))
if sFieldSet:
confTokens = self.getVar(sSection, pathVar + "_tokens", default={})
sConfFieldSet = set(confTokens.iterkeys())
for sField in sFieldSet:
if sField in confTokens:
continue
value = self.getVar(sSection, sField, "")
if value:
sConfFieldSet.add(sField)
else:
value = '{' + sField + '}'
confTokens[sField] = value
if confTokens:
sRcPath = sRcPath.format(**confTokens)
sFieldSet -= sConfFieldSet
# resolve remaining vars from input tokens
if tokens:
if not isinstance(tokens, dict):
raise TypeError("argument 'tokens' must be of type <dict>. Got {}"
.format(type(tokens)))
sFieldSet = sFieldSet - set(tokens.iterkeys())
if sFieldSet:
msg = ("Cannot resolve path: '{}'. \n\tMissing tokens: {}"
.format(sRcPath, list(sFieldSet)))
raise RuntimeError(msg)
sRcPath = sRcPath.format(**tokens)
return pathNorm(sRcPath, keepEndSlash=True)