def setUp(self):
self.pool = getPool()
dbfile = DvPath(conn["dbName"])
if not dbfile.IsFile():
dbfile.CreateDirectories()
self.checkdb()
self.connect()
def myapp(modules=None):
a = ApplicationBase()
a.Register(appconf)
a.Register(dbconfMySql)
if modules:
for m in modules:
a.Register(m)
p = Portal()
p.Register(a, "nive")
a.LoadConfiguration()
root = DvPath(a.dbConfiguration.fileRoot)
if not root.IsDirectory():
root.CreateDirectories()
try:
a.Query("select id from pool_meta where id=1")
a.Query("select id from data1 where id=1")
a.Query("select id from data2 where id=1")
a.Query("select id from data3 where id=1")
a.Query("select id from pool_files where id=1")
a.Query("select id from pool_sys where id=1")
a.Query("select id from pool_groups where id=1")
except:
a.GetTool("nive.components.tools.dbStructureUpdater")()
a.Startup(None)
return a
def setUp(self):
dbfile = DvPath(conn["dbName"])
if not dbfile.IsFile():
dbfile.CreateDirectories()
checkdb()
db = Sqlite3(conf, conn)
db.GetPoolStructureObj().SetStructure(struct)
self.pool = db
def test_load(self, **kw):
p = DvPath(__file__)
p.SetNameExtension("app.json")
i, c = ResolveConfiguration(str(p))
self.assert_(c)
p.SetNameExtension("db.json")
c = LoadConfiguration(str(p))
self.assert_(c)
def exists(self):
"""
check if the file physically exists
"""
if not self.path:
return True
path = DvPath(self._Path())
return path.Exists()
def setUp(self):
conn = DatabaseConf(DB_CONF)
p = Sqlite3(connParam=conn, **test_Base.conf)
p.structure.Init(structure=test_Base.struct, stdMeta=test_Base.struct[u"pool_meta"])
dbfile = DvPath(conn["dbName"])
if not dbfile.IsFile():
dbfile.CreateDirectories()
self.pool = p
self.pool.connection.connect()
def InitFileStorage(self, root, connectionParam):
"""
Set the local root path for files
"""
self.root = DvPath()
self.root.SetStr(root)
if root == u"":
return
self.root.AppendSeperator()
self.root.CreateDirectoriesExcp()
def test_Set(self):
n = self.name
p = DvPath(n)
self.assert_(str(p) == n)
p.SetName("new")
self.assert_(str(p) == self.base + "new.txt", str(p))
p.SetExtension("png")
self.assert_(str(p) == self.base + "new.png")
p.SetNameExtension("another.txt")
self.assert_(str(p) == self.base + "another.txt")
def _Convert(self, profile):
source = self.files.get(profile.source)
if not source or not source.tempfile:
# convert only if tempfile
return False, ()
if not source:
return False, [_(u"Image not found: ") + profile.source]
p = DvPath()
p.SetUniqueTempFileName()
p.SetExtension(profile.extension)
destPath = str(p)
try:
try:
source.file.seek(0)
except:
pass
try:
iObj = Image.open(source)
except IOError:
# no file to be converted
return False, ()
iObj = iObj.convert("RGB")
# resize
size = [profile.width, profile.height]
if size[0] != 0 or size[1] != 0:
if size[0] == 0:
size[0] = size[1]
elif size[1] == 0:
size[1] = size[0]
x, y = iObj.size
if x > size[0]: y = y * size[0] / x; x = size[0]
if y > size[1]: x = x * size[1] / y; y = size[1]
size = x, y
iObj = iObj.resize(size, Image.ANTIALIAS)
iObj.save(destPath, profile.format)
try:
source.file.seek(0)
except:
pass
# file meta data
imgFile = open(destPath)
filename = DvPath(profile.dest+"_"+source.filename)
filename.SetExtension(profile.extension)
file = File(filekey=profile.dest,
filename=str(filename),
file=imgFile,
size=p.GetSize(),
path=destPath,
extension=profile.extension,
tempfile=True)
self.files.set(profile.dest, file)
finally:
# clean temp file
p.Delete()
return True, []
def delete(self):
if not self.path:
return True
originalPath = DvPath(self._Path())
if not originalPath.IsFile():
#not a file
return True
if originalPath.Exists() and not self.fileentry().pool._MoveToTrashcan(originalPath, self.fileentry().id):
#Delete failed!
return False
return True
def fromPath(self, path):
"""
Set temp file and load file values from file path
"""
if self.filename == "":
self.filename = os.path.basename(path)
f, fileExtension = os.path.splitext(path)
self.extension = fileExtension[1:6]
if self.size == 0:
p = DvPath(path)
self.size = p.GetSize()
self.tempfile = True
self.file = open(path, "rb")
def test_Dirs(self):
n = self.base
p = DvPath(n)
self.assert_(str(p) == n)
p.Append("another_dir" + os.sep + "and.file")
self.assert_(str(p) == n + "another_dir" + os.sep + "and.file", str(p))
p = DvPath(n)
p.AppendDirectory("the_last")
self.assert_(str(p) == n + "the_last" + os.sep, str(p))
p.RemoveLastDirectory()
self.assert_(str(p) == n)
p = DvPath(n[:-1])
p.AppendSeperator()
self.assert_(str(p) == self.base)
def Cleanup(self, files):
"""
Cleanup tempfiles after succesful writes
"""
if not isinstance(files, dict):
files = {"":files}
for key, file in files.items():
if not hasattr(file, "deleteOnSuccess"):
continue
path = DvPath(file.deleteOnSuccess)
try:
path.Delete()
except:
pass
def app_db(modules=None):
a = ApplicationBase()
a.Register(appconf)
if modules:
for m in modules:
a.Register(m)
p = Portal()
p.Register(a, "nive")
a.SetupApplication()
dbfile = DvPath(a.dbConfiguration.dbName)
if not dbfile.IsFile():
dbfile.CreateDirectories()
root = DvPath(a.dbConfiguration.fileRoot)
if not root.IsDirectory():
root.CreateDirectories()
try:
a.Query("select id from pool_meta where id=1")
a.Query("select id from data1 where id=1")
a.Query("select id from data2 where id=1")
a.Query("select id from data3 where id=1")
a.Query("select id from pool_files where id=1")
a.Query("select id from pool_sys where id=1")
a.Query("select id from pool_groups where id=1")
a.Query("select title from pool_meta where id=1")
except:
a.GetTool("nive.tools.dbStructureUpdater")()
# disable this to update test tables each time the tests are called
#a.GetTool("nive.tools.dbStructureUpdater")()
a.Startup(None)
# this will reset all testdata
#emptypool(a)
return a
def _GetTrashcanDirectory(self, id):
aP = DvPath()
aP.SetStr(str(self.root))
aP.AppendSeperator()
aP.AppendDirectory(self.Trashcan)
aP.AppendSeperator()
aP.AppendDirectory(self._GetDirectory(id))
aP.AppendSeperator()
return aP
def _CreatePath(self, key, filename):
"""
Create the physical path of the file
"""
root = str(self.fileentry().pool.root)
aP = DvPath(root)
aP.AppendSeperator()
aP.AppendDirectory(self.fileentry().pool._GetDirectory(self.fileentry().id))
aP.AppendSeperator()
aP.SetName(u"%06d_%s_" % (self.fileentry().id, key))
aP.SetExtension(DvPath(filename).GetExtension())
return aP.GetStr()
def test_Create(self):
n = self.name
p = DvPath(n)
self.assert_(p._path == n)
self.assert_(str(p) == n)
p = DvPath(DvPath(n))
self.assert_(str(p) == n)
p.SetStr(n)
self.assert_(str(p) == n)
self.assert_(p.GetStr() == n)
def app_db(confs=None):
a = DataStorage()
a.Register(appconf)
if confs:
for c in confs:
a.Register(c)
p = Portal()
p.Register(a)
a.Startup(None)
dbfile = DvPath(a.dbConfiguration.dbName)
if not dbfile.IsFile():
dbfile.CreateDirectories()
try:
a.Query("select id from pool_meta where id=1")
a.Query("select pool_wfp from pool_meta where id=1")
a.Query("select id from bookmarks where id=1")
a.Query("select id from tracks where id=1")
a.Query("select id from pool_files where id=1")
except:
a.GetTool("nive.tools.dbStructureUpdater")()
return a
def __test():
dbfile = DvPath(conn["dbName"])
if not dbfile.IsFile():
dbfile.CreateDirectories()
checkdb()
db = Sqlite3(conf, conn)
db.GetPoolStructureObj().SetStructure(struct)
app = Sqlite3ConnMultithreading()
app.pool = db
app.setUp()
threadcnt = 10
ts = []
for i in range(threadcnt):
t = ThreadClass()
ts.append(t)
t.app = app
t.start()
for t in ts:
t.join()
print "OK"
def app(confs=[]):
a = WebsitePublisher()
a.Register(appconf)
a.Register(dbconf)
for c in confs:
a.Register(c)
p = Portal()
p.Register(a)
a.Startup(None)
dbfile = DvPath(a.dbConfiguration.dbName)
if not dbfile.IsFile():
dbfile.CreateDirectories()
try:
a.Query("select id from pool_meta where id=1")
a.Query("select id from data1 where id=1")
a.Query("select id from data2 where id=1")
a.Query("select id from data3 where id=1")
a.Query("select id from pool_files where id=1")
except:
a.GetTool("nive.components.tools.dbStructureUpdater")()
return a
def app(extmodules=None):
appconf = AppConf("nive.userdb.app")
appconf.modules.append("nive.userdb.userview.view")
appconf.modules.append("nive.components.tools.sendMail")
a = UserDB(appconf)
a.dbConfiguration = dbconf
p = Portal()
p.Register(a)
a.Startup(None)
dbfile = DvPath(a.dbConfiguration.dbName)
if not dbfile.IsFile():
dbfile.CreateDirectories()
try:
a.Query("select id from pool_meta where id=1")
a.Query("select id from data1 where id=1")
a.Query("select id from data2 where id=1")
a.Query("select id from data3 where id=1")
a.Query("select id from pool_files where id=1")
except:
a.GetTool("nive.components.tools.dbStructureUpdater")()
return a
def app_db(confs=None):
appconf = AppConf("nive_cms.app")
a = WebsitePublisher()
a.Register(appconf)
if confs:
for c in confs:
a.Register(c)
p = Portal()
p.Register(a)
a.Startup(None)
dbfile = DvPath(a.dbConfiguration.dbName)
if not dbfile.IsFile():
dbfile.CreateDirectories()
try:
a.Query("select id from pool_meta where id=1")
a.Query("select id from box where id=1")
a.Query("select id from columnbox where id=1")
a.Query("select id from texts where id=1")
a.Query("select id from pool_files where id=1")
except:
a.GetTool("nive.tools.dbStructureUpdater")()
return a
def app_db(confs=None):
appconf = AppConf("nive_userdb.app")
appconf.modules.append("nive_userdb.userview.view")
appconf.modules.append("nive.tools.sendMailTester")
a = UserDB()
a.Register(appconf)
if confs:
for c in confs:
a.Register(c)
p = Portal()
p.Register(a)
a.Startup(None)
dbfile = DvPath(a.dbConfiguration.dbName)
if not dbfile.IsFile():
dbfile.CreateDirectories()
try:
a.Query("select id from pool_meta where id=1")
a.Query("select id from users where id=1")
a.Query("select id from users where token='1'")
a.Query("select id from users where tempcache='1'")
a.Query("select id from pool_files where id=1")
except:
a.GetTool("nive.tools.dbStructureUpdater")()
return a
def _Path(self, absolute = True):
"""
Get the physical path of the file. Checks the database.
"""
if self.tempfile or not self.path:
return u""
root = str(self.fileentry().pool.root)
if absolute and self.path[:len(root)] != root:
path = DvPath(root)
path.AppendSeperator()
path.Append(self.path)
else:
path = DvPath(self.path)
return path.GetStr()
def setUp(self):
p = DvPath(dbpath)
p.Delete()
self.db = Sqlite3Manager()
self.db.Connect(dbpath)
pass
class FileManager(object):
"""
Data Pool File Manager class for SQL Database with version support.
Files are stored in filesystem, aditional information in database table.
Table "pool_files" ("id", "fileid", "filekey", "path", "filename", "size", "extension", "version").
Field path stores internal path to the file in filesystem without root.
Preperty descriptions are dictionaries with key:value pairs.
Property values:
id = unit id to store file for (id is required)
version = the version of the file
filekey = custom value
key:
id_filekey_version
directory structure:
root/id[-4:-2]00/id_filekey_version.ext
"""
DirectoryCnt = -4 # directory id range limit
FileTable = u"pool_files" # file table name
FileTableFields = (u"id", u"fileid", u"filekey", u"path", u"filename", u"size", u"extension", u"version")
Trashcan = u"_trashcan"
def GetFileClass(self):
"""
Returns the required file class for File object instantiation
"""
return File
def InitFileStorage(self, root, connectionParam):
"""
Set the local root path for files
"""
self.root = DvPath()
self.root.SetStr(root)
if root == u"":
return
self.root.AppendSeperator()
self.root.CreateDirectoriesExcp()
def SearchFilename(self, filename):
"""
search for filename
"""
return self.SearchFiles({u"filename": filename})
def SearchFiles(self, parameter, sort=u"filename", start=0, max=100, ascending = 1, **kw):
"""
search files
"""
flds = self.FileTableFields
kw["singleTable"] = 1
sql, values = self.FmtSQLSelect(flds, parameter, dataTable=self.FileTable, sort = sort, start=start, max=max, ascending = ascending, **kw)
files = self.Query(sql, values)
f2 = []
for f in files:
f2.append(self.ConvertRecToDict(f, flds))
return f2
def DeleteFiles(self, id, cursor=None, version=None):
"""
Delete the file with the prop description
"""
files = self.SearchFiles({u"id":id}, sort=u"id")
if not files:
return True
entry = self.GetEntry(id, version=version)
for f in files:
file = self.GetFileClass()(filedict=f,fileentry=entry)
file.delete()
if len(files):
sql = u"delete from %s where id = %d" % (self.FileTable, id)
self.Query(sql, cursor=cursor, getResult=False)
return True
# Internal --------------------------------------------------------------
def _GetDirectory(self, id):
"""
construct directory path without root
"""
return (u"%06d" % (id))[self.DirectoryCnt:-2] + u"00/" + (u"%06d" % (id))[self.DirectoryCnt+2:]
def _MoveToTrashcan(self, path, id):
if not self.useTrashcan:
return path.Delete()
aP = self._GetTrashcanDirectory(id)
aP.SetNameExtension(path.GetNameExtension())
if aP.Exists():
aP.Delete()
return path.Rename(str(aP))
def _GetTrashcanDirectory(self, id):
aP = DvPath()
aP.SetStr(str(self.root))
aP.AppendSeperator()
aP.AppendDirectory(self.Trashcan)
aP.AppendSeperator()
aP.AppendDirectory(self._GetDirectory(id))
aP.AppendSeperator()
return aP
def tearDown(self):
self.db.Close()
p = DvPath(dbpath)
p.Delete()
pass
def test_Get(self):
n = self.name
p = DvPath(n)
self.assert_(p.GetPath() == self.base)
self.assert_(p.GetName() == "nofile")
self.assert_(p.GetExtension() == "txt")
self.assert_(p.GetNameExtension() == "nofile.txt")
self.assert_(p.GetSize() == -1)
self.assert_(p.GetLastDirectory() == "tmp_nivepathtest_000")
p.IsFile()
p.IsDirectory()
p.Exists()
p.IsValid()
def test_tempfilename(self):
p = DvPath("file.txt")
p.SetUniqueTempFileName()
self.assert_(p.GetExtension() == "txt", p.GetExtension())
def test_fncs(self):
temp = tempfile.gettempdir()
p = DvPath(temp)
p.AppendDirectory("tmp_nivepathtest_000")
p.Delete(deleteSubdirs=True)
p.CreateDirectories()
self.assert_(p.IsDirectory())
p.CreateDirectoriesExcp()
p.Rename("tmp_nivepathtest_111")
p.AppendDirectory("tmp_nivepathtest_000")
p.AppendDirectory("tmp_nivepathtest_000")
p.CreateDirectories()
self.assert_(p.IsDirectory())
p = DvPath(temp)
p.AppendDirectory("tmp_nivepathtest_000")
p.Delete(deleteSubdirs=False)
self.assert_(p.IsDirectory() == True)
p.Delete(deleteSubdirs=True)
self.assert_(p.IsDirectory() == False)