def UpdateDatastore(self, id, share, errorOnExists=True):
sess = self.db.CreateSession()
try:
with model.AutoSession(sess):
ds = self.GetDatastore(sess, id)
log.info("Updating existing datastore [name:%s,id:%d]",
ds.name, ds.id)
# Ensure 'system' and 'internal' names remain unchanged.
if ds.name in (SYSTEM, INTERNAL) and ds.name != share.name:
raise DatastoreException(
'Changing the %s datastore name is not allowed.',
ds.name)
if ds.status != 'offline':
# XXX: Use a better exception.
raise DataStoreInUseError(
ds.name % " is not in the offline mode yet!")
if errorOnExists:
raise DatastoreExistsError(id)
# Just update the info in case it changed.
with model.AutoSession(sess):
updated = model.DataStore.FromShare(share)
updated.id = ds.id
sess.merge(updated)
except DataStoreNotFoundError:
log.debug('The %s[id=%d] datastore was not found', share.name, id)
raise DataStoreNotFoundError('The %s datastore was not found.',
share.name)
def Refresh(self, projectId, datastores):
sess = self.db.CreateSession()
with model.AutoSession(sess):
project = sess.query(model.Project).get(projectId)
datastore = datastores.GetDatastore(sess, project.datastore.id)
projectRoot = path(datastore.localPath) / project.subdir
lease = datastores.Acquire(datastore.id)
try:
# Always completely repopulate files since entry points
# may have been removed.
del project.files[:]
files = self.GetFiles(projectRoot)
self.RegisterFiles(project, files, sess)
finally:
datastores.Release(lease)
sess = self.db.CreateSession()
with model.AutoSession(sess):
# Have to do this in a new session otherwise registryIcon
# will fail because it's looking for some state that isn't
# flushed when it tries to read Package.ini settings.
project = sess.query(model.Project).get(projectId)
try:
self._registerIcon(project, projectRoot)
except Exception:
log.exception(
'Unable to register an icon for the project. Continuing.')
project.state = defs.Projects.AVAILABLE
def Acquire(self, dsId):
sess = self.db.CreateSession()
with model.AutoSession(sess):
try:
ds = sess.query(model.DataStore).filter_by(id=dsId).one()
if ds.name == SYSTEM:
raise DatastoreException(
'The system partition is read-only.')
share = ds.GetAsShare()
if ds.status != 'online':
raise DataStoreOfflineError
except ormexc.NoResultFound:
# raised by .one()
raise DataStoreNotFoundError(dsId)
# Note that share is not a SQLAlchemy object, so no need
# to keep the session open or expunge the object
lease = DataStoreLease(self.leaseNumber, dsId, share)
self.leases[dsId][lease.id] = lease
self.leaseNumber += 1
return lease
def Create(self, targetDatastoreId, runtimeId, datastores):
sess = self.db.CreateSession()
project = model.Project()
with model.AutoSession(sess):
project.datastore = datastores.GetDatastore(
sess, targetDatastoreId)
project.runtime_id = runtimeId
project.state = defs.Projects.CREATED
# subdir isn't nullable so set to a dummy value until we have
# the primary key after flushing below.
project.subdir = ''
sess.add(project)
sess.flush() # So that we have the id.
project.subdir = 'project-%d' % project.id
# Have to flush again for project.subdir to get set.
sess.flush()
lease = None
# Create the base directory for the project.
try:
ds = project.datastore
lease = datastores.Acquire(ds.id)
# Make sure that the datastore is up-to-date since it
# could have gone into the online state between looking
# the project up and acquiring the datastore lease.
sess.refresh(ds)
supportPath = path(ds.localPath) / project.subdir / 'Support'
logPath = supportPath / 'taf.log'
try:
supportPath.makedirs()
# to ensure the Support/appfactory.log is
# writeable by tomcat and readable by user
logPath.touch()
logPath.chmod(0664)
except OSError, e:
if e.errno != errno.EEXIST:
raise
finally:
if lease:
datastores.Release(lease)
# project has been expired since it was just committed.
# Therefore the next time one of its attributes is accessed
# it will need to hit the database. So go ahead and refresh
# it so that its values are already loaded for the caller to
# access.
sess.refresh(project)
# Remove it from the session since we just need to read
# values.
sess.expunge(project)
return project
def _processDelete(self, projectId, datastores):
sess = self.db.CreateSession()
with closing(sess):
lease = None
try:
project = sess.query(model.Project).get(projectId)
# Ensure it won't be unmounted from underneath us.
lease = datastores.Acquire(project.datastore.id)
projectDir = path(project.subdir)
assert not projectDir.isabs()
fullPath = path(lease.share.localPath) / projectDir
log.info('Deleting project %d from %s.', projectId, fullPath)
log.info('Removing read-only bits from project.')
# path.walk traverses symlinks which we don't want to do.
eventlet.tpool.execute(util.FixPermissions, fullPath)
log.info(
'Read-only bits removed from all files and directories.')
# XXX: Who knows what errors could happen here but we ignore
# them and just set the project state to deleted anyway. The
# client will have no idea something went wrong. Maybe it needs
# to go to a failure state so the client can retry?
try:
# XXX: If this fails the client will think they got
# deleted when they really didn't. Network might have
# gone out, etc.
# Also note that leading directories
# $localPath/path/to/project will not get cleaned up
# when they become empty.
# Must run in threadpool to prevent blocking greenthreads.
eventlet.tpool.execute(fullPath.rmtree)
log.info('Project %d deleted.', projectId)
except Exception:
log.exception('There was an error removing project %d.',
project.id)
with model.AutoSession(sess):
# NULL-out icon to avoid database from growing too
# large.
project.icon = None
project.state = defs.Projects.DELETED
finally:
if lease:
datastores.Release(lease)
def GetDataStoreList(self, sess=None):
if not sess:
sess = self.db.CreateSession()
with model.AutoSession(sess):
dsList = [
n[0]
for n in sess.execute(sqlalchemy.select([model.DataStore.id]))
]
return dsList
def DeleteDatastore(self, id):
sess = self.db.CreateSession()
with model.AutoSession(sess):
ds = self.GetDatastore(sess, id)
if ds.name in (SYSTEM, INTERNAL):
raise DatastoreException(
'Deleting the %s datastore is prohibited!', ds.name)
self.VerifyLeases(id)
self.GoOffline(id)
sess.delete(ds)
def CreateDirectory(self, projectId, createPath):
# Verify that parent exists in DB
dir, base = path(createPath).splitpath()
sess = self.db.CreateSession()
with model.AutoSession(sess):
project = sess.query(model.Project).get(projectId)
root = path(project.datastore.localPath) / project.subdir
sysPath = root / createPath
if sysPath.isdir():
raise Exception('Directory %s already exists for project' %
createPath)
elif '..' in sysPath:
raise Exception('Relative paths not allowed: %s' % createPath)
# Create the directory before touching the DB. That way, the second
# the database is flushed, there is no race to create the directory.
sysPath.mkdir()
try:
# We want to raise the exception directly if the parent is not
# found for whatever reason.
parent = self.GetProjectFileByPath(projectId,
dir,
directory=True,
session=sess)
for child in parent.children:
# Verify that a name with a different (or equal) case does not
# already exist. Windows is case insensitive.
if path(child.path).name == base:
raise Exception(
'Directory %s already exists for project' %
createPath)
node = model.ThinAppFile()
node.isDirectory = True
node.root = parent.root
node.path = createPath
parent.children.append(node)
sess.add(node)
sess.flush()
sess.refresh(node)
return node.id
except:
sysPath.rmdir()
sess.rollback()
raise
def UpdateRegistryKey(self, updateKey, newValues, projectId=None):
"""
updateKey is meant to be a detached object returned by GetRegKey.
The caller will modify this object and send it back.
"""
sess = self.db.CreateSession()
with model.AutoSession(sess):
# Bring this into the session
updateKey = sess.merge(updateKey)
# Mark the key as no longer intermediate since it has
# values bound to it.
updateKey.intermediate = False
newValueDict = dict([(v.name, v) for v in newValues])
oldValueDict = dict([(v.name, v) for v in updateKey.values])
newValueSet = set([v.name for v in newValues])
oldValueSet = set([v.name for v in updateKey.values])
finalValues = []
# Start by populating totally new values, the easiest case
for newValueName in newValueSet - oldValueSet:
newValue = newValueDict[newValueName]
finalValues.append(self.MakeRegistryValue(newValue))
# For the rest, must check the data carefully
for existingValueName in newValueSet & oldValueSet:
updateValue = newValueDict[existingValueName]
oldValue = oldValueDict[existingValueName]
# XXX: We unnecessarily delete and recreate here
if updateValue.regType != oldValue.regType or \
updateValue.data != oldValue.data or \
updateValue.nameExpand != oldValue.nameExpand or \
updateValue.dataExpand != oldValue.dataExpand:
finalValues.append(self.MakeRegistryValue(updateValue))
else:
# If neither registry type nor data have changed, consider
# it a no-op for now.
finalValues.append(oldValue)
# Deleted values are automatically handled by SQLAlchemy
updateKey.values = finalValues
# If supplied, mark project as dirty
if projectId:
project = sess.query(model.Project).get(projectId)
project.state = defs.Projects.DIRTY
def Fsck(self):
# TODO: This could contain more consistency checks in the future.
log.debug('Performing consistency check.')
# Go through ALL projects and fix in-flight states.
STATE_CLEANUP_MAP = {
'deleting': 'deleted',
'rebuilding': 'dirty',
}
sess = self.db.CreateSession()
with model.AutoSession(sess):
dirty = sess.query(model.Project).filter(
model.Project.state.in_(STATE_CLEANUP_MAP))
for proj in dirty:
newState = STATE_CLEANUP_MAP[proj.state]
log.info('Set state to %s for project %d', newState, proj.id)
proj.state = newState
def DeleteProjectData(self, recordType, recordId, projectId=None):
PROJECT_DATA_TYPE_MAP = {
defs.Projects.TYPE_REGKEY: model.RegistryKey,
defs.Projects.TYPE_REGVALUE: model.RegistryValue,
defs.Projects.TYPE_FILE: model.ThinAppFile,
}
if recordType not in PROJECT_DATA_TYPE_MAP:
raise RuntimeError('Unrecognized type name %s' % recordType)
sess = self.db.CreateSession()
with model.AutoSession(sess):
chosenType = PROJECT_DATA_TYPE_MAP[recordType]
obj = sess.query(chosenType).get(recordId)
sess.delete(obj)
# If supplied, mark project as dirty
if projectId:
project = sess.query(model.Project).get(projectId)
project.state = defs.Projects.DIRTY
def __init__(self, db, config):
# name: reference count
self.leases = collections.defaultdict(
lambda: collections.defaultdict(lambda: 0))
self.db = db
self.config = config
self.leaseNumber = 0
# Reset all datastores to known state.
log.info('Reseting all datastores to offline state.')
sess = self.db.CreateSession()
# We need the autosession out here so that the inner AutoSession in
# GetDataStoreList doesn't expire the objects we want to use.
with model.AutoSession(sess):
for ds in self.GetDataStoreList(sess):
try:
self.GoOffline(ds)
except MountError, e:
log.debug('%s was probably already offline.', ds)
def GetStatus(self, id):
sess = self.db.CreateSession()
with model.AutoSession(sess):
ds = self.GetDatastore(sess, id)
capacity = None
used = None
if ds.status == 'online':
stat = mount.statfs(ds.localPath)
blockSize = stat['f_bsize']
# There is also f_bfree which gives the total amount of free
# space. f_bavail is the number of blocks available to an
# unprivileged user (which we are).
available = stat['f_bavail'] * blockSize
capacity = stat['f_blocks'] * blockSize
used = capacity - available
# Get file used, total, etc.
return {
'id': id,
'name': ds.name,
'type': 'cifs',
'server': ds.server,
'share': ds.share,
'size': capacity,
'used': used,
'status': ds.status,
'domain': ds.domain,
'username': ds.username,
'password': ds.password,
'mountAtBoot': True,
'leases': self.GetLeases(id),
'mountPath': ds.localPath,
'baseUrl': ds.baseUrl,
}
def Import(self, targetDatastoreId, runtimeId, datastores):
newProjects = {}
sess = self.db.CreateSession()
with model.AutoSession(sess):
ds = datastores.GetDatastore(sess, targetDatastoreId)
if ds.status != 'online':
raise DatastoreException(
'Datastore [%s] must be online to import projects!',
ds.name)
result = util.ScanProjectDirs(ds.localPath)
dirs = result['Valid_Dirs']
if len(dirs) == 0:
log.info('No ThinApp project directories found in %s',
ds.localPath)
return {
'newProjects': newProjects,
'errors': result['Invalid_Dirs_Map']
}
for dir in dirs:
project = model.Project()
project.datastore = ds
project.state = defs.Projects.CREATED
project.subdir = dir
project.runtime_id = runtimeId
sess.add(project)
sess.flush()
newProjects[project.id] = dir
return {
'newProjects': newProjects,
'errors': result['Invalid_Dirs_Map']
}
def CreateRegistryKey(self, parentId, key, values, projectId=None):
sess = self.db.CreateSession()
with model.AutoSession(sess):
newKey = self.MakeRegistryKey(key)
parent = sess.query(model.RegistryKey).get(parentId)
parent.subkeys.append(newKey)
for v in values:
newKey.values.append(self.MakeRegistryValue(v))
sess.add(newKey)
sess.flush()
sess.refresh(newKey)
newId = newKey.id
# If supplied, mark project as dirty
if projectId:
project = sess.query(model.Project).get(projectId)
project.state = defs.Projects.DIRTY
return newId
def DeleteFile(self, projectId, fileId, internal=False):
sess = self.db.CreateSession()
# We want to close the session only on exceptions.
project = sess.query(model.Project).get(projectId)
fileObj = sess.query(model.ThinAppFile).get(fileId)
if project is None or fileObj is None:
sess.close()
raise Exception('Project or file object not found')
root = path(project.datastore.localPath) / project.subdir
dir, base = path(fileObj.path).splitpath()
if not internal and self.IsRestrictedPath(fileObj.path):
raise Exception('Cannot delete restricted file %s' % base)
else:
pathSave = fileObj.path
if fileObj.isDirectory:
# If the directory has more than one file in it, it is obviously
# not empty. If it only has one file in it, and that file is not
# ##Attributes.ini, also consider it not empty.
if len(fileObj.children) > 1:
raise Exception('Cannot delete nonempty directory %s' %
fileObj.path)
elif len(fileObj.children) == 1:
# Only automatically delete ##Attributes.ini. Not anything else
child = fileObj.children[0]
if path(child.path).name != '##Attributes.ini':
raise Exception('Cannot delete nonempty directory %s' %
fileObj.path)
self.DeleteFile(projectId, child.id, internal=True)
# Delete the DB object before the file object to ensure consistency
with model.AutoSession(sess):
# XXX: fileObj seems to get expired in certain cases here.
# Ask for it again. TODO: why?
fileObj = sess.query(model.ThinAppFile).get(fileId)
try:
with self.LockProjectFile(fileObj.id):
sess.delete(fileObj)
except NoResultFound:
log.warning(
'File deleted before it could be locked. Returning.')
return
# Now actually delete the file/directory
try:
if fileObj.isDirectory:
(root / pathSave).rmdir()
else:
(root / pathSave).unlink()
except OSError, e:
if e.errno != errno.ENOENT:
raise
log.warning('File %s already deleted. Continuing anyway.',
pathSave)
def _processRebuild(self, projectId, datastores, config):
sess = self.db.CreateSession()
with closing(sess):
lease = None
try:
project = sess.query(model.Project).get(projectId)
# Ensure it won't be unmounted from underneath us.
lease = datastores.Acquire(project.datastore.id)
projectDir = path(project.subdir)
assert not projectDir.isabs()
# Dump the registry from the database back out to a file.
# (But don't affect the REBUILDING state.)
self.WriteRegistry(project, makeDirty=False)
fullPath = path(lease.share.localPath) / projectDir
binPath = fullPath / 'bin'
# Usually build.bat clears this out but when running under
# wine it doesn't.
try:
binPath.rmtree()
except OSError, e:
if e.errno == errno.ENOENT:
log.debug('%s did not exist.', binPath)
else:
raise
buildBat = fullPath / 'build.bat'
if not buildBat.exists():
raise RuntimeError('Expected %s to exist but it does not' %
buildBat)
runtimesUrl = config['runtimes.url']
runtimes = json.load(urllib2.urlopen(runtimesUrl))
for runtime in runtimes:
if runtime['id'] == project.runtime_id:
log.debug('Using runtime: %s.', runtime)
break
else:
raise Exception('Unable to locate runtime version %s' %
project.runtime_id)
taRoot = path(runtime['path'])
# Unfortunately the build.bat files don't exit 1 when the
# ThinApp runtime is not found. So check for it ourselves first
# as a sanity check so that we are a little more sure that exit
# status 0 means success.
files = ('vregtool.exe', 'vftool.exe', 'tlink.exe')
for f in files:
p = taRoot / f
if not p.exists():
raise RuntimeError(
'ThinApp runtime file unavailable: %s' % p)
rebuildProc = subprocess.Popen(
['wine', 'cmd.exe', '/c', 'build.bat'],
env={'THINSTALL_BIN': taRoot},
cwd=str(fullPath),
stderr=subprocess.PIPE,
stdout=subprocess.PIPE)
stdout, stderr = rebuildProc.communicate()
stderr.strip() and log.error(stderr)
stdout.strip() and log.info(stdout)
ret = rebuildProc.returncode
# Check if build.bat left behind package.vo.tvr* files. This
# indicates a build failure.
pkgInvalid = self._validateBuildGeneratedFiles(binPath)
# For now, if a rebuild fails for any reason, the project becomes
# dirty. Even if you didn't make any changes prior to the rebuild,
# this rebuild could have created partial output and we don't
# want people hitting any of the bin/ files until a rebuild does
# succeed.
if ret != 0 or pkgInvalid:
log.info('Rebuild for project %d has failed, status %d',
projectId, ret)
with model.AutoSession(sess):
project.state = defs.Projects.DIRTY
return
# Else, if successful, let Refresh set up the AVAILABLE
# state after it scans everything.
log.info('Rebuild for project %d succeeded.', projectId)
self.Refresh(projectId)
finally:
def Update(self, project):
sess = self.db.CreateSession()
with model.AutoSession(sess):
sess.merge(project)
def _changeState(self, id, state, sess):
log.debug('Request received for %s to go %s.', id, state)
ds = self.GetDatastore(sess, id)
# XXX: Session use could be more fine grained here, but we do need to
# close the session if anything happens.
with model.AutoSession(sess):
# Return if already online.
if ds.status == state:
log.info('%s[%d] is already %s.', ds.name, ds.id, state)
return
if ds.name in (SYSTEM, INTERNAL):
ds.status = state
log.info(
'No mounting/unmounting is required for the %s datastore',
ds.name)
return
share = ds.GetAsShare()
dsId = str(ds.id)
# XXX: Can't figure out how to look up
# where scripts were installed to (not sure if it's even
# possible) so use the default directory.
#
# An absolute path is required even though cifsmount is in
# our PATH to do sudo being built with SECURE_PATH.
bin = path(sys.exec_prefix) / 'bin'
if state == 'online':
ds.localPath = mounter.MOUNT_ROOT / dsId
cmd = bin / 'cifsmount'
# The username may have the domain in it in either form
# user\domain or user/domain. The two values have to be
# separated out when calling the mounter.
username = share.username.replace('\\', '/')
domainUser = username.split('/')
if len(domainUser) == 2:
domain, username = domainUser
else:
domain, username = '', share.username
res = subprocess.Popen([
'sudo', cmd, '--datastore', dsId, '--domain', domain,
'--username', username, '--password', share.password,
'--unc', share.uncPath
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = res.communicate()
for s in (stderr, stdout):
if s.strip():
log.warning('Output from cifsmount:\n')
log.warning(s)
log.warning('Output done from cifsmount.\n')
else:
ds.localPath = None
cmd = bin / 'cifsumount'
res = subprocess.Popen(['sudo', cmd, '--datastore', dsId],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = res.communicate()
for s in (stderr, stdout):
if s.strip():
log.warning('Output from cifsmount:\n')
log.warning(s)
log.warning('Output done from cifsmount.\n')
code = res.wait()
if code != 0:
if state == 'offline':
log.info(
'Failed to unmount datastore %s with code: %d. '
'Assuming already offline.', ds.name, code)
else:
log.error('%s was unable to go %s with code: %d', ds.name,
state, code)
raise MountError('Mounter exited non-zero: %d.' % code,
code)
ds.status = state
log.info('%s has gone %s.', ds.name, state)
def SetState(self, projectId, state):
sess = self.db.CreateSession()
with model.AutoSession(sess):
project = sess.query(model.Project).get(projectId)
project.state = state
def OpenProjectFile(self,
projectId,
_filePath,
mode,
charset=None,
internal=False,
makeDirty=True):
sess = self.db.CreateSession()
try:
project = sess.query(model.Project).get(projectId)
except NoResultFound:
sess.close()
raise
root = path(project.datastore.localPath) / project.subdir
filePath = path(_filePath)
fileDir, fileBase = filePath.splitpath()
# Some sanity checks.
if not (root / filePath).abspath().startswith(root):
sess.close()
raise Exception(
'Directory traversal outside project root not allowed')
if 'r' in mode:
# Do not allow opening of a file that is not backed by a ThinAppFile.
# Note that reading locked files is OK.
dbFile = self.GetProjectFileByPath(project.id,
filePath,
session=sess)
# XXX: Is this necessary? If you yield a file, does its enter/exit get
# automatically called as if you just did with open(...)?
try:
fobj = codecs.open(root / filePath, mode, encoding=charset)
yield fobj, dbFile.id
except:
sess.close()
raise
finally:
fobj.close()
elif 'w' in mode:
# Only internal consumers may write to restricted paths.
if not internal and self.IsRestrictedPath(filePath):
raise Exception('Cannot write to restricted filename %s' %
fileBase)
# Write to a randomly generated file first. Return a wrapped file
# that upon successful closing will rename it to the target, and
# upon exception will delete the temp file.
# XXX: Better way to generate temp file names.
tempPath = root / filePath + '.%d.tmp' % time.time()
# NB: codecs.open always opens in binary mode even if no charset
# is specified
fobj = codecs.open(tempPath, mode, encoding=charset)
try:
fileId = self.GetProjectFileByPath(projectId,
filePath,
session=sess).id
existingFile = True
except NoResultFound:
log.debug('File %s not found -- creating a new one', filePath)
existingFile = False
# Verify that a differently cased version of the same name does not
# already exist.
listing = (root / fileDir).listdir()
fileLower = fileBase.lower()
for f in listing:
if f.name.lower() == fileLower:
raise Exception(
'Conflicting filename, tried to create %s but %s already exists'
% (fileBase, f.name))
# New file - create a new entry and lock and hide it.
parentObj = self.GetProjectFileByPath(project.id,
fileDir,
directory=True,
session=sess)
dbFile = model.ThinAppFile()
dbFile.path = filePath
dbFile.isDirectory = False
dbFile.hidden = True
dbFile.root = project.directory
# implies sess.add(dbFile)
with model.AutoSession(sess):
parentObj.children.append(dbFile)
sess.flush()
sess.refresh(dbFile)
fileId = dbFile.id
except: # Catch-all
sess.close()
raise
# At this point, we have no need for the session, close it if
# not already closed by an AutoSession.
sess.close()
# Now lock the file for writing and pass control to the caller.
with self.LockProjectFile(fileId):
try:
# Pass the file ID on as well (XXX: Better alternative?)
yield fobj, fileId
except:
# Catch-all - exception is reraised after nuking temp file
tempPath.unlink()
raise
log.debug('Rename file from %s to %s', tempPath,
root / filePath)
tempPath.rename(root / filePath)
# File is created hidden by default. If it's not a system file, then
# unhide it now.
with model.AutoSession(sess):
if not existingFile and not self.IsRestrictedPath(filePath) and \
fileBase not in self.RESTRICTED_PROJECT_FILENAMES:
dbFile = sess.query(model.ThinAppFile).get(fileId)
dbFile.hidden = False
# Any write operation to a project file causes the project to
# become dirty.
if makeDirty:
project = sess.query(model.Project).get(projectId)
project.state = defs.Projects.DIRTY
else:
sess.close()
raise Exception('Malformed open mode %s' % mode)