def UpdateDatastore(self, id, share, errorOnExists=True): sess = self.db.CreateSession() try: with model.AutoSession(sess): ds = self.GetDatastore(sess, id) log.info("Updating existing datastore [name:%s,id:%d]", ds.name, ds.id) # Ensure 'system' and 'internal' names remain unchanged. if ds.name in (SYSTEM, INTERNAL) and ds.name != share.name: raise DatastoreException( 'Changing the %s datastore name is not allowed.', ds.name) if ds.status != 'offline': # XXX: Use a better exception. raise DataStoreInUseError( ds.name % " is not in the offline mode yet!") if errorOnExists: raise DatastoreExistsError(id) # Just update the info in case it changed. with model.AutoSession(sess): updated = model.DataStore.FromShare(share) updated.id = ds.id sess.merge(updated) except DataStoreNotFoundError: log.debug('The %s[id=%d] datastore was not found', share.name, id) raise DataStoreNotFoundError('The %s datastore was not found.', share.name)
def Refresh(self, projectId, datastores): sess = self.db.CreateSession() with model.AutoSession(sess): project = sess.query(model.Project).get(projectId) datastore = datastores.GetDatastore(sess, project.datastore.id) projectRoot = path(datastore.localPath) / project.subdir lease = datastores.Acquire(datastore.id) try: # Always completely repopulate files since entry points # may have been removed. del project.files[:] files = self.GetFiles(projectRoot) self.RegisterFiles(project, files, sess) finally: datastores.Release(lease) sess = self.db.CreateSession() with model.AutoSession(sess): # Have to do this in a new session otherwise registryIcon # will fail because it's looking for some state that isn't # flushed when it tries to read Package.ini settings. project = sess.query(model.Project).get(projectId) try: self._registerIcon(project, projectRoot) except Exception: log.exception( 'Unable to register an icon for the project. Continuing.') project.state = defs.Projects.AVAILABLE
def Acquire(self, dsId): sess = self.db.CreateSession() with model.AutoSession(sess): try: ds = sess.query(model.DataStore).filter_by(id=dsId).one() if ds.name == SYSTEM: raise DatastoreException( 'The system partition is read-only.') share = ds.GetAsShare() if ds.status != 'online': raise DataStoreOfflineError except ormexc.NoResultFound: # raised by .one() raise DataStoreNotFoundError(dsId) # Note that share is not a SQLAlchemy object, so no need # to keep the session open or expunge the object lease = DataStoreLease(self.leaseNumber, dsId, share) self.leases[dsId][lease.id] = lease self.leaseNumber += 1 return lease
def Create(self, targetDatastoreId, runtimeId, datastores): sess = self.db.CreateSession() project = model.Project() with model.AutoSession(sess): project.datastore = datastores.GetDatastore( sess, targetDatastoreId) project.runtime_id = runtimeId project.state = defs.Projects.CREATED # subdir isn't nullable so set to a dummy value until we have # the primary key after flushing below. project.subdir = '' sess.add(project) sess.flush() # So that we have the id. project.subdir = 'project-%d' % project.id # Have to flush again for project.subdir to get set. sess.flush() lease = None # Create the base directory for the project. try: ds = project.datastore lease = datastores.Acquire(ds.id) # Make sure that the datastore is up-to-date since it # could have gone into the online state between looking # the project up and acquiring the datastore lease. sess.refresh(ds) supportPath = path(ds.localPath) / project.subdir / 'Support' logPath = supportPath / 'taf.log' try: supportPath.makedirs() # to ensure the Support/appfactory.log is # writeable by tomcat and readable by user logPath.touch() logPath.chmod(0664) except OSError, e: if e.errno != errno.EEXIST: raise finally: if lease: datastores.Release(lease) # project has been expired since it was just committed. # Therefore the next time one of its attributes is accessed # it will need to hit the database. So go ahead and refresh # it so that its values are already loaded for the caller to # access. sess.refresh(project) # Remove it from the session since we just need to read # values. sess.expunge(project) return project
def _processDelete(self, projectId, datastores): sess = self.db.CreateSession() with closing(sess): lease = None try: project = sess.query(model.Project).get(projectId) # Ensure it won't be unmounted from underneath us. lease = datastores.Acquire(project.datastore.id) projectDir = path(project.subdir) assert not projectDir.isabs() fullPath = path(lease.share.localPath) / projectDir log.info('Deleting project %d from %s.', projectId, fullPath) log.info('Removing read-only bits from project.') # path.walk traverses symlinks which we don't want to do. eventlet.tpool.execute(util.FixPermissions, fullPath) log.info( 'Read-only bits removed from all files and directories.') # XXX: Who knows what errors could happen here but we ignore # them and just set the project state to deleted anyway. The # client will have no idea something went wrong. Maybe it needs # to go to a failure state so the client can retry? try: # XXX: If this fails the client will think they got # deleted when they really didn't. Network might have # gone out, etc. # Also note that leading directories # $localPath/path/to/project will not get cleaned up # when they become empty. # Must run in threadpool to prevent blocking greenthreads. eventlet.tpool.execute(fullPath.rmtree) log.info('Project %d deleted.', projectId) except Exception: log.exception('There was an error removing project %d.', project.id) with model.AutoSession(sess): # NULL-out icon to avoid database from growing too # large. project.icon = None project.state = defs.Projects.DELETED finally: if lease: datastores.Release(lease)
def GetDataStoreList(self, sess=None): if not sess: sess = self.db.CreateSession() with model.AutoSession(sess): dsList = [ n[0] for n in sess.execute(sqlalchemy.select([model.DataStore.id])) ] return dsList
def DeleteDatastore(self, id): sess = self.db.CreateSession() with model.AutoSession(sess): ds = self.GetDatastore(sess, id) if ds.name in (SYSTEM, INTERNAL): raise DatastoreException( 'Deleting the %s datastore is prohibited!', ds.name) self.VerifyLeases(id) self.GoOffline(id) sess.delete(ds)
def CreateDirectory(self, projectId, createPath): # Verify that parent exists in DB dir, base = path(createPath).splitpath() sess = self.db.CreateSession() with model.AutoSession(sess): project = sess.query(model.Project).get(projectId) root = path(project.datastore.localPath) / project.subdir sysPath = root / createPath if sysPath.isdir(): raise Exception('Directory %s already exists for project' % createPath) elif '..' in sysPath: raise Exception('Relative paths not allowed: %s' % createPath) # Create the directory before touching the DB. That way, the second # the database is flushed, there is no race to create the directory. sysPath.mkdir() try: # We want to raise the exception directly if the parent is not # found for whatever reason. parent = self.GetProjectFileByPath(projectId, dir, directory=True, session=sess) for child in parent.children: # Verify that a name with a different (or equal) case does not # already exist. Windows is case insensitive. if path(child.path).name == base: raise Exception( 'Directory %s already exists for project' % createPath) node = model.ThinAppFile() node.isDirectory = True node.root = parent.root node.path = createPath parent.children.append(node) sess.add(node) sess.flush() sess.refresh(node) return node.id except: sysPath.rmdir() sess.rollback() raise
def UpdateRegistryKey(self, updateKey, newValues, projectId=None): """ updateKey is meant to be a detached object returned by GetRegKey. The caller will modify this object and send it back. """ sess = self.db.CreateSession() with model.AutoSession(sess): # Bring this into the session updateKey = sess.merge(updateKey) # Mark the key as no longer intermediate since it has # values bound to it. updateKey.intermediate = False newValueDict = dict([(v.name, v) for v in newValues]) oldValueDict = dict([(v.name, v) for v in updateKey.values]) newValueSet = set([v.name for v in newValues]) oldValueSet = set([v.name for v in updateKey.values]) finalValues = [] # Start by populating totally new values, the easiest case for newValueName in newValueSet - oldValueSet: newValue = newValueDict[newValueName] finalValues.append(self.MakeRegistryValue(newValue)) # For the rest, must check the data carefully for existingValueName in newValueSet & oldValueSet: updateValue = newValueDict[existingValueName] oldValue = oldValueDict[existingValueName] # XXX: We unnecessarily delete and recreate here if updateValue.regType != oldValue.regType or \ updateValue.data != oldValue.data or \ updateValue.nameExpand != oldValue.nameExpand or \ updateValue.dataExpand != oldValue.dataExpand: finalValues.append(self.MakeRegistryValue(updateValue)) else: # If neither registry type nor data have changed, consider # it a no-op for now. finalValues.append(oldValue) # Deleted values are automatically handled by SQLAlchemy updateKey.values = finalValues # If supplied, mark project as dirty if projectId: project = sess.query(model.Project).get(projectId) project.state = defs.Projects.DIRTY
def Fsck(self): # TODO: This could contain more consistency checks in the future. log.debug('Performing consistency check.') # Go through ALL projects and fix in-flight states. STATE_CLEANUP_MAP = { 'deleting': 'deleted', 'rebuilding': 'dirty', } sess = self.db.CreateSession() with model.AutoSession(sess): dirty = sess.query(model.Project).filter( model.Project.state.in_(STATE_CLEANUP_MAP)) for proj in dirty: newState = STATE_CLEANUP_MAP[proj.state] log.info('Set state to %s for project %d', newState, proj.id) proj.state = newState
def DeleteProjectData(self, recordType, recordId, projectId=None): PROJECT_DATA_TYPE_MAP = { defs.Projects.TYPE_REGKEY: model.RegistryKey, defs.Projects.TYPE_REGVALUE: model.RegistryValue, defs.Projects.TYPE_FILE: model.ThinAppFile, } if recordType not in PROJECT_DATA_TYPE_MAP: raise RuntimeError('Unrecognized type name %s' % recordType) sess = self.db.CreateSession() with model.AutoSession(sess): chosenType = PROJECT_DATA_TYPE_MAP[recordType] obj = sess.query(chosenType).get(recordId) sess.delete(obj) # If supplied, mark project as dirty if projectId: project = sess.query(model.Project).get(projectId) project.state = defs.Projects.DIRTY
def __init__(self, db, config): # name: reference count self.leases = collections.defaultdict( lambda: collections.defaultdict(lambda: 0)) self.db = db self.config = config self.leaseNumber = 0 # Reset all datastores to known state. log.info('Reseting all datastores to offline state.') sess = self.db.CreateSession() # We need the autosession out here so that the inner AutoSession in # GetDataStoreList doesn't expire the objects we want to use. with model.AutoSession(sess): for ds in self.GetDataStoreList(sess): try: self.GoOffline(ds) except MountError, e: log.debug('%s was probably already offline.', ds)
def GetStatus(self, id): sess = self.db.CreateSession() with model.AutoSession(sess): ds = self.GetDatastore(sess, id) capacity = None used = None if ds.status == 'online': stat = mount.statfs(ds.localPath) blockSize = stat['f_bsize'] # There is also f_bfree which gives the total amount of free # space. f_bavail is the number of blocks available to an # unprivileged user (which we are). available = stat['f_bavail'] * blockSize capacity = stat['f_blocks'] * blockSize used = capacity - available # Get file used, total, etc. return { 'id': id, 'name': ds.name, 'type': 'cifs', 'server': ds.server, 'share': ds.share, 'size': capacity, 'used': used, 'status': ds.status, 'domain': ds.domain, 'username': ds.username, 'password': ds.password, 'mountAtBoot': True, 'leases': self.GetLeases(id), 'mountPath': ds.localPath, 'baseUrl': ds.baseUrl, }
def Import(self, targetDatastoreId, runtimeId, datastores): newProjects = {} sess = self.db.CreateSession() with model.AutoSession(sess): ds = datastores.GetDatastore(sess, targetDatastoreId) if ds.status != 'online': raise DatastoreException( 'Datastore [%s] must be online to import projects!', ds.name) result = util.ScanProjectDirs(ds.localPath) dirs = result['Valid_Dirs'] if len(dirs) == 0: log.info('No ThinApp project directories found in %s', ds.localPath) return { 'newProjects': newProjects, 'errors': result['Invalid_Dirs_Map'] } for dir in dirs: project = model.Project() project.datastore = ds project.state = defs.Projects.CREATED project.subdir = dir project.runtime_id = runtimeId sess.add(project) sess.flush() newProjects[project.id] = dir return { 'newProjects': newProjects, 'errors': result['Invalid_Dirs_Map'] }
def CreateRegistryKey(self, parentId, key, values, projectId=None): sess = self.db.CreateSession() with model.AutoSession(sess): newKey = self.MakeRegistryKey(key) parent = sess.query(model.RegistryKey).get(parentId) parent.subkeys.append(newKey) for v in values: newKey.values.append(self.MakeRegistryValue(v)) sess.add(newKey) sess.flush() sess.refresh(newKey) newId = newKey.id # If supplied, mark project as dirty if projectId: project = sess.query(model.Project).get(projectId) project.state = defs.Projects.DIRTY return newId
def DeleteFile(self, projectId, fileId, internal=False): sess = self.db.CreateSession() # We want to close the session only on exceptions. project = sess.query(model.Project).get(projectId) fileObj = sess.query(model.ThinAppFile).get(fileId) if project is None or fileObj is None: sess.close() raise Exception('Project or file object not found') root = path(project.datastore.localPath) / project.subdir dir, base = path(fileObj.path).splitpath() if not internal and self.IsRestrictedPath(fileObj.path): raise Exception('Cannot delete restricted file %s' % base) else: pathSave = fileObj.path if fileObj.isDirectory: # If the directory has more than one file in it, it is obviously # not empty. If it only has one file in it, and that file is not # ##Attributes.ini, also consider it not empty. if len(fileObj.children) > 1: raise Exception('Cannot delete nonempty directory %s' % fileObj.path) elif len(fileObj.children) == 1: # Only automatically delete ##Attributes.ini. Not anything else child = fileObj.children[0] if path(child.path).name != '##Attributes.ini': raise Exception('Cannot delete nonempty directory %s' % fileObj.path) self.DeleteFile(projectId, child.id, internal=True) # Delete the DB object before the file object to ensure consistency with model.AutoSession(sess): # XXX: fileObj seems to get expired in certain cases here. # Ask for it again. TODO: why? fileObj = sess.query(model.ThinAppFile).get(fileId) try: with self.LockProjectFile(fileObj.id): sess.delete(fileObj) except NoResultFound: log.warning( 'File deleted before it could be locked. Returning.') return # Now actually delete the file/directory try: if fileObj.isDirectory: (root / pathSave).rmdir() else: (root / pathSave).unlink() except OSError, e: if e.errno != errno.ENOENT: raise log.warning('File %s already deleted. Continuing anyway.', pathSave)
def _processRebuild(self, projectId, datastores, config): sess = self.db.CreateSession() with closing(sess): lease = None try: project = sess.query(model.Project).get(projectId) # Ensure it won't be unmounted from underneath us. lease = datastores.Acquire(project.datastore.id) projectDir = path(project.subdir) assert not projectDir.isabs() # Dump the registry from the database back out to a file. # (But don't affect the REBUILDING state.) self.WriteRegistry(project, makeDirty=False) fullPath = path(lease.share.localPath) / projectDir binPath = fullPath / 'bin' # Usually build.bat clears this out but when running under # wine it doesn't. try: binPath.rmtree() except OSError, e: if e.errno == errno.ENOENT: log.debug('%s did not exist.', binPath) else: raise buildBat = fullPath / 'build.bat' if not buildBat.exists(): raise RuntimeError('Expected %s to exist but it does not' % buildBat) runtimesUrl = config['runtimes.url'] runtimes = json.load(urllib2.urlopen(runtimesUrl)) for runtime in runtimes: if runtime['id'] == project.runtime_id: log.debug('Using runtime: %s.', runtime) break else: raise Exception('Unable to locate runtime version %s' % project.runtime_id) taRoot = path(runtime['path']) # Unfortunately the build.bat files don't exit 1 when the # ThinApp runtime is not found. So check for it ourselves first # as a sanity check so that we are a little more sure that exit # status 0 means success. files = ('vregtool.exe', 'vftool.exe', 'tlink.exe') for f in files: p = taRoot / f if not p.exists(): raise RuntimeError( 'ThinApp runtime file unavailable: %s' % p) rebuildProc = subprocess.Popen( ['wine', 'cmd.exe', '/c', 'build.bat'], env={'THINSTALL_BIN': taRoot}, cwd=str(fullPath), stderr=subprocess.PIPE, stdout=subprocess.PIPE) stdout, stderr = rebuildProc.communicate() stderr.strip() and log.error(stderr) stdout.strip() and log.info(stdout) ret = rebuildProc.returncode # Check if build.bat left behind package.vo.tvr* files. This # indicates a build failure. pkgInvalid = self._validateBuildGeneratedFiles(binPath) # For now, if a rebuild fails for any reason, the project becomes # dirty. Even if you didn't make any changes prior to the rebuild, # this rebuild could have created partial output and we don't # want people hitting any of the bin/ files until a rebuild does # succeed. if ret != 0 or pkgInvalid: log.info('Rebuild for project %d has failed, status %d', projectId, ret) with model.AutoSession(sess): project.state = defs.Projects.DIRTY return # Else, if successful, let Refresh set up the AVAILABLE # state after it scans everything. log.info('Rebuild for project %d succeeded.', projectId) self.Refresh(projectId) finally:
def Update(self, project): sess = self.db.CreateSession() with model.AutoSession(sess): sess.merge(project)
def _changeState(self, id, state, sess): log.debug('Request received for %s to go %s.', id, state) ds = self.GetDatastore(sess, id) # XXX: Session use could be more fine grained here, but we do need to # close the session if anything happens. with model.AutoSession(sess): # Return if already online. if ds.status == state: log.info('%s[%d] is already %s.', ds.name, ds.id, state) return if ds.name in (SYSTEM, INTERNAL): ds.status = state log.info( 'No mounting/unmounting is required for the %s datastore', ds.name) return share = ds.GetAsShare() dsId = str(ds.id) # XXX: Can't figure out how to look up # where scripts were installed to (not sure if it's even # possible) so use the default directory. # # An absolute path is required even though cifsmount is in # our PATH to do sudo being built with SECURE_PATH. bin = path(sys.exec_prefix) / 'bin' if state == 'online': ds.localPath = mounter.MOUNT_ROOT / dsId cmd = bin / 'cifsmount' # The username may have the domain in it in either form # user\domain or user/domain. The two values have to be # separated out when calling the mounter. username = share.username.replace('\\', '/') domainUser = username.split('/') if len(domainUser) == 2: domain, username = domainUser else: domain, username = '', share.username res = subprocess.Popen([ 'sudo', cmd, '--datastore', dsId, '--domain', domain, '--username', username, '--password', share.password, '--unc', share.uncPath ], stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = res.communicate() for s in (stderr, stdout): if s.strip(): log.warning('Output from cifsmount:\n') log.warning(s) log.warning('Output done from cifsmount.\n') else: ds.localPath = None cmd = bin / 'cifsumount' res = subprocess.Popen(['sudo', cmd, '--datastore', dsId], stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = res.communicate() for s in (stderr, stdout): if s.strip(): log.warning('Output from cifsmount:\n') log.warning(s) log.warning('Output done from cifsmount.\n') code = res.wait() if code != 0: if state == 'offline': log.info( 'Failed to unmount datastore %s with code: %d. ' 'Assuming already offline.', ds.name, code) else: log.error('%s was unable to go %s with code: %d', ds.name, state, code) raise MountError('Mounter exited non-zero: %d.' % code, code) ds.status = state log.info('%s has gone %s.', ds.name, state)
def SetState(self, projectId, state): sess = self.db.CreateSession() with model.AutoSession(sess): project = sess.query(model.Project).get(projectId) project.state = state
def OpenProjectFile(self, projectId, _filePath, mode, charset=None, internal=False, makeDirty=True): sess = self.db.CreateSession() try: project = sess.query(model.Project).get(projectId) except NoResultFound: sess.close() raise root = path(project.datastore.localPath) / project.subdir filePath = path(_filePath) fileDir, fileBase = filePath.splitpath() # Some sanity checks. if not (root / filePath).abspath().startswith(root): sess.close() raise Exception( 'Directory traversal outside project root not allowed') if 'r' in mode: # Do not allow opening of a file that is not backed by a ThinAppFile. # Note that reading locked files is OK. dbFile = self.GetProjectFileByPath(project.id, filePath, session=sess) # XXX: Is this necessary? If you yield a file, does its enter/exit get # automatically called as if you just did with open(...)? try: fobj = codecs.open(root / filePath, mode, encoding=charset) yield fobj, dbFile.id except: sess.close() raise finally: fobj.close() elif 'w' in mode: # Only internal consumers may write to restricted paths. if not internal and self.IsRestrictedPath(filePath): raise Exception('Cannot write to restricted filename %s' % fileBase) # Write to a randomly generated file first. Return a wrapped file # that upon successful closing will rename it to the target, and # upon exception will delete the temp file. # XXX: Better way to generate temp file names. tempPath = root / filePath + '.%d.tmp' % time.time() # NB: codecs.open always opens in binary mode even if no charset # is specified fobj = codecs.open(tempPath, mode, encoding=charset) try: fileId = self.GetProjectFileByPath(projectId, filePath, session=sess).id existingFile = True except NoResultFound: log.debug('File %s not found -- creating a new one', filePath) existingFile = False # Verify that a differently cased version of the same name does not # already exist. listing = (root / fileDir).listdir() fileLower = fileBase.lower() for f in listing: if f.name.lower() == fileLower: raise Exception( 'Conflicting filename, tried to create %s but %s already exists' % (fileBase, f.name)) # New file - create a new entry and lock and hide it. parentObj = self.GetProjectFileByPath(project.id, fileDir, directory=True, session=sess) dbFile = model.ThinAppFile() dbFile.path = filePath dbFile.isDirectory = False dbFile.hidden = True dbFile.root = project.directory # implies sess.add(dbFile) with model.AutoSession(sess): parentObj.children.append(dbFile) sess.flush() sess.refresh(dbFile) fileId = dbFile.id except: # Catch-all sess.close() raise # At this point, we have no need for the session, close it if # not already closed by an AutoSession. sess.close() # Now lock the file for writing and pass control to the caller. with self.LockProjectFile(fileId): try: # Pass the file ID on as well (XXX: Better alternative?) yield fobj, fileId except: # Catch-all - exception is reraised after nuking temp file tempPath.unlink() raise log.debug('Rename file from %s to %s', tempPath, root / filePath) tempPath.rename(root / filePath) # File is created hidden by default. If it's not a system file, then # unhide it now. with model.AutoSession(sess): if not existingFile and not self.IsRestrictedPath(filePath) and \ fileBase not in self.RESTRICTED_PROJECT_FILENAMES: dbFile = sess.query(model.ThinAppFile).get(fileId) dbFile.hidden = False # Any write operation to a project file causes the project to # become dirty. if makeDirty: project = sess.query(model.Project).get(projectId) project.state = defs.Projects.DIRTY else: sess.close() raise Exception('Malformed open mode %s' % mode)