def testTopLevelLockFile(self): """Try a simple process-upload run. Expect it to exit earlier due the occupied lockfile """ # acquire the process-upload lockfile locally from contrib.glock import GlobalLock locker = GlobalLock('/var/lock/process-upload-insecure.lock') locker.acquire() returncode, out, err = self.runProcessUpload( extra_args=['-C', 'insecure']) # the process-upload call terminated with ERROR and # proper log message self.assertEqual(1, returncode) self.assert_( 'INFO Creating lockfile: ' '/var/lock/process-upload-insecure.lock' in err.splitlines()) self.assert_( 'INFO Lockfile /var/lock/process-upload-insecure.lock in use' in err.splitlines()) # release the locally acquired lockfile locker.release()
def locateDirectories(self, fsroot): """Return a list of upload directories in a given queue. This method operates on the queue atomically, i.e. it suppresses changes in the queue directory, like new uploads, by acquiring the shared upload_queue lockfile while the directory are listed. :param fsroot: path to a 'queue' directory to be inspected. :return: a list of upload directories found in the queue alphabetically sorted. """ # Protecting listdir by a lock ensures that we only get completely # finished directories listed. See lp.poppy.hooks for the other # locking place. lockfile_path = os.path.join(fsroot, ".lock") fsroot_lock = GlobalLock(lockfile_path) mode = stat.S_IMODE(os.stat(lockfile_path).st_mode) # XXX cprov 20081024 bug=185731: The lockfile permission can only be # changed by its owner. Since we can't predict which process will # create it in production systems we simply ignore errors when trying # to grant the right permission. At least, one of the process will # be able to do so. try: os.chmod(lockfile_path, mode | stat.S_IWGRP) except OSError as err: self.log.debug('Could not fix the lockfile permission: %s' % err) try: fsroot_lock.acquire(blocking=True) dir_names = os.listdir(fsroot) finally: # Skip lockfile deletion, see similar code in lp.poppy.hooks. fsroot_lock.release(skip_delete=True) sorted_dir_names = sorted( dir_name for dir_name in dir_names if os.path.isdir(os.path.join(fsroot, dir_name))) return sorted_dir_names
class LaunchpadScript: """A base class for runnable scripts and cronscripts. Inherit from this base class to simplify the setup work that your script needs to do. What you define: - main() - add_my_options(), if you have any - usage and description, if you want output for --help What you call: - lock_and_run() If you are picky: - lock_or_die() - run() - unlock() - build_options() What you get: - self.logger - self.txn - self.parser (the OptionParser) - self.options (the parsed options) "Give me convenience or give me death." """ lock = None txn = None usage = None description = None lockfilepath = None loglevel = logging.INFO # State for the log_unhandled_exceptions decorator. _log_unhandled_exceptions_level = 0 def __init__(self, name=None, dbuser=None, test_args=None, logger=None): """Construct new LaunchpadScript. Name is a short name for this script; it will be used to assemble a lock filename and to identify the logger object. Use dbuser to specify the user to connect to the database; if not supplied a default will be used. Specify test_args when you want to override sys.argv. This is useful in test scripts. :param logger: Use this logger, instead of initializing global logging. """ if name is None: self._name = self.__class__.__name__.lower() else: self._name = name self._dbuser = dbuser self.logger = logger # The construction of the option parser is a bit roundabout, but # at least it's isolated here. First we build the parser, then # we add options that our logger object uses, then call our # option-parsing hook, and finally pull out and store the # supplied options and args. if self.description is None: description = self.__doc__ else: description = self.description self.parser = OptionParser(usage=self.usage, description=description) if logger is None: scripts.logger_options(self.parser, default=self.loglevel) self.parser.add_option( '--profile', dest='profile', metavar='FILE', help=( "Run the script under the profiler and save the " "profiling stats in FILE.")) else: scripts.dummy_logger_options(self.parser) self.add_my_options() self.options, self.args = self.parser.parse_args(args=test_args) # Enable subclasses to easily override these __init__() # arguments using command-line arguments. self.handle_options() def handle_options(self): if self.logger is None: self.logger = scripts.logger(self.options, self.name) @property def name(self): """Enable subclasses to override with command-line arguments.""" return self._name @property def dbuser(self): """Enable subclasses to override with command-line arguments.""" return self._dbuser # # Hooks that we expect users to redefine. # def main(self): """Define the meat of your script here. Must be defined. Raise LaunchpadScriptFailure if you encounter an error condition that makes it impossible for you to proceed; sys.exit(1) will be invoked in that situation. """ raise NotImplementedError def add_my_options(self): """Optionally customize this hook to define your own options. This method should contain only a set of lines that follow the template: self.parser.add_option("-f", "--foo", dest="foo", default="foobar-makes-the-world-go-round", help="You are joking, right?") """ # # Convenience or death # @log_unhandled_exception_and_exit def login(self, user=ANONYMOUS): """Super-convenience method that avoids the import.""" setupInteractionByEmail(user) # # Locking and running methods. Users only call these explicitly if # they really want to control the run-and-locking semantics of the # script carefully. # @property def lockfilename(self): """Return lockfilename. May be overridden in targeted scripts in order to have more specific lockfilename. """ return "launchpad-%s.lock" % self.name @property def lockfilepath(self): return os.path.join(LOCK_PATH, self.lockfilename) def setup_lock(self): """Create lockfile. Note that this will create a lockfile even if you don't actually use it. GlobalLock.__del__ is meant to clean it up though. """ self.lock = GlobalLock(self.lockfilepath, logger=self.logger) @log_unhandled_exception_and_exit def lock_or_die(self, blocking=False): """Attempt to lock, and sys.exit(1) if the lock's already taken. Say blocking=True if you want to block on the lock being available. """ self.setup_lock() try: self.lock.acquire(blocking=blocking) except LockAlreadyAcquired: self.logger.info('Lockfile %s in use' % self.lockfilepath) sys.exit(1) @log_unhandled_exception_and_exit def unlock(self, skip_delete=False): """Release the lock. Do this before going home. If you skip_delete, we won't try to delete the lock when it's freed. This is useful if you have moved the directory in which the lockfile resides. """ self.lock.release(skip_delete=skip_delete) @log_unhandled_exception_and_exit def run(self, use_web_security=False, isolation=None): """Actually run the script, executing zcml and initZopeless.""" if isolation is None: isolation = 'read_committed' self._init_zca(use_web_security=use_web_security) self._init_db(isolation=isolation) # XXX wgrant 2011-09-24 bug=29744: initZopeless used to do this. # Should be called directly by scripts that actually need it. set_immediate_mail_delivery(True) date_started = datetime.datetime.now(UTC) profiler = None if self.options.profile: profiler = Profile() original_feature_controller = get_relevant_feature_controller() install_feature_controller(make_script_feature_controller(self.name)) try: if profiler: profiler.runcall(self.main) else: self.main() except LaunchpadScriptFailure as e: self.logger.error(str(e)) sys.exit(e.exit_status) except SilentLaunchpadScriptFailure as e: sys.exit(e.exit_status) else: date_completed = datetime.datetime.now(UTC) self.record_activity(date_started, date_completed) finally: install_feature_controller(original_feature_controller) if profiler: profiler.dump_stats(self.options.profile) def _init_zca(self, use_web_security): """Initialize the ZCA, this can be overridden for testing purposes.""" scripts.execute_zcml_for_scripts(use_web_security=use_web_security) def _init_db(self, isolation): """Initialize the database transaction. Can be overridden for testing purposes. """ dbuser = self.dbuser if dbuser is None: connstr = ConnectionString(dbconfig.main_master) dbuser = connstr.user or dbconfig.dbuser dbconfig.override(dbuser=dbuser, isolation_level=isolation) self.txn = transaction def record_activity(self, date_started, date_completed): """Hook to record script activity.""" # # Make things happen # @log_unhandled_exception_and_exit def lock_and_run(self, blocking=False, skip_delete=False, use_web_security=False, isolation='read_committed'): """Call lock_or_die(), and then run() the script. Will die with sys.exit(1) if the locking call fails. """ self.lock_or_die(blocking=blocking) try: self.run(use_web_security=use_web_security, isolation=isolation) finally: self.unlock(skip_delete=skip_delete)
class Hooks: clients = {} LOG_MAGIC = "Post-processing finished" _targetcount = 0 def __init__(self, targetpath, logger, allow_user, cmd=None, targetstart=0, perms=None, prefix=''): self.targetpath = targetpath self.logger = logging.getLogger("%s.Hooks" % logger.name) self.cmd = cmd self.allow_user = allow_user self.perms = perms self.prefix = prefix @property def targetcount(self): """A guaranteed unique integer for ensuring unique upload dirs.""" Hooks._targetcount += 1 return Hooks._targetcount def new_client_hook(self, fsroot, host, port): """Prepare a new client record indexed by fsroot...""" self.clients[fsroot] = {"host": host, "port": port} self.logger.debug("Accepting new session in fsroot: %s" % fsroot) self.logger.debug("Session from %s:%s" % (host, port)) def client_done_hook(self, fsroot, host, port): """A client has completed. If it authenticated then it stands a chance of having uploaded a file to the set. If not; then it is simply an aborted transaction and we remove the fsroot.""" if fsroot not in self.clients: raise PoppyInterfaceFailure("Unable to find fsroot in client set") self.logger.debug("Processing session complete in %s" % fsroot) client = self.clients[fsroot] if "distro" not in client: # Login username defines the distribution context of the upload. # So abort unauthenticated sessions by removing its contents shutil.rmtree(fsroot) return # Protect from race condition between creating the directory # and creating the distro file, and also in cases where the # temporary directory and the upload directory are not in the # same filesystem (non-atomic "rename"). lockfile_path = os.path.join(self.targetpath, ".lock") self.lock = GlobalLock(lockfile_path) # XXX cprov 20071024 bug=156795: We try to acquire the lock as soon # as possible after creating the lockfile but are still open to # a race. self.lock.acquire(blocking=True) mode = stat.S_IMODE(os.stat(lockfile_path).st_mode) # XXX cprov 20081024 bug=185731: The lockfile permission can only be # changed by its owner. Since we can't predict which process will # create it in production systems we simply ignore errors when trying # to grant the right permission. At least, one of the process will # be able to do so. try: os.chmod(lockfile_path, mode | stat.S_IWGRP) except OSError: pass try: timestamp = time.strftime("%Y%m%d-%H%M%S") path = "upload%s-%s-%06d" % (self.prefix, timestamp, self.targetcount) target_fsroot = os.path.join(self.targetpath, path) # Create file to store the distro used. self.logger.debug("Upload was targetted at %s" % client["distro"]) distro_filename = target_fsroot + ".distro" distro_file = open(distro_filename, "w") distro_file.write(client["distro"]) distro_file.close() # Move the session directory to the target directory. if os.path.exists(target_fsroot): self.logger.warn("Targeted upload already present: %s" % path) self.logger.warn("System clock skewed ?") else: try: shutil.move(fsroot, target_fsroot) except (OSError, IOError): if not os.path.exists(target_fsroot): raise # XXX cprov 20071024: We should replace os.system call by os.chmod # and fix the default permission value accordingly in poppy-upload if self.perms is not None: os.system("chmod %s -R %s" % (self.perms, target_fsroot)) # Invoke processing script, if provided. if self.cmd: cmd = self.cmd cmd = cmd.replace("@fsroot@", target_fsroot) cmd = cmd.replace("@distro@", client["distro"]) self.logger.debug("Running upload handler: %s" % cmd) os.system(cmd) finally: # We never delete the lockfile, this way the inode will be # constant while the machine is up. See comment on 'acquire' self.lock.release(skip_delete=True) self.clients.pop(fsroot) # This is mainly done so that tests know when the # post-processing hook has finished. self.logger.info(self.LOG_MAGIC) def auth_verify_hook(self, fsroot, user, password): """Verify that the username matches a distribution we care about. The password is irrelevant to auth, as is the fsroot""" if fsroot not in self.clients: raise PoppyInterfaceFailure("Unable to find fsroot in client set") # local authentication self.clients[fsroot]["distro"] = self.allow_user return True
class JobScheduler: """Schedule and manage the mirroring of branches. The jobmanager is responsible for organizing the mirroring of all branches. """ def __init__(self, codehosting_endpoint, logger, branch_type_names): self.codehosting_endpoint = codehosting_endpoint self.logger = logger self.branch_type_names = branch_type_names self.actualLock = None self.name = 'branch-puller' self.lockfilename = '/var/lock/launchpad-%s.lock' % self.name def _turnJobTupleIntoTask(self, job_tuple): """Turn the return value of `acquireBranchToPull` into a job. `IBranchPuller.acquireBranchToPull` returns either an empty tuple (indicating there are no branches to pull currently) or a tuple of 6 arguments, which are more or less those needed to construct a `PullerMaster` object. """ if len(job_tuple) == 0: return None (branch_id, pull_url, unique_name, default_stacked_on_url, branch_type_name) = job_tuple master = PullerMaster(branch_id, pull_url, unique_name, branch_type_name, default_stacked_on_url, self.logger, self.codehosting_endpoint) return master.run def _poll(self): deferred = self.codehosting_endpoint.callRemote( 'acquireBranchToPull', self.branch_type_names) deferred.addCallback(self._turnJobTupleIntoTask) return deferred def run(self): consumer = ParallelLimitedTaskConsumer( config.supermirror.maximum_workers, logger=self.logger) self.consumer = consumer source = PollingTaskSource(config.supermirror.polling_interval, self._poll, logger=self.logger) deferred = consumer.consume(source) deferred.addCallback(self._finishedRunning) return deferred def _finishedRunning(self, ignored): self.logger.info('Mirroring complete') return ignored def lock(self): self.actualLock = GlobalLock(self.lockfilename) try: self.actualLock.acquire() except LockAlreadyAcquired: raise LockError(self.lockfilename) def unlock(self): self.actualLock.release() def recordActivity(self, date_started, date_completed): """Record successful completion of the script.""" started_tuple = tuple(date_started.utctimetuple()) completed_tuple = tuple(date_completed.utctimetuple()) return self.codehosting_endpoint.callRemote('recordSuccess', self.name, socket.gethostname(), started_tuple, completed_tuple)
class Hooks: clients = {} LOG_MAGIC = "Post-processing finished" _targetcount = 0 def __init__(self, targetpath, logger, allow_user, cmd=None, targetstart=0, perms=None, prefix=''): self.targetpath = targetpath self.logger = logging.getLogger("%s.Hooks" % logger.name) self.cmd = cmd self.allow_user = allow_user self.perms = perms self.prefix = prefix @property def targetcount(self): """A guaranteed unique integer for ensuring unique upload dirs.""" Hooks._targetcount += 1 return Hooks._targetcount def new_client_hook(self, fsroot, host, port): """Prepare a new client record indexed by fsroot...""" self.clients[fsroot] = { "host": host, "port": port } self.logger.debug("Accepting new session in fsroot: %s" % fsroot) self.logger.debug("Session from %s:%s" % (host, port)) def client_done_hook(self, fsroot, host, port): """A client has completed. If it authenticated then it stands a chance of having uploaded a file to the set. If not; then it is simply an aborted transaction and we remove the fsroot.""" if fsroot not in self.clients: raise PoppyInterfaceFailure("Unable to find fsroot in client set") self.logger.debug("Processing session complete in %s" % fsroot) client = self.clients[fsroot] if "distro" not in client: # Login username defines the distribution context of the upload. # So abort unauthenticated sessions by removing its contents shutil.rmtree(fsroot) return # Protect from race condition between creating the directory # and creating the distro file, and also in cases where the # temporary directory and the upload directory are not in the # same filesystem (non-atomic "rename"). lockfile_path = os.path.join(self.targetpath, ".lock") self.lock = GlobalLock(lockfile_path) # XXX cprov 20071024 bug=156795: We try to acquire the lock as soon # as possible after creating the lockfile but are still open to # a race. self.lock.acquire(blocking=True) mode = stat.S_IMODE(os.stat(lockfile_path).st_mode) # XXX cprov 20081024 bug=185731: The lockfile permission can only be # changed by its owner. Since we can't predict which process will # create it in production systems we simply ignore errors when trying # to grant the right permission. At least, one of the process will # be able to do so. try: os.chmod(lockfile_path, mode | stat.S_IWGRP) except OSError: pass try: timestamp = time.strftime("%Y%m%d-%H%M%S") path = "upload%s-%s-%06d" % ( self.prefix, timestamp, self.targetcount) target_fsroot = os.path.join(self.targetpath, path) # Create file to store the distro used. self.logger.debug("Upload was targetted at %s" % client["distro"]) distro_filename = target_fsroot + ".distro" distro_file = open(distro_filename, "w") distro_file.write(client["distro"]) distro_file.close() # Move the session directory to the target directory. if os.path.exists(target_fsroot): self.logger.warn("Targeted upload already present: %s" % path) self.logger.warn("System clock skewed ?") else: try: shutil.move(fsroot, target_fsroot) except (OSError, IOError): if not os.path.exists(target_fsroot): raise # XXX cprov 20071024: We should replace os.system call by os.chmod # and fix the default permission value accordingly in poppy-upload if self.perms is not None: os.system("chmod %s -R %s" % (self.perms, target_fsroot)) # Invoke processing script, if provided. if self.cmd: cmd = self.cmd cmd = cmd.replace("@fsroot@", target_fsroot) cmd = cmd.replace("@distro@", client["distro"]) self.logger.debug("Running upload handler: %s" % cmd) os.system(cmd) finally: # We never delete the lockfile, this way the inode will be # constant while the machine is up. See comment on 'acquire' self.lock.release(skip_delete=True) self.clients.pop(fsroot) # This is mainly done so that tests know when the # post-processing hook has finished. self.logger.info(self.LOG_MAGIC) def auth_verify_hook(self, fsroot, user, password): """Verify that the username matches a distribution we care about. The password is irrelevant to auth, as is the fsroot""" if fsroot not in self.clients: raise PoppyInterfaceFailure("Unable to find fsroot in client set") # local authentication self.clients[fsroot]["distro"] = self.allow_user return True
class JobScheduler: """Schedule and manage the mirroring of branches. The jobmanager is responsible for organizing the mirroring of all branches. """ def __init__(self, codehosting_endpoint, logger, branch_type_names): self.codehosting_endpoint = codehosting_endpoint self.logger = logger self.branch_type_names = branch_type_names self.actualLock = None self.name = 'branch-puller' self.lockfilename = '/var/lock/launchpad-%s.lock' % self.name def _turnJobTupleIntoTask(self, job_tuple): """Turn the return value of `acquireBranchToPull` into a job. `IBranchPuller.acquireBranchToPull` returns either an empty tuple (indicating there are no branches to pull currently) or a tuple of 6 arguments, which are more or less those needed to construct a `PullerMaster` object. """ if len(job_tuple) == 0: return None (branch_id, pull_url, unique_name, default_stacked_on_url, branch_type_name) = job_tuple master = PullerMaster( branch_id, pull_url, unique_name, branch_type_name, default_stacked_on_url, self.logger, self.codehosting_endpoint) return master.run def _poll(self): deferred = self.codehosting_endpoint.callRemote( 'acquireBranchToPull', self.branch_type_names) deferred.addCallback(self._turnJobTupleIntoTask) return deferred def run(self): consumer = ParallelLimitedTaskConsumer( config.supermirror.maximum_workers, logger=self.logger) self.consumer = consumer source = PollingTaskSource( config.supermirror.polling_interval, self._poll, logger=self.logger) deferred = consumer.consume(source) deferred.addCallback(self._finishedRunning) return deferred def _finishedRunning(self, ignored): self.logger.info('Mirroring complete') return ignored def lock(self): self.actualLock = GlobalLock(self.lockfilename) try: self.actualLock.acquire() except LockAlreadyAcquired: raise LockError(self.lockfilename) def unlock(self): self.actualLock.release() def recordActivity(self, date_started, date_completed): """Record successful completion of the script.""" started_tuple = tuple(date_started.utctimetuple()) completed_tuple = tuple(date_completed.utctimetuple()) return self.codehosting_endpoint.callRemote( 'recordSuccess', self.name, socket.gethostname(), started_tuple, completed_tuple)