def setup_lock(self):
"""Create lockfile.
Note that this will create a lockfile even if you don't actually
use it. GlobalLock.__del__ is meant to clean it up though.
"""
self.lock = GlobalLock(self.lockfilepath, logger=self.logger)
def testTopLevelLockFile(self):
"""Try a simple process-upload run.
Expect it to exit earlier due the occupied lockfile
"""
# acquire the process-upload lockfile locally
from contrib.glock import GlobalLock
locker = GlobalLock('/var/lock/process-upload-insecure.lock')
locker.acquire()
returncode, out, err = self.runProcessUpload(
extra_args=['-C', 'insecure'])
# the process-upload call terminated with ERROR and
# proper log message
self.assertEqual(1, returncode)
self.assert_(
'INFO Creating lockfile: '
'/var/lock/process-upload-insecure.lock' in err.splitlines())
self.assert_(
'INFO Lockfile /var/lock/process-upload-insecure.lock in use'
in err.splitlines())
# release the locally acquired lockfile
locker.release()
def setup_lock(self):
"""Create lockfile.
Note that this will create a lockfile even if you don't actually
use it. GlobalLock.__del__ is meant to clean it up though.
"""
self.lock = GlobalLock(self.lockfilepath, logger=self.logger)
def locateDirectories(self, fsroot):
"""Return a list of upload directories in a given queue.
This method operates on the queue atomically, i.e. it suppresses
changes in the queue directory, like new uploads, by acquiring
the shared upload_queue lockfile while the directory are listed.
:param fsroot: path to a 'queue' directory to be inspected.
:return: a list of upload directories found in the queue
alphabetically sorted.
"""
# Protecting listdir by a lock ensures that we only get completely
# finished directories listed. See lp.poppy.hooks for the other
# locking place.
lockfile_path = os.path.join(fsroot, ".lock")
fsroot_lock = GlobalLock(lockfile_path)
mode = stat.S_IMODE(os.stat(lockfile_path).st_mode)
# XXX cprov 20081024 bug=185731: The lockfile permission can only be
# changed by its owner. Since we can't predict which process will
# create it in production systems we simply ignore errors when trying
# to grant the right permission. At least, one of the process will
# be able to do so.
try:
os.chmod(lockfile_path, mode | stat.S_IWGRP)
except OSError as err:
self.log.debug('Could not fix the lockfile permission: %s' % err)
try:
fsroot_lock.acquire(blocking=True)
dir_names = os.listdir(fsroot)
finally:
# Skip lockfile deletion, see similar code in lp.poppy.hooks.
fsroot_lock.release(skip_delete=True)
sorted_dir_names = sorted(
dir_name
for dir_name in dir_names
if os.path.isdir(os.path.join(fsroot, dir_name)))
return sorted_dir_names
def testTopLevelLockFile(self):
"""Try a simple process-upload run.
Expect it to exit earlier due the occupied lockfile
"""
# acquire the process-upload lockfile locally
from contrib.glock import GlobalLock
locker = GlobalLock('/var/lock/process-upload-insecure.lock')
locker.acquire()
returncode, out, err = self.runProcessUpload(
extra_args=['-C', 'insecure'])
# the process-upload call terminated with ERROR and
# proper log message
self.assertEqual(1, returncode)
self.assert_(
'INFO Creating lockfile: '
'/var/lock/process-upload-insecure.lock' in err.splitlines())
self.assert_(
'INFO Lockfile /var/lock/process-upload-insecure.lock in use' in
err.splitlines())
# release the locally acquired lockfile
locker.release()
def locateDirectories(self, fsroot):
"""Return a list of upload directories in a given queue.
This method operates on the queue atomically, i.e. it suppresses
changes in the queue directory, like new uploads, by acquiring
the shared upload_queue lockfile while the directory are listed.
:param fsroot: path to a 'queue' directory to be inspected.
:return: a list of upload directories found in the queue
alphabetically sorted.
"""
# Protecting listdir by a lock ensures that we only get completely
# finished directories listed. See lp.poppy.hooks for the other
# locking place.
lockfile_path = os.path.join(fsroot, ".lock")
fsroot_lock = GlobalLock(lockfile_path)
mode = stat.S_IMODE(os.stat(lockfile_path).st_mode)
# XXX cprov 20081024 bug=185731: The lockfile permission can only be
# changed by its owner. Since we can't predict which process will
# create it in production systems we simply ignore errors when trying
# to grant the right permission. At least, one of the process will
# be able to do so.
try:
os.chmod(lockfile_path, mode | stat.S_IWGRP)
except OSError as err:
self.log.debug('Could not fix the lockfile permission: %s' % err)
try:
fsroot_lock.acquire(blocking=True)
dir_names = os.listdir(fsroot)
finally:
# Skip lockfile deletion, see similar code in lp.poppy.hooks.
fsroot_lock.release(skip_delete=True)
sorted_dir_names = sorted(
dir_name for dir_name in dir_names
if os.path.isdir(os.path.join(fsroot, dir_name)))
return sorted_dir_names
class LaunchpadScript:
"""A base class for runnable scripts and cronscripts.
Inherit from this base class to simplify the setup work that your
script needs to do.
What you define:
- main()
- add_my_options(), if you have any
- usage and description, if you want output for --help
What you call:
- lock_and_run()
If you are picky:
- lock_or_die()
- run()
- unlock()
- build_options()
What you get:
- self.logger
- self.txn
- self.parser (the OptionParser)
- self.options (the parsed options)
"Give me convenience or give me death."
"""
lock = None
txn = None
usage = None
description = None
lockfilepath = None
loglevel = logging.INFO
# State for the log_unhandled_exceptions decorator.
_log_unhandled_exceptions_level = 0
def __init__(self, name=None, dbuser=None, test_args=None, logger=None):
"""Construct new LaunchpadScript.
Name is a short name for this script; it will be used to
assemble a lock filename and to identify the logger object.
Use dbuser to specify the user to connect to the database; if
not supplied a default will be used.
Specify test_args when you want to override sys.argv. This is
useful in test scripts.
:param logger: Use this logger, instead of initializing global
logging.
"""
if name is None:
self._name = self.__class__.__name__.lower()
else:
self._name = name
self._dbuser = dbuser
self.logger = logger
# The construction of the option parser is a bit roundabout, but
# at least it's isolated here. First we build the parser, then
# we add options that our logger object uses, then call our
# option-parsing hook, and finally pull out and store the
# supplied options and args.
if self.description is None:
description = self.__doc__
else:
description = self.description
self.parser = OptionParser(usage=self.usage,
description=description)
if logger is None:
scripts.logger_options(self.parser, default=self.loglevel)
self.parser.add_option(
'--profile', dest='profile', metavar='FILE', help=(
"Run the script under the profiler and save the "
"profiling stats in FILE."))
else:
scripts.dummy_logger_options(self.parser)
self.add_my_options()
self.options, self.args = self.parser.parse_args(args=test_args)
# Enable subclasses to easily override these __init__()
# arguments using command-line arguments.
self.handle_options()
def handle_options(self):
if self.logger is None:
self.logger = scripts.logger(self.options, self.name)
@property
def name(self):
"""Enable subclasses to override with command-line arguments."""
return self._name
@property
def dbuser(self):
"""Enable subclasses to override with command-line arguments."""
return self._dbuser
#
# Hooks that we expect users to redefine.
#
def main(self):
"""Define the meat of your script here. Must be defined.
Raise LaunchpadScriptFailure if you encounter an error condition
that makes it impossible for you to proceed; sys.exit(1) will be
invoked in that situation.
"""
raise NotImplementedError
def add_my_options(self):
"""Optionally customize this hook to define your own options.
This method should contain only a set of lines that follow the
template:
self.parser.add_option("-f", "--foo", dest="foo",
default="foobar-makes-the-world-go-round",
help="You are joking, right?")
"""
#
# Convenience or death
#
@log_unhandled_exception_and_exit
def login(self, user=ANONYMOUS):
"""Super-convenience method that avoids the import."""
setupInteractionByEmail(user)
#
# Locking and running methods. Users only call these explicitly if
# they really want to control the run-and-locking semantics of the
# script carefully.
#
@property
def lockfilename(self):
"""Return lockfilename.
May be overridden in targeted scripts in order to have more specific
lockfilename.
"""
return "launchpad-%s.lock" % self.name
@property
def lockfilepath(self):
return os.path.join(LOCK_PATH, self.lockfilename)
def setup_lock(self):
"""Create lockfile.
Note that this will create a lockfile even if you don't actually
use it. GlobalLock.__del__ is meant to clean it up though.
"""
self.lock = GlobalLock(self.lockfilepath, logger=self.logger)
@log_unhandled_exception_and_exit
def lock_or_die(self, blocking=False):
"""Attempt to lock, and sys.exit(1) if the lock's already taken.
Say blocking=True if you want to block on the lock being
available.
"""
self.setup_lock()
try:
self.lock.acquire(blocking=blocking)
except LockAlreadyAcquired:
self.logger.info('Lockfile %s in use' % self.lockfilepath)
sys.exit(1)
@log_unhandled_exception_and_exit
def unlock(self, skip_delete=False):
"""Release the lock. Do this before going home.
If you skip_delete, we won't try to delete the lock when it's
freed. This is useful if you have moved the directory in which
the lockfile resides.
"""
self.lock.release(skip_delete=skip_delete)
@log_unhandled_exception_and_exit
def run(self, use_web_security=False, isolation=None):
"""Actually run the script, executing zcml and initZopeless."""
if isolation is None:
isolation = 'read_committed'
self._init_zca(use_web_security=use_web_security)
self._init_db(isolation=isolation)
# XXX wgrant 2011-09-24 bug=29744: initZopeless used to do this.
# Should be called directly by scripts that actually need it.
set_immediate_mail_delivery(True)
date_started = datetime.datetime.now(UTC)
profiler = None
if self.options.profile:
profiler = Profile()
original_feature_controller = get_relevant_feature_controller()
install_feature_controller(make_script_feature_controller(self.name))
try:
if profiler:
profiler.runcall(self.main)
else:
self.main()
except LaunchpadScriptFailure as e:
self.logger.error(str(e))
sys.exit(e.exit_status)
except SilentLaunchpadScriptFailure as e:
sys.exit(e.exit_status)
else:
date_completed = datetime.datetime.now(UTC)
self.record_activity(date_started, date_completed)
finally:
install_feature_controller(original_feature_controller)
if profiler:
profiler.dump_stats(self.options.profile)
def _init_zca(self, use_web_security):
"""Initialize the ZCA, this can be overridden for testing purposes."""
scripts.execute_zcml_for_scripts(use_web_security=use_web_security)
def _init_db(self, isolation):
"""Initialize the database transaction.
Can be overridden for testing purposes.
"""
dbuser = self.dbuser
if dbuser is None:
connstr = ConnectionString(dbconfig.main_master)
dbuser = connstr.user or dbconfig.dbuser
dbconfig.override(dbuser=dbuser, isolation_level=isolation)
self.txn = transaction
def record_activity(self, date_started, date_completed):
"""Hook to record script activity."""
#
# Make things happen
#
@log_unhandled_exception_and_exit
def lock_and_run(self, blocking=False, skip_delete=False,
use_web_security=False,
isolation='read_committed'):
"""Call lock_or_die(), and then run() the script.
Will die with sys.exit(1) if the locking call fails.
"""
self.lock_or_die(blocking=blocking)
try:
self.run(use_web_security=use_web_security, isolation=isolation)
finally:
self.unlock(skip_delete=skip_delete)
def client_done_hook(self, fsroot, host, port):
"""A client has completed. If it authenticated then it stands a chance
of having uploaded a file to the set. If not; then it is simply an
aborted transaction and we remove the fsroot."""
if fsroot not in self.clients:
raise PoppyInterfaceFailure("Unable to find fsroot in client set")
self.logger.debug("Processing session complete in %s" % fsroot)
client = self.clients[fsroot]
if "distro" not in client:
# Login username defines the distribution context of the upload.
# So abort unauthenticated sessions by removing its contents
shutil.rmtree(fsroot)
return
# Protect from race condition between creating the directory
# and creating the distro file, and also in cases where the
# temporary directory and the upload directory are not in the
# same filesystem (non-atomic "rename").
lockfile_path = os.path.join(self.targetpath, ".lock")
self.lock = GlobalLock(lockfile_path)
# XXX cprov 20071024 bug=156795: We try to acquire the lock as soon
# as possible after creating the lockfile but are still open to
# a race.
self.lock.acquire(blocking=True)
mode = stat.S_IMODE(os.stat(lockfile_path).st_mode)
# XXX cprov 20081024 bug=185731: The lockfile permission can only be
# changed by its owner. Since we can't predict which process will
# create it in production systems we simply ignore errors when trying
# to grant the right permission. At least, one of the process will
# be able to do so.
try:
os.chmod(lockfile_path, mode | stat.S_IWGRP)
except OSError:
pass
try:
timestamp = time.strftime("%Y%m%d-%H%M%S")
path = "upload%s-%s-%06d" % (self.prefix, timestamp,
self.targetcount)
target_fsroot = os.path.join(self.targetpath, path)
# Create file to store the distro used.
self.logger.debug("Upload was targetted at %s" % client["distro"])
distro_filename = target_fsroot + ".distro"
distro_file = open(distro_filename, "w")
distro_file.write(client["distro"])
distro_file.close()
# Move the session directory to the target directory.
if os.path.exists(target_fsroot):
self.logger.warn("Targeted upload already present: %s" % path)
self.logger.warn("System clock skewed ?")
else:
try:
shutil.move(fsroot, target_fsroot)
except (OSError, IOError):
if not os.path.exists(target_fsroot):
raise
# XXX cprov 20071024: We should replace os.system call by os.chmod
# and fix the default permission value accordingly in poppy-upload
if self.perms is not None:
os.system("chmod %s -R %s" % (self.perms, target_fsroot))
# Invoke processing script, if provided.
if self.cmd:
cmd = self.cmd
cmd = cmd.replace("@fsroot@", target_fsroot)
cmd = cmd.replace("@distro@", client["distro"])
self.logger.debug("Running upload handler: %s" % cmd)
os.system(cmd)
finally:
# We never delete the lockfile, this way the inode will be
# constant while the machine is up. See comment on 'acquire'
self.lock.release(skip_delete=True)
self.clients.pop(fsroot)
# This is mainly done so that tests know when the
# post-processing hook has finished.
self.logger.info(self.LOG_MAGIC)
class LaunchpadScript:
"""A base class for runnable scripts and cronscripts.
Inherit from this base class to simplify the setup work that your
script needs to do.
What you define:
- main()
- add_my_options(), if you have any
- usage and description, if you want output for --help
What you call:
- lock_and_run()
If you are picky:
- lock_or_die()
- run()
- unlock()
- build_options()
What you get:
- self.logger
- self.txn
- self.parser (the OptionParser)
- self.options (the parsed options)
"Give me convenience or give me death."
"""
lock = None
txn = None
usage = None
description = None
lockfilepath = None
loglevel = logging.INFO
# State for the log_unhandled_exceptions decorator.
_log_unhandled_exceptions_level = 0
def __init__(self, name=None, dbuser=None, test_args=None, logger=None):
"""Construct new LaunchpadScript.
Name is a short name for this script; it will be used to
assemble a lock filename and to identify the logger object.
Use dbuser to specify the user to connect to the database; if
not supplied a default will be used.
Specify test_args when you want to override sys.argv. This is
useful in test scripts.
:param logger: Use this logger, instead of initializing global
logging.
"""
if name is None:
self._name = self.__class__.__name__.lower()
else:
self._name = name
self._dbuser = dbuser
self.logger = logger
# The construction of the option parser is a bit roundabout, but
# at least it's isolated here. First we build the parser, then
# we add options that our logger object uses, then call our
# option-parsing hook, and finally pull out and store the
# supplied options and args.
if self.description is None:
description = self.__doc__
else:
description = self.description
self.parser = OptionParser(usage=self.usage,
description=description)
if logger is None:
scripts.logger_options(self.parser, default=self.loglevel)
self.parser.add_option(
'--profile', dest='profile', metavar='FILE', help=(
"Run the script under the profiler and save the "
"profiling stats in FILE."))
else:
scripts.dummy_logger_options(self.parser)
self.add_my_options()
self.options, self.args = self.parser.parse_args(args=test_args)
# Enable subclasses to easily override these __init__()
# arguments using command-line arguments.
self.handle_options()
def handle_options(self):
if self.logger is None:
self.logger = scripts.logger(self.options, self.name)
@property
def name(self):
"""Enable subclasses to override with command-line arguments."""
return self._name
@property
def dbuser(self):
"""Enable subclasses to override with command-line arguments."""
return self._dbuser
#
# Hooks that we expect users to redefine.
#
def main(self):
"""Define the meat of your script here. Must be defined.
Raise LaunchpadScriptFailure if you encounter an error condition
that makes it impossible for you to proceed; sys.exit(1) will be
invoked in that situation.
"""
raise NotImplementedError
def add_my_options(self):
"""Optionally customize this hook to define your own options.
This method should contain only a set of lines that follow the
template:
self.parser.add_option("-f", "--foo", dest="foo",
default="foobar-makes-the-world-go-round",
help="You are joking, right?")
"""
#
# Convenience or death
#
@log_unhandled_exception_and_exit
def login(self, user=ANONYMOUS):
"""Super-convenience method that avoids the import."""
setupInteractionByEmail(user)
#
# Locking and running methods. Users only call these explicitly if
# they really want to control the run-and-locking semantics of the
# script carefully.
#
@property
def lockfilename(self):
"""Return lockfilename.
May be overridden in targeted scripts in order to have more specific
lockfilename.
"""
return "launchpad-%s.lock" % self.name
@property
def lockfilepath(self):
return os.path.join(LOCK_PATH, self.lockfilename)
def setup_lock(self):
"""Create lockfile.
Note that this will create a lockfile even if you don't actually
use it. GlobalLock.__del__ is meant to clean it up though.
"""
self.lock = GlobalLock(self.lockfilepath, logger=self.logger)
@log_unhandled_exception_and_exit
def lock_or_die(self, blocking=False):
"""Attempt to lock, and sys.exit(1) if the lock's already taken.
Say blocking=True if you want to block on the lock being
available.
"""
self.setup_lock()
try:
self.lock.acquire(blocking=blocking)
except LockAlreadyAcquired:
self.logger.info('Lockfile %s in use' % self.lockfilepath)
sys.exit(1)
@log_unhandled_exception_and_exit
def unlock(self, skip_delete=False):
"""Release the lock. Do this before going home.
If you skip_delete, we won't try to delete the lock when it's
freed. This is useful if you have moved the directory in which
the lockfile resides.
"""
self.lock.release(skip_delete=skip_delete)
@log_unhandled_exception_and_exit
def run(self, use_web_security=False, isolation=None):
"""Actually run the script, executing zcml and initZopeless."""
if isolation is None:
isolation = 'read_committed'
self._init_zca(use_web_security=use_web_security)
self._init_db(isolation=isolation)
# XXX wgrant 2011-09-24 bug=29744: initZopeless used to do this.
# Should be called directly by scripts that actually need it.
set_immediate_mail_delivery(True)
date_started = datetime.datetime.now(UTC)
profiler = None
if self.options.profile:
profiler = Profile()
original_feature_controller = get_relevant_feature_controller()
install_feature_controller(make_script_feature_controller(self.name))
try:
if profiler:
profiler.runcall(self.main)
else:
self.main()
except LaunchpadScriptFailure as e:
self.logger.error(str(e))
sys.exit(e.exit_status)
except SilentLaunchpadScriptFailure as e:
sys.exit(e.exit_status)
else:
date_completed = datetime.datetime.now(UTC)
self.record_activity(date_started, date_completed)
finally:
install_feature_controller(original_feature_controller)
if profiler:
profiler.dump_stats(self.options.profile)
def _init_zca(self, use_web_security):
"""Initialize the ZCA, this can be overridden for testing purposes."""
scripts.execute_zcml_for_scripts(use_web_security=use_web_security)
def _init_db(self, isolation):
"""Initialize the database transaction.
Can be overridden for testing purposes.
"""
dbuser = self.dbuser
if dbuser is None:
connstr = ConnectionString(dbconfig.main_master)
dbuser = connstr.user or dbconfig.dbuser
dbconfig.override(dbuser=dbuser, isolation_level=isolation)
self.txn = transaction
def record_activity(self, date_started, date_completed):
"""Hook to record script activity."""
#
# Make things happen
#
@log_unhandled_exception_and_exit
def lock_and_run(self, blocking=False, skip_delete=False,
use_web_security=False,
isolation='read_committed'):
"""Call lock_or_die(), and then run() the script.
Will die with sys.exit(1) if the locking call fails.
"""
self.lock_or_die(blocking=blocking)
try:
self.run(use_web_security=use_web_security, isolation=isolation)
finally:
self.unlock(skip_delete=skip_delete)
def lock(self):
self.actualLock = GlobalLock(self.lockfilename)
try:
self.actualLock.acquire()
except LockAlreadyAcquired:
raise LockError(self.lockfilename)
class JobScheduler:
"""Schedule and manage the mirroring of branches.
The jobmanager is responsible for organizing the mirroring of all
branches.
"""
def __init__(self, codehosting_endpoint, logger, branch_type_names):
self.codehosting_endpoint = codehosting_endpoint
self.logger = logger
self.branch_type_names = branch_type_names
self.actualLock = None
self.name = 'branch-puller'
self.lockfilename = '/var/lock/launchpad-%s.lock' % self.name
def _turnJobTupleIntoTask(self, job_tuple):
"""Turn the return value of `acquireBranchToPull` into a job.
`IBranchPuller.acquireBranchToPull` returns either an empty tuple
(indicating there are no branches to pull currently) or a tuple of 6
arguments, which are more or less those needed to construct a
`PullerMaster` object.
"""
if len(job_tuple) == 0:
return None
(branch_id, pull_url, unique_name, default_stacked_on_url,
branch_type_name) = job_tuple
master = PullerMaster(branch_id, pull_url, unique_name,
branch_type_name, default_stacked_on_url,
self.logger, self.codehosting_endpoint)
return master.run
def _poll(self):
deferred = self.codehosting_endpoint.callRemote(
'acquireBranchToPull', self.branch_type_names)
deferred.addCallback(self._turnJobTupleIntoTask)
return deferred
def run(self):
consumer = ParallelLimitedTaskConsumer(
config.supermirror.maximum_workers, logger=self.logger)
self.consumer = consumer
source = PollingTaskSource(config.supermirror.polling_interval,
self._poll,
logger=self.logger)
deferred = consumer.consume(source)
deferred.addCallback(self._finishedRunning)
return deferred
def _finishedRunning(self, ignored):
self.logger.info('Mirroring complete')
return ignored
def lock(self):
self.actualLock = GlobalLock(self.lockfilename)
try:
self.actualLock.acquire()
except LockAlreadyAcquired:
raise LockError(self.lockfilename)
def unlock(self):
self.actualLock.release()
def recordActivity(self, date_started, date_completed):
"""Record successful completion of the script."""
started_tuple = tuple(date_started.utctimetuple())
completed_tuple = tuple(date_completed.utctimetuple())
return self.codehosting_endpoint.callRemote('recordSuccess', self.name,
socket.gethostname(),
started_tuple,
completed_tuple)
def client_done_hook(self, fsroot, host, port):
"""A client has completed. If it authenticated then it stands a chance
of having uploaded a file to the set. If not; then it is simply an
aborted transaction and we remove the fsroot."""
if fsroot not in self.clients:
raise PoppyInterfaceFailure("Unable to find fsroot in client set")
self.logger.debug("Processing session complete in %s" % fsroot)
client = self.clients[fsroot]
if "distro" not in client:
# Login username defines the distribution context of the upload.
# So abort unauthenticated sessions by removing its contents
shutil.rmtree(fsroot)
return
# Protect from race condition between creating the directory
# and creating the distro file, and also in cases where the
# temporary directory and the upload directory are not in the
# same filesystem (non-atomic "rename").
lockfile_path = os.path.join(self.targetpath, ".lock")
self.lock = GlobalLock(lockfile_path)
# XXX cprov 20071024 bug=156795: We try to acquire the lock as soon
# as possible after creating the lockfile but are still open to
# a race.
self.lock.acquire(blocking=True)
mode = stat.S_IMODE(os.stat(lockfile_path).st_mode)
# XXX cprov 20081024 bug=185731: The lockfile permission can only be
# changed by its owner. Since we can't predict which process will
# create it in production systems we simply ignore errors when trying
# to grant the right permission. At least, one of the process will
# be able to do so.
try:
os.chmod(lockfile_path, mode | stat.S_IWGRP)
except OSError:
pass
try:
timestamp = time.strftime("%Y%m%d-%H%M%S")
path = "upload%s-%s-%06d" % (
self.prefix, timestamp, self.targetcount)
target_fsroot = os.path.join(self.targetpath, path)
# Create file to store the distro used.
self.logger.debug("Upload was targetted at %s" % client["distro"])
distro_filename = target_fsroot + ".distro"
distro_file = open(distro_filename, "w")
distro_file.write(client["distro"])
distro_file.close()
# Move the session directory to the target directory.
if os.path.exists(target_fsroot):
self.logger.warn("Targeted upload already present: %s" % path)
self.logger.warn("System clock skewed ?")
else:
try:
shutil.move(fsroot, target_fsroot)
except (OSError, IOError):
if not os.path.exists(target_fsroot):
raise
# XXX cprov 20071024: We should replace os.system call by os.chmod
# and fix the default permission value accordingly in poppy-upload
if self.perms is not None:
os.system("chmod %s -R %s" % (self.perms, target_fsroot))
# Invoke processing script, if provided.
if self.cmd:
cmd = self.cmd
cmd = cmd.replace("@fsroot@", target_fsroot)
cmd = cmd.replace("@distro@", client["distro"])
self.logger.debug("Running upload handler: %s" % cmd)
os.system(cmd)
finally:
# We never delete the lockfile, this way the inode will be
# constant while the machine is up. See comment on 'acquire'
self.lock.release(skip_delete=True)
self.clients.pop(fsroot)
# This is mainly done so that tests know when the
# post-processing hook has finished.
self.logger.info(self.LOG_MAGIC)
def lock(self):
self.actualLock = GlobalLock(self.lockfilename)
try:
self.actualLock.acquire()
except LockAlreadyAcquired:
raise LockError(self.lockfilename)
class JobScheduler:
"""Schedule and manage the mirroring of branches.
The jobmanager is responsible for organizing the mirroring of all
branches.
"""
def __init__(self, codehosting_endpoint, logger, branch_type_names):
self.codehosting_endpoint = codehosting_endpoint
self.logger = logger
self.branch_type_names = branch_type_names
self.actualLock = None
self.name = 'branch-puller'
self.lockfilename = '/var/lock/launchpad-%s.lock' % self.name
def _turnJobTupleIntoTask(self, job_tuple):
"""Turn the return value of `acquireBranchToPull` into a job.
`IBranchPuller.acquireBranchToPull` returns either an empty tuple
(indicating there are no branches to pull currently) or a tuple of 6
arguments, which are more or less those needed to construct a
`PullerMaster` object.
"""
if len(job_tuple) == 0:
return None
(branch_id, pull_url, unique_name,
default_stacked_on_url, branch_type_name) = job_tuple
master = PullerMaster(
branch_id, pull_url, unique_name, branch_type_name,
default_stacked_on_url, self.logger,
self.codehosting_endpoint)
return master.run
def _poll(self):
deferred = self.codehosting_endpoint.callRemote(
'acquireBranchToPull', self.branch_type_names)
deferred.addCallback(self._turnJobTupleIntoTask)
return deferred
def run(self):
consumer = ParallelLimitedTaskConsumer(
config.supermirror.maximum_workers, logger=self.logger)
self.consumer = consumer
source = PollingTaskSource(
config.supermirror.polling_interval, self._poll,
logger=self.logger)
deferred = consumer.consume(source)
deferred.addCallback(self._finishedRunning)
return deferred
def _finishedRunning(self, ignored):
self.logger.info('Mirroring complete')
return ignored
def lock(self):
self.actualLock = GlobalLock(self.lockfilename)
try:
self.actualLock.acquire()
except LockAlreadyAcquired:
raise LockError(self.lockfilename)
def unlock(self):
self.actualLock.release()
def recordActivity(self, date_started, date_completed):
"""Record successful completion of the script."""
started_tuple = tuple(date_started.utctimetuple())
completed_tuple = tuple(date_completed.utctimetuple())
return self.codehosting_endpoint.callRemote(
'recordSuccess', self.name, socket.gethostname(), started_tuple,
completed_tuple)