class ComboLock:
""" A combined process and thread lock.
Arguments:
path (str): path to the lockfile for the lock
"""
def __init__(self, path):
self.path = path
self.plock = InterProcessLock(self.path)
self.tlock = Lock()
def acquire(self, blocking=True):
""" Acquire lock, locks thread and process lock.
Arguments:
blocking(bool): Set's blocking mode of acquire operation.
Default True.
Returns: True if lock succeeded otherwise False
"""
# Create lock file if it doesn't exist and set permissions for
# all users to lock/unlock
if not exists(self.path):
f = open(self.path, 'w+')
f.close()
chmod(self.path, 0o777)
if not blocking:
# Lock thread
tlocked = self.tlock.acquire(blocking=False)
if not tlocked:
return False
# Lock process
plocked = self.plock.acquire(blocking=False)
if not plocked:
# Release thread lock if process couldn't be locked
self.tlock.release()
return False
else: # blocking, just wait and acquire ALL THE LOCKS!!!
self.tlock.acquire()
self.plock.acquire()
return True
def release(self):
""" Release acquired lock. """
self.plock.release()
self.tlock.release()
def __enter__(self):
""" Context handler, acquires lock in blocking mode. """
self.acquire()
return self
def __exit__(self, _type, value, traceback):
""" Releases the lock. """
self.release()
class ComboLock:
""" A combined process and thread lock.
Arguments:
path (str): path to the lockfile for the lock
"""
def __init__(self, path):
# Create lock file if it doesn't exist and set permissions for
# all users to lock/unlock
if not exists(path):
f = open(path, 'w+')
f.close()
chmod(path, 0o777)
self.plock = InterProcessLock(path)
self.tlock = Lock()
def acquire(self, blocking=True):
""" Acquire lock, locks thread and process lock.
Arguments:
blocking(bool): Set's blocking mode of acquire operation.
Default True.
Returns: True if lock succeeded otherwise False
"""
if not blocking:
# Lock thread
tlocked = self.tlock.acquire(blocking=False)
if not tlocked:
return False
# Lock process
plocked = self.plock.acquire(blocking=False)
if not plocked:
# Release thread lock if process couldn't be locked
self.tlock.release()
return False
else: # blocking, just wait and acquire ALL THE LOCKS!!!
self.tlock.acquire()
self.plock.acquire()
return True
def release(self):
""" Release acquired lock. """
self.plock.release()
self.tlock.release()
def __enter__(self):
""" Context handler, acquires lock in blocking mode. """
self.acquire()
return self
def __exit__(self, _type, value, traceback):
""" Releases the lock. """
self.release()
def _get_unused_port(start, end=65536):
"""Get an unused port in the range [start, end) .
Args:
start (int) : port range start
end (int): port range end
"""
process_lock = None
unused_port = None
try:
for port in range(start, end):
process_lock = InterProcessLock(
path='/tmp/socialbot/{}.lock'.format(port))
if not process_lock.acquire(blocking=False):
process_lock.lockfile.close()
process_lock = None
continue
try:
with contextlib.closing(socket.socket()) as sock:
sock.bind(('', port))
unused_port = port
break
except socket.error:
process_lock.release()
process_lock = None
if unused_port is None:
raise socket.error("No unused port in [{}, {})".format(start, end))
yield unused_port
finally:
if process_lock is not None:
process_lock.release()
def __init__(self, start=4000, stop=6000):
self.lock = None
self.bind = None
self.port = None
from fasteners.process_lock import InterProcessLock
import time
while True:
bind = BindFreePort(start, stop)
if bind.port in self.used_ports:
bind.release()
continue
'''
Since we cannot be certain the user will bind the port 'immediately' (actually it is not possible using
this flow. We must ensure that the port will not be reacquired even it is not bound to anything
'''
lock = InterProcessLock(
path='/tmp/socialdna/port_{}_lock'.format(bind.port))
success = lock.acquire(blocking=False)
if success:
self.lock = lock
self.port = bind.port
self.used_ports.add(bind.port)
bind.release()
break
bind.release()
time.sleep(0.01)
class ReducerStats(object):
"""ReducerStats holds stats for the Grizzly reducer.
"""
FILE = "reducer-stats.json"
PATH = grz_tmp("status")
__slots__ = ("_file", "_lock", "error", "failed", "passed")
def __init__(self):
self._file = pathjoin(self.PATH, self.FILE)
self._lock = None
self.error = 0
self.failed = 0
self.passed = 0
def __enter__(self):
self._lock = InterProcessLock("%s.lock" % (self._file, ))
self._lock.acquire()
try:
with open(self._file, "r") as in_fp:
data = load(in_fp)
self.error = data["error"]
self.failed = data["failed"]
self.passed = data["passed"]
except KeyError:
LOG.debug("invalid status data in %r", self._file)
except OSError:
LOG.debug("%r does not exist", self._file)
except ValueError:
LOG.debug("failed to load stats from %r", self._file)
return self
def __exit__(self, exc_type, exc_val, exc_tb):
try:
with open(self._file, "w") as out_fp:
dump(
{
"error": self.error,
"failed": self.failed,
"passed": self.passed
}, out_fp)
finally:
if self._lock:
self._lock.release()
self._lock = None
def get_next(self):
a_lock = None
locked = False
try:
a_lock = InterProcessLock(self.uid_pidlock_filepath)
locked = a_lock.acquire(blocking=True)
if locked:
v = self.load_value()
v_next = v + 1
self.save_value(v_next)
return v_next
finally:
if locked and a_lock is not None:
a_lock.release()
def main():
"""Command line interface for the ``executor`` program."""
# Enable logging to the terminal and system log.
coloredlogs.install(syslog=True)
# Command line option defaults.
command_timeout = 0
exclusive = False
fudge_factor = 0
lock_name = None
lock_timeout = 0
# Parse the command line options.
try:
options, arguments = getopt.getopt(sys.argv[1:], 'eT:l:t:f:vqh', [
'exclusive', 'lock-timeout=', 'lock-file=', 'timeout=',
'fudge-factor=', 'verbose', 'quiet', 'help',
])
for option, value in options:
if option in ('-e', '--exclusive'):
exclusive = True
elif option in ('-T', '--lock-timeout'):
lock_timeout = parse_timespan(value)
elif option in ('-l', '--lock-file'):
lock_name = value
elif option in ('-t', '--timeout'):
command_timeout = parse_timespan(value)
elif option in ('-f', '--fudge-factor'):
fudge_factor = parse_timespan(value)
elif option in ('-v', '--verbose'):
coloredlogs.increase_verbosity()
elif option in ('-q', '--quiet'):
coloredlogs.decrease_verbosity()
elif option in ('-h', '--help'):
usage(__doc__)
sys.exit(0)
else:
assert False, "Unhandled option!"
# Make sure the operator provided a program to execute.
if not arguments:
usage(__doc__)
sys.exit(0)
# Make sure the program actually exists.
program_name = arguments[0]
if not os.path.isfile(program_name):
# Only search the $PATH if the given program name
# doesn't already include one or more path segments.
if program_name == os.path.basename(program_name):
matching_programs = which(program_name)
if matching_programs:
program_name = matching_programs[0]
# The subprocess.Popen() call later on doesn't search the $PATH so we
# make sure to give it the absolute pathname to the program.
arguments[0] = program_name
except Exception as e:
warning("Failed to parse command line arguments: %s", e)
sys.exit(1)
# Apply the requested fudge factor.
apply_fudge_factor(fudge_factor)
# Run the requested command.
try:
if exclusive:
# Select a default lock file name?
if not lock_name:
lock_name = os.path.basename(arguments[0])
logger.debug("Using base name of command as lock file name (%s).", lock_name)
lock_file = get_lock_path(lock_name)
lock = InterProcessLock(path=lock_file, logger=logger)
logger.debug("Trying to acquire exclusive lock: %s", lock_file)
if lock.acquire(blocking=(lock_timeout > 0), max_delay=lock_timeout):
logger.info("Successfully acquired exclusive lock: %s", lock_file)
run_command(arguments, timeout=command_timeout)
else:
logger.error("Failed to acquire exclusive lock: %s", lock_file)
sys.exit(1)
else:
run_command(arguments, timeout=command_timeout)
except ExternalCommandFailed as e:
logger.error("%s", e.error_message)
sys.exit(e.command.returncode)
def main():
"""Command line interface for the ``executor`` program."""
# Enable logging to the terminal and system log.
coloredlogs.install(syslog=True)
# Command line option defaults.
command_timeout = 0
exclusive = False
fudge_factor = 0
lock_name = None
lock_timeout = 0
# Parse the command line options.
try:
options, arguments = getopt.getopt(sys.argv[1:], 'eT:l:t:f:vqh', [
'exclusive', 'lock-timeout=', 'lock-file=', 'timeout=',
'fudge-factor=', 'verbose', 'quiet', 'help',
])
for option, value in options:
if option in ('-e', '--exclusive'):
exclusive = True
elif option in ('-T', '--lock-timeout'):
lock_timeout = parse_timespan(value)
elif option in ('-l', '--lock-file'):
lock_name = value
elif option in ('-t', '--timeout'):
command_timeout = parse_timespan(value)
elif option in ('-f', '--fudge-factor'):
fudge_factor = parse_timespan(value)
elif option in ('-v', '--verbose'):
coloredlogs.increase_verbosity()
elif option in ('-q', '--quiet'):
coloredlogs.decrease_verbosity()
elif option in ('-h', '--help'):
usage(__doc__)
sys.exit(0)
else:
assert False, "Unhandled option!"
# Make sure the operator provided a program to execute.
if not arguments:
usage(__doc__)
sys.exit(0)
# Make sure the program actually exists.
program_name = arguments[0]
if not os.path.isfile(program_name):
# Only search the $PATH if the given program name
# doesn't already include one or more path segments.
if program_name == os.path.basename(program_name):
matching_programs = which(program_name)
if matching_programs:
program_name = matching_programs[0]
# The subprocess.Popen() call later on doesn't search the $PATH so we
# make sure to give it the absolute pathname to the program.
arguments[0] = program_name
except Exception as e:
warning("Failed to parse command line arguments: %s", e)
sys.exit(1)
# Apply the requested fudge factor.
apply_fudge_factor(fudge_factor)
# Run the requested command.
try:
if exclusive:
# Select a default lock file name?
if not lock_name:
lock_name = os.path.basename(arguments[0])
logger.debug("Using base name of command as lock file name (%s).", lock_name)
lock_file = get_lock_path(lock_name)
lock = InterProcessLock(path=lock_file, logger=logger)
logger.debug("Trying to acquire exclusive lock: %s", lock_file)
if lock.acquire(blocking=(lock_timeout > 0), max_delay=lock_timeout):
logger.info("Successfully acquired exclusive lock: %s", lock_file)
run_command(arguments, timeout=command_timeout)
else:
logger.error("Failed to acquire exclusive lock: %s", lock_file)
sys.exit(1)
else:
run_command(arguments, timeout=command_timeout)
except ExternalCommandFailed as e:
logger.error("%s", e.error_message)
sys.exit(e.command.returncode)
if do_collect:
try:
subprocess.check_output((bin_path('collect'), '--commands'),
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
log.error('collect --commands failed:\n{}',
e.output.decode('utf8'))
# We don't want to hose the machine, so even when collecting fails, we
# still wait for the configured interval.
update_stamp(collect_stamp_file)
try:
subprocess.check_output((bin_path('submit'),),
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
log.error('submit failed:\n{}', e.output.decode('utf8'))
else:
update_stamp(submit_stamp_file)
if __name__ == '__main__':
lock_file = os.path.join(var_dir, 'client-cron.lock')
lock = InterProcessLock(lock_file)
# Note: Not necessary to explicitly release the lock, since fasteners
# guarantees that the lock is released when the process exits.
if not lock.acquire(timeout=59, delay=1, max_delay=1):
log.warn('Failed to acquire lock')
sys.exit(1)
main()
if len(sys.argv) > 1:
port = sys.argv[1]
for i in range(2, len(sys.argv)):
if sys.argv[i] == '--shutdown_on_idle':
shutdown_on_idle = True
if sys.argv[i] == '--launch_client':
launch_client = True
if sys.argv[i] == '--debug':
debug = True
app_data_path = Dirs.app_data_dir()
lock_path = Dirs.app_data_dir() + '/lock'
lock = Lock(lock_path)
if lock.acquire(blocking=False):
left_over_lock_files = glob.glob(app_data_path + '/*.lock')
for left_over in left_over_lock_files:
os.remove(left_over)
server = Server(port, shutdown_on_idle=shutdown_on_idle, debug=debug)
server.add_ports_opened_listener(_ports_opened)
server.start()
else:
print('Server already running')
if launch_client:
server_found = False