def mutex(self, args, kwargs, delete=False):
'''Creates the mutex locks and yields the mutex status.'''
conf = self.app.conf
global_timeout = getattr(conf, 'MUTEX_TIMEOUT', None)
items = getcallargs(self.run, *args, **kwargs)
timeout = items.get('mutex_timeout') or global_timeout or 3600
success = False
try:
hosts = getattr(conf, 'ZOOKEEPER_HOSTS', '127.0.0.1:2181')
client = kazoo.client.KazooClient(hosts=hosts)
client.start()
lock_node = self._get_node(args, kwargs)
if client.exists(lock_node):
if time.time() - client.get(lock_node)[1].created > timeout:
client.delete(lock_node)
success = True
else:
success = True
except kazoo.exceptions.KazooException:
yield False
else:
if success:
client.create(lock_node, makepath=True)
yield True
if delete:
client.delete(lock_node)
else:
yield False
finally:
try:
client.stop()
client.close()
except kazoo.exceptions.KazooException:
pass
def mutex(self, args, kwargs, delete=False):
'''Creates the mutex locks and yields the mutex status.'''
conf = self.app.conf
global_timeout = getattr(conf, 'MUTEX_TIMEOUT', None)
items = getcallargs(self.run, *args, **kwargs)
timeout = items.get('mutex_timeout') or global_timeout or 3600
success = False
try:
hosts = getattr(conf, 'ZOOKEEPER_HOSTS', '127.0.0.1:2181')
client = kazoo.client.KazooClient(hosts=hosts)
client.start()
lock_node = self._get_node(args, kwargs)
if client.exists(lock_node):
if time.time() - client.get(lock_node)[1].created > timeout:
client.delete(lock_node)
success = True
else:
success = True
except kazoo.exceptions.KazooException:
yield False
else:
if success:
client.create(lock_node, makepath=True)
yield True
if delete:
client.delete(lock_node)
else:
yield False
finally:
try:
client.stop()
client.close()
except kazoo.exceptions.KazooException:
pass
def get_or_create_node(self,
path,
value='',
acl=None,
ephemeral=False,
sequence=False,
makepath=False):
client = None
try:
client = self.get_kazoo_client()
client.start()
if not client.exists(path):
client.create(path,
value=value,
acl=acl,
ephemeral=ephemeral,
sequence=sequence,
makepath=makepath)
return client.get(path)
except Exception as ex:
raise ex
finally:
if client:
client.stop()
client.close()
def get_or_create_node(self,
path,
value='',
acl=None,
ephemeral=False,
sequence=False,
makepath=False):
client = None
hosts = getattr(self.app.conf, 'ZOOKEEPER_HOSTS', '127.0.0.1:2181')
try:
client = kazoo.client.KazooClient(hosts=hosts)
client.start()
if not client.exists(path):
client.create(path,
value=value,
acl=acl,
ephemeral=ephemeral,
sequence=sequence,
makepath=makepath)
return client.get(path)
except Exception as ex:
raise ex
finally:
if client:
client.stop()
client.close()
def init(client, fatal=False):
for path in (
INPUT_PATH,
READY_PATH,
RUNNING_PATH,
CONTROL_JOBS_PATH,
HEAD_PATH,
JOBS_COUNTER_PATH,
join(STATE_PATH, STATE_SPLITTER),
join(STATE_PATH, STATE_WORKER),
join(STATE_PATH, STATE_COLLECTOR),
USER_PATH,
):
try:
client.create(path, makepath=True)
_logger.info("Created zoo path: %s", path)
except NodeExistsError:
level = ( logging.ERROR if fatal else logging.DEBUG )
_logger.log(level, "Zoo path is already exists: %s", path)
if fatal:
raise
def init(client, fatal=False):
for path in (
INPUT_PATH,
READY_PATH,
RUNNING_PATH,
CONTROL_JOBS_PATH,
HEAD_PATH,
JOBS_COUNTER_PATH,
join(STATE_PATH, STATE_SPLITTER),
join(STATE_PATH, STATE_WORKER),
join(STATE_PATH, STATE_COLLECTOR),
USER_PATH,
):
try:
client.create(path, makepath=True)
_logger.info("Created zoo path: %s", path)
except NodeExistsError:
level = (logging.ERROR if fatal else logging.DEBUG)
_logger.log(level, "Zoo path is already exists: %s", path)
if fatal:
raise
def mutex(self, args, kwargs, delete=False):
'''Creates the mutex locks and yields the mutex status.'''
conf = self._get_app().conf
global_timeout = getattr(conf, 'MUTEX_TIMEOUT', None)
items = inspect.getcallargs(self.run, *args, **kwargs)
timeout = items.get('mutex_timeout') or global_timeout or 3600
success = False
try:
hosts = conf.ZOOKEEPER_HOSTS
client = kazoo.client.KazooClient(hosts=hosts)
client.start()
lock_node = self._get_node(args, kwargs)
if client.exists(lock_node):
if time.time() - float(client.get(lock_node)[0]) > timeout:
client.delete(lock_node)
success = True
else:
success = True
except kazoo.exceptions.KazooException as exc:
logger.exception('Error stopping execution')
yield False
else:
if success:
client.create(lock_node, six.text_type(time.time()).encode(), makepath=True)
yield True
if delete:
client.delete(lock_node)
else:
logger.debug('This task has been locked.')
yield False
finally:
try:
client.stop()
client.close()
except kazoo.exceptions.KazooException as exc:
pass
def maybe_create(client, path):
try:
client.create(path)
except kazoo.exceptions.NodeExistsError:
pass