def wrapper(*args, **kwargs): job_id = None if 'job_id' in kwargs: job_id = kwargs['job_id'] else: job_id = args[func.func_code.co_varnames.index('job_id')] logging.debug('args: %s -- kwargs: %s', args, kwargs) job = models.Job.query.get(job_id) logger = get_instance_logger(job.instance) lock = redis.lock('tyr.lock|' + job.instance.name, timeout=self.timeout) if not lock.acquire(blocking=False): logger.info('lock on %s retry %s in 300sec', job.instance.name, func.__name__) task = args[func.func_code.co_varnames.index('self')] task.retry(countdown=60, max_retries=10) else: try: logger.debug('lock acquired on %s for %s', job.instance.name, func.__name__) return func(*args, **kwargs) finally: logger.debug('release lock on %s for %s', job.instance.name, func.__name__) lock.release()
def wrapper(*args, **kwargs): job_id = get_named_arg('job_id', func, args, kwargs) logging.debug('args: %s -- kwargs: %s', args, kwargs) job = models.Job.query.get(job_id) logger = get_instance_logger(job.instance, task_id=job_id) task = args[func.func_code.co_varnames.index('self')] try: lock = redis.lock('tyr.lock|' + job.instance.name, timeout=self.timeout) locked = lock.acquire(blocking=False) except ConnectionError: logging.exception('Exception with redis while locking. Retrying in 10sec') task.retry(countdown=10, max_retries=10) if not locked: countdown = 300 logger.info('lock on %s retry %s in %s sec', job.instance.name, func.__name__, countdown) task.retry(countdown=countdown, max_retries=10) else: try: logger.debug('lock acquired on %s for %s', job.instance.name, func.__name__) return func(*args, **kwargs) finally: logger.debug('release lock on %s for %s', job.instance.name, func.__name__) # sometimes we are disconnected from redis when we want to release the lock, # so we retry only the release try: retrying.Retrying(stop_max_attempt_number=5, wait_fixed=1000).call( lock_release, lock, logger ) except ValueError: # LockError(ValueError) since redis 3.0 logger.exception( "impossible to release lock: continue but following task may be locked :(" )
def fare2ed(instance_config, filename, job_id): """ launch fare2ed """ job = models.Job.query.get(job_id) instance = job.instance lock = redis.lock('tyr.lock|' + instance.name) if not lock.acquire(blocking=False): fare2ed.retry(countdown=300, max_retries=10) logger = get_instance_logger(instance) try: working_directory = os.path.dirname(filename) zip_file = zipfile.ZipFile(filename) zip_file.extractall(path=working_directory) res = launch_exec("fare2ed", ['-f', working_directory, '--connection-string', make_connection_string(instance_config)], logger) if res != 0: #@TODO: exception raise ValueError('fare2ed failed') except: logger.exception('') job.state = 'failed' models.db.session.commit() raise finally: lock.release()
def nav2rt(instance_config, job_id): """ Launch nav2rt""" job = models.Job.query.get(job_id) instance = job.instance lock = redis.lock('tyr.lock|' + instance.name) if not lock.acquire(blocking=False): nav2rt.retry(countdown=300, max_retries=10) logger = get_instance_logger(instance) try: source_filename = instance_config.tmp_file target_filename = instance_config.target_file connection_string = make_connection_string(instance_config) res = launch_exec('nav2rt', ["-i", source_filename, "-o", target_filename, "--connection-string", connection_string], logger) if res != 0: raise ValueError('nav2rt failed') except: logger.exception('') job.state = 'failed' models.db.session.commit() raise finally: lock.release()
def synonym2ed(instance_config, filename, job_id): """ launch synonym2ed """ job = models.Job.query.get(job_id) instance = job.instance lock = redis.lock('tyr.lock|' + instance.name) if not lock.acquire(blocking=False): synonym2ed.retry(countdown=300, max_retries=10) logger = get_instance_logger(instance) try: connection_string = make_connection_string(instance_config) res = launch_exec('synonym2ed', ["-i", filename, "--connection-string", connection_string], logger) if res != 0: #@TODO: exception raise ValueError('synonym2ed failed') except: logger.exception('') job.state = 'failed' models.db.session.commit() raise finally: lock.release()
def fusio2ed(instance_config, filename, job_id): """ Unzip fusio file and launch fusio2ed """ job = models.Job.query.get(job_id) instance = job.instance lock = redis.lock('tyr.lock|' + instance.name) if not lock.acquire(blocking=False): fusio2ed.retry(countdown=300, max_retries=10) logger = get_instance_logger(instance) try: working_directory = os.path.dirname(filename) zip_file = zipfile.ZipFile(filename) zip_file.extractall(path=working_directory) params = ["-i", working_directory] if instance_config.aliases_file: params.append("-a") params.append(instance_config.aliases_file) if instance_config.synonyms_file: params.append("-s") params.append(instance_config.synonyms_file) connection_string = make_connection_string(instance_config) params.append("--connection-string") params.append(connection_string) res = launch_exec("fusio2ed", params, logger) if res != 0: raise ValueError('fusio2ed failed') except: logger.exception('') job.state = 'failed' models.db.session.commit() raise finally: lock.release()