def acquire_master_lock(): ''' Acquire the master lock or raise an exception ''' rc = glider_qc.get_redis_connection() lock = rc.lock('gliderdac:glider_qartod', blocking_timeout=60) return lock
def acquire_master_lock(): ''' Acquire the master lock or raise an exception ''' rc = glider_qc.get_redis_connection() lock = rc.lock('gliderdac:glider_qartod', blocking_timeout=60) return lock
def process(file_paths, config, sync=False): queue = Queue('gliderdac', connection=glider_qc.get_redis_connection()) for nc_path in file_paths: sync_lock() try: glider_qc.log.info("Inspecting %s", nc_path) with Dataset(nc_path, 'r') as nc: if not glider_qc.check_needs_qc(nc): continue glider_qc.log.info("Applying QC to dataset %s", nc_path) if sync: glider_qc.qc_task(nc_path, config) else: queue.enqueue(glider_qc.qc_task, nc_path, config) except Exception as e: glider_qc.log.exception("Failed to check %s for QC", nc_path)
def process(file_paths, config, sync=False): queue = Queue('gliderdac', connection=glider_qc.get_redis_connection()) for nc_path in file_paths: sync_lock() try: glider_qc.log.info("Inspecting %s", nc_path) with Dataset(nc_path, 'r') as nc: if not glider_qc.check_needs_qc(nc): continue glider_qc.log.info("Applying QC to dataset %s", nc_path) if sync: glider_qc.qc_task(nc_path, config) else: queue.enqueue(glider_qc.qc_task, nc_path, config) except Exception as e: glider_qc.log.exception("Failed to check %s for QC", nc_path)
def main(): ''' Apply QARTOD QC to GliderDAC submitted netCDF files Example:: python scripts/glider_qartod.py -c data/qc_config.yml ~/Desktop/revellie/revellie.nc ''' args = get_args() if args.clear: clear_master_lock() return 0 if args.worker: with Connection(glider_qc.get_redis_connection()): worker = Worker(map(Queue, ['gliderdac'])) worker.work() return 0 lock = acquire_master_lock() if not lock.acquire(): raise glider_qc.ProcessError( "Master lock already held by another process") try: file_paths = [] if args.verbose: setup_logging() if args.recursive: file_paths = get_files(args.netcdf_files) else: file_paths = args.netcdf_files if args.config is None: raise ValueError("No configuration found, please set using -c") process(file_paths, args.config, sync=args.sync) finally: lock.release() return 0
def main(): ''' Apply QARTOD QC to GliderDAC submitted netCDF files Example:: python scripts/glider_qartod.py -c data/qc_config.yml ~/Desktop/revellie/revellie.nc ''' args = get_args() if args.clear: clear_master_lock() return 0 if args.worker: with Connection(glider_qc.get_redis_connection()): worker = Worker(map(Queue, ['gliderdac'])) worker.work() return 0 lock = acquire_master_lock() if not lock.acquire(): raise glider_qc.ProcessError("Master lock already held by another process") try: file_paths = [] if args.verbose: setup_logging() if args.recursive: file_paths = get_files(args.netcdf_files) else: file_paths = args.netcdf_files if args.config is None: raise ValueError("No configuration found, please set using -c") process(file_paths, args.config, sync=args.sync) finally: lock.release() return 0
def clear_master_lock(): ''' Clears the master lock regardless if it is acquired ''' rc = glider_qc.get_redis_connection() rc.delete('gliderdac:glider_qartod')
def clear_master_lock(): ''' Clears the master lock regardless if it is acquired ''' rc = glider_qc.get_redis_connection() rc.delete('gliderdac:glider_qartod')