def acquire_master_lock():
'''
Acquire the master lock or raise an exception
'''
rc = glider_qc.get_redis_connection()
lock = rc.lock('gliderdac:glider_qartod', blocking_timeout=60)
return lock
def acquire_master_lock():
'''
Acquire the master lock or raise an exception
'''
rc = glider_qc.get_redis_connection()
lock = rc.lock('gliderdac:glider_qartod', blocking_timeout=60)
return lock
def process(file_paths, config, sync=False):
queue = Queue('gliderdac', connection=glider_qc.get_redis_connection())
for nc_path in file_paths:
sync_lock()
try:
glider_qc.log.info("Inspecting %s", nc_path)
with Dataset(nc_path, 'r') as nc:
if not glider_qc.check_needs_qc(nc):
continue
glider_qc.log.info("Applying QC to dataset %s", nc_path)
if sync:
glider_qc.qc_task(nc_path, config)
else:
queue.enqueue(glider_qc.qc_task, nc_path, config)
except Exception as e:
glider_qc.log.exception("Failed to check %s for QC", nc_path)
def process(file_paths, config, sync=False):
queue = Queue('gliderdac', connection=glider_qc.get_redis_connection())
for nc_path in file_paths:
sync_lock()
try:
glider_qc.log.info("Inspecting %s", nc_path)
with Dataset(nc_path, 'r') as nc:
if not glider_qc.check_needs_qc(nc):
continue
glider_qc.log.info("Applying QC to dataset %s", nc_path)
if sync:
glider_qc.qc_task(nc_path, config)
else:
queue.enqueue(glider_qc.qc_task, nc_path, config)
except Exception as e:
glider_qc.log.exception("Failed to check %s for QC", nc_path)
def main():
'''
Apply QARTOD QC to GliderDAC submitted netCDF files
Example::
python scripts/glider_qartod.py -c data/qc_config.yml ~/Desktop/revellie/revellie.nc
'''
args = get_args()
if args.clear:
clear_master_lock()
return 0
if args.worker:
with Connection(glider_qc.get_redis_connection()):
worker = Worker(map(Queue, ['gliderdac']))
worker.work()
return 0
lock = acquire_master_lock()
if not lock.acquire():
raise glider_qc.ProcessError(
"Master lock already held by another process")
try:
file_paths = []
if args.verbose:
setup_logging()
if args.recursive:
file_paths = get_files(args.netcdf_files)
else:
file_paths = args.netcdf_files
if args.config is None:
raise ValueError("No configuration found, please set using -c")
process(file_paths, args.config, sync=args.sync)
finally:
lock.release()
return 0
def main():
'''
Apply QARTOD QC to GliderDAC submitted netCDF files
Example::
python scripts/glider_qartod.py -c data/qc_config.yml ~/Desktop/revellie/revellie.nc
'''
args = get_args()
if args.clear:
clear_master_lock()
return 0
if args.worker:
with Connection(glider_qc.get_redis_connection()):
worker = Worker(map(Queue, ['gliderdac']))
worker.work()
return 0
lock = acquire_master_lock()
if not lock.acquire():
raise glider_qc.ProcessError("Master lock already held by another process")
try:
file_paths = []
if args.verbose:
setup_logging()
if args.recursive:
file_paths = get_files(args.netcdf_files)
else:
file_paths = args.netcdf_files
if args.config is None:
raise ValueError("No configuration found, please set using -c")
process(file_paths, args.config, sync=args.sync)
finally:
lock.release()
return 0
def clear_master_lock():
'''
Clears the master lock regardless if it is acquired
'''
rc = glider_qc.get_redis_connection()
rc.delete('gliderdac:glider_qartod')
def clear_master_lock():
'''
Clears the master lock regardless if it is acquired
'''
rc = glider_qc.get_redis_connection()
rc.delete('gliderdac:glider_qartod')