def add_to_list(key, value): setup_rq_connection() keytype = get_current_connection().type(key) if keytype == 'none': set_value(key, [value,]) elif keytype == 'list': get_current_connection().rpush(key, value)
def get_value(key): setup_rq_connection() keytype = get_current_connection().type(key) if keytype == 'list': return get_current_connection().lrange(key, 0, -1) else: return get_current_connection().get(key)
def set_value(key, value): setup_rq_connection() if value is None: delete_key(key) elif type(value) in [int, float, long, bool, str, unicode]: get_current_connection().set(key, value) elif type(value) in [list, tuple]: delete_key(key) for entry in value: get_current_connection().rpush(key, entry) else: raise ValueError("Type '%s' not supported!" % type(value))
def process(id_, data_dir, clean_in=10): """ Job to remove the exif data from an uploaded image. The exif data is saved as a json file. If the image had an exif thumbnail, it is saved as a separate file. """ path = os.path.join(data_dir, "{}.jpg".format(id_)) exif = ExifImage(path) exif.thumb() exif.dump() exif.clean() job = get_current_job() # schedule the cleanup task now = datetime.datetime.now() scheduler = Scheduler(queue_name=job.origin, connection=get_current_connection()) scheduler.enqueue_in(datetime.timedelta(minutes=clean_in), cleanup, id_, data_dir) removed_by = now+datetime.timedelta(minutes=clean_in) print("Added at: {}".format(now.isoformat())) print("Removed by: {}".format(removed_by.isoformat())) return { 'thumb': exif.thumb_name, 'json': exif.json_name, 'removed_around': removed_by.isoformat() }
def compliance_check(job_id, dataset, checker): try: redis = get_current_connection() cs = CheckSuite() if dataset.startswith('http'): dataset = check_redirect(dataset) ds = cs.load_dataset(dataset) score_groups = cs.run(ds, [], checker) rpair = score_groups[checker] groups, errors = rpair aggregates = cs.build_structure(checker, groups, dataset) aggregates = cs.serialize(aggregates) aggregates['all_priorities'] = sorted(aggregates['all_priorities'], key=lambda x: x['weight'], reverse=True) # We use b64 to keep the filenames safe but it's helpful to the user to see # the filename they uploaded if not aggregates['source_name'].startswith('http'): decoded = base64.b64decode(aggregates['source_name'].split('/')[-1]) if isinstance(decoded, str): aggregates['source_name'] = decoded else: aggregates['source_name'] = decoded.decode('utf-8') aggregates['ncdump'] = ncdump(dataset) buf = json.dumps(aggregates) redis.set('processing:job:%s' % job_id, buf, 3600) return True except Exception as e: redis.set('processing:job:%s' % job_id, json.dumps({"error":type(e).__name__, "message":e.message}), 3600) return False
def setup_rq_connection(): redis_conn = get_current_connection() if redis_conn == None: opts = OPTIONS.get('connection') logger.debug('Establishing Redis connection to DB %(db)s at %(host)s:%(port)s' % opts) redis_conn = Redis(**opts) push_connection(redis_conn)
def __init__(self, queue_name='default', interval=60, connection=None): if connection is None: connection = get_current_connection() self.connection = connection self.queue_name = queue_name self._interval = interval self.log = Logger('scheduler')
def list_workers_api(): workers_list = Worker.all() rq_workers = [] for worker in workers_list: host_ip_using_name = "N/A" try: host_ip_using_name = socket.gethostbyname(worker.hostname) except socket.gaierror as addr_error: pass rq_workers.append({ 'worker_name': worker.name, 'listening_on': ', '.join(queue.name for queue in worker.queues), 'status': worker.get_state() if not is_suspended(get_current_connection()) else "suspended", 'host_ip': host_ip_using_name, 'current_job_id': worker.get_current_job_id(), 'failed_jobs': worker.failed_job_count, }) return { 'data': rq_workers, }
def setup_rq_connection(): redis_conn = get_current_connection() if redis_conn is None: opts = constants.REDIS_SETTINGS.get('connection') logger.debug('Establishing Redis connection to DB %(db)s at %(host)s:%(port)s' % opts) redis_conn = Redis(**opts) push_connection(redis_conn)
def assert_current_connection(self, expected_host, expected_port, expected_db): connection = connections.get_current_connection() connection_kwargs = connection.connection_pool.connection_kwargs self.assertEqual(connection_kwargs["host"], expected_host) self.assertEqual(connection_kwargs["port"], expected_port) self.assertEqual(connection_kwargs["db"], expected_db)
def compliance_check(job_id, dataset, checker, path=None): ''' Performs the Check Suite for the specified checker and sets the result in a redis result for the job_id :param str job_id: ID for the rq job :param dataset: Dataset handle :param str checker: Check Suite ID for a checker :param str path: Full path to dataset directory (OPeNDAP only) ''' try: redis = get_current_connection() cs = CheckSuite() if dataset.startswith('http'): dataset = check_redirect(dataset) ds = cs.load_dataset(dataset) score_groups = cs.run(ds, [], checker) rpair = score_groups[checker] groups, errors = rpair aggregates = cs.build_structure(checker, groups, dataset) aggregates = cs.serialize(aggregates) aggregates['all_priorities'] = sorted(aggregates['all_priorities'], key=lambda x: x['weight'], reverse=True) # We use b64 to keep the filenames safe but it's helpful to the user to see # the filename they uploaded if not aggregates['source_name'].startswith('http'): decoded = base64.b64decode( aggregates['source_name'].split('/')[-1]) if isinstance(decoded, str): aggregates['source_name'] = decoded else: aggregates['source_name'] = decoded.decode('utf-8') aggregates['ncdump'] = ncdump(dataset) buf = json.dumps(aggregates) # Write the report to a text file for download if path is None: # Must be a local file, get the path from the dataset path = os.path.dirname(dataset) fname = 'compliance_{}.txt'.format(job_id) output_filename = os.path.join(path, fname) with io.open(output_filename, 'w', encoding='utf-8') as f: with stdout_redirector(f): stdout_output(cs, score_groups, aggregates['source_name']) redis.set('processing:job:%s' % job_id, buf, 3600) return True except Exception as e: logger.exception("Failed to process job") error_message = {"error": type(e).__name__, "message": e.message} redis.set('processing:job:%s' % job_id, json.dumps(error_message), 3600) return False
def setup_redis_with_arguments(self, arguments): parser = rqworker.setup_parser() args = parser.parse_args(arguments) setup_default_arguments(args, {}) self.current_connection = connections.get_current_connection() setup_redis(args)
def _get_connection(): setup_redis(type('lamdbaobject', (object,), { 'host': settings.REDIS_HOST, 'port': settings.REDIS_PORT, 'db': settings.REDIS_DB, 'url': settings.REDIS_URL, 'password': settings.REDIS_PASSWORD, })()) return get_current_connection()
def resume_workers_api(): if request.method == 'POST': try: resume(connection=get_current_connection()) except ActionFailed: raise RQMonitorException('Unable to resume worker/s', status_code=500) return {'message': 'Successfully resumed all workers'} raise RQMonitorException('Invalid HTTP Request type', status_code=400)
def _get_connection(): setup_redis( type( 'lamdbaobject', (object, ), { 'host': settings.REDIS_HOST, 'port': settings.REDIS_PORT, 'db': settings.REDIS_DB, 'url': settings.REDIS_URL, 'password': settings.REDIS_PASSWORD, 'socket': settings.REDIS_SOCKET, })()) return get_current_connection()
def log_duration(timestamp_id, **kwargs): ''' A poor man's timer. There has to be a better way to time job batches... ''' conn = get_current_connection() start_time = conn.get(timestamp_id) duration = time.time() - float(start_time) td = datetime.timedelta(seconds=round(duration)) time.sleep(.5) # just to make it show up at the bottom logger.info(yellow('Completed in {}'.format(td)))
def __init__(self, queues, number_of_processes=1, connection=None): if connection is None: conenction = get_current_connection() self.connection = connection if isinstance(queues, Queue): queues = [queues] self.log = logger self.queues = queues self.validate_queues() self.number_of_processes = number_of_processes self.failed_queue = get_failed_queue(connection=self.connection) self._stopped = False
def compliance_check(job_id, dataset, checker): try: redis = get_current_connection() cs = CheckSuite() if dataset.startswith('http'): dataset = check_redirect(dataset) ds = cs.load_dataset(dataset) score_groups = cs.run(ds, checker) rpair = score_groups[checker] groups, errors = rpair aggregates = cs.build_structure(checker, groups, dataset) aggregates = cs.serialize(aggregates) aggregates['all_priorities'] = sorted(aggregates['all_priorities'], key=lambda x: x['weight'], reverse=True) # We use b64 to keep the filenames safe but it's helpful to the user to see # the filename they uploaded if not aggregates['source_name'].startswith('http'): decoded = base64.b64decode( aggregates['source_name'].split('/')[-1]) if isinstance(decoded, str): aggregates['source_name'] = decoded else: aggregates['source_name'] = decoded.decode('utf-8') aggregates['ncdump'] = ncdump(dataset) buf = json.dumps(aggregates) redis.set('processing:job:%s' % job_id, buf, 3600) return True except Exception as e: redis.set( 'processing:job:%s' % job_id, json.dumps({ "error": type(e).__name__, "message": e.message }), 3600) return False
def list_workers_api(): workers_list = Worker.all() rq_workers = [] for worker in workers_list: rq_workers.append({ 'worker_name': worker.name, 'listening_on': ', '.join(queue.name for queue in worker.queues), 'status': worker.get_state() if not is_suspended(get_current_connection()) else "suspended", 'current_job_id': worker.get_current_job_id(), 'success_jobs': worker.successful_job_count, 'failed_jobs': worker.failed_job_count, }) return { 'data': rq_workers, }
def suspend_workers_api(): if request.method == 'POST': suspend(connection=get_current_connection()) return {'message': 'Successfully suspended all workers'}
def get_workers_dashboard(): return render_template( 'rqmonitor/workers.html', is_suspended=is_suspended(connection=get_current_connection()))
def record_start_time(timestamp_id, **kwargs): conn = get_current_connection() conn.set(timestamp_id, time.time())
def resume_workers_api(): if request.method == 'POST': resume(connection=get_current_connection()) return {'message': 'Successfully resumed all workers'}
def delete_key(key): get_current_connection().delete(key)