def get_executor(self): """ Returns threadpool executor """ if not self.executor: self.logger.debug("Run thread executor (%d threads)", self.max_threads) self.executor = ThreadPoolExecutor(self.max_threads, name=self.name) return self.executor
def get_executor(self, name: str) -> ThreadPoolExecutor: """ Return or start named executor """ executor = self.executors.get(name) if not executor: xt = "%s.%s_threads" % (self.name, name) max_threads = config.get_parameter(xt) self.logger.info( "Starting threadpool executor %s (up to %d threads)", name, max_threads ) executor = ThreadPoolExecutor(max_threads, name=name) self.executors[name] = executor return executor
def iter_data(self): def fetch_sql(query, params): cursor = self.get_cursor() cursor.execute(query, params) return list(cursor) concurrency = int(self.config.get("ORACLE_ARRAYSIZE", 1)) if concurrency == 1: cursor = self.get_cursor() # Fetch data self.logger.info("Fetching data") for query, params in self.get_sql(): cursor.execute(query, params) for row in cursor: yield row else: with ThreadPoolExecutor(max_workers=concurrency) as pool: futures = [ pool.submit(fetch_sql, query, params) for query, params in self.get_sql() ] for f in as_completed(futures): for row in f.result(): yield row
def iter_data_parallel(self, concurrency): def fetch_sql(query, params): cursor = self.get_cursor() cursor.execute(query, params) out = list(cursor) with cond: buffer.append(out) cond.notify() cond = Condition() buffer = [] with ThreadPoolExecutor(max_workers=concurrency) as pool: left = 0 for query, params in self.get_sql(): left += 1 pool.submit(fetch_sql, query, params) while left > 0: with cond: if not buffer: cond.wait() data, buffer = buffer, [] for seq in data: yield from seq left -= 1
class Scheduler(object): COLLECTION_BASE = "noc.schedules." SUBMIT_THRESHOLD_FACTOR = config.scheduler.submit_threshold_factor MAX_CHUNK_FACTOR = config.scheduler.max_chunk_factor UPDATES_PER_CHECK = config.scheduler.updates_per_check CACHE_DEFAULT_TTL = config.scheduler.cache_default_ttl def __init__( self, name, pool=None, reset_running=False, max_threads=5, ioloop=None, check_time=1000, submit_threshold=None, max_chunk=None, filter=None, service=None, sample=0, ): """ Create scheduler :param name: Unique scheduler name :param pool: Pool name, if run in pooled mode :param reset_running: Reset all running jobs to wait state on start :param max_threads: Thread executor threads :param ioloop: IOLoop instance :param check_time: time in milliseconds to check for new jobs :param max_chunk: Maximum amount of jobs which can be submitted at one step :param submit_threshold: Maximal executor queue length when submitting of next chunk is allowed :param filter: Additional filter to be applied to pending jobs :param tracing_sample: Tracing sample rate. 0 - do not sample, 1 - sample every job N > 1 - sample very Nth job """ self.logger = logging.getLogger("scheduler.%s" % name) self.name = name self.collection_name = self.COLLECTION_BASE + self.name self.pool = pool if pool: self.collection_name += ".%s" % pool self.ioloop = ioloop or tornado.ioloop.IOLoop.current() self.to_reset_running = reset_running self.running_groups = set() self.collection = None self.bulk = [] self.bulk_lock = threading.Lock() self.max_threads = max_threads self.executor = None self.run_callback = None self.check_time = check_time self.read_ahead_interval = datetime.timedelta(milliseconds=check_time) if submit_threshold: self.submit_threshold = submit_threshold else: self.submit_threshold = max( self.max_threads // self.SUBMIT_THRESHOLD_FACTOR, 1) if max_chunk: self.max_chunk = max_chunk else: self.max_chunk = max(self.max_threads // self.MAX_CHUNK_FACTOR, 2) self.filter = filter self.to_shutdown = False self.min_sleep = float(check_time) / self.UPDATES_PER_CHECK / 1000.0 self.jobs_burst = [] self.cache = None self.cache_lock = threading.Lock() self.cache_set_ops = {} self.service = service if self.service: self.scheduler_id = "%s[%s:%s]" % ( self.service.service_id, self.service.address, self.service.port, ) else: self.scheduler_id = "standalone scheduler" self.sample = sample def get_cache(self): with self.cache_lock: if not self.cache: from noc.core.cache.base import cache self.cache = cache return self.cache def run(self): """ Run scheduler. Common usage scheduler.run(ioloop=ioloop) ioloop.run() """ if self.to_reset_running: self.reset_running() self.ensure_indexes() self.logger.info("Running scheduler") self.ioloop.spawn_callback(self.scheduler_loop) def get_collection(self): """ Returns mongo collection instance """ if not self.collection: self.logger.debug("Open collection %s", self.collection_name) self.collection = get_db()[self.collection_name] self.bulk = [] return self.collection def get_executor(self): """ Returns threadpool executor """ if not self.executor: self.logger.debug("Run thread executor (%d threads)", self.max_threads) self.executor = ThreadPoolExecutor(self.max_threads, name=self.name) return self.executor def reset_running(self): """ Reset all running jobs to waiting status """ self.logger.debug("Reset running jobs") r = self.get_collection().update_many( self.get_query({Job.ATTR_STATUS: Job.S_RUN}), {"$set": { Job.ATTR_STATUS: Job.S_WAIT }}) if r.acknowledged: if r.modified_count: self.logger.info("Reset: %d", r.modified_count) else: self.logger.info("Nothing to reset") else: self.logger.info("Failed to reset running jobs") def ensure_indexes(self): """ Create all nesessary indexes """ self.logger.debug("Check indexes") self.get_collection().create_index([("ts", 1)], partialFilterExpression={"s": "W"}) self.get_collection().create_index([("jcls", 1), ("key", 1)]) self.logger.debug("Indexes are ready") def get_query(self, q): """ Combine filter with query and return resulting query """ if self.filter: qq = self.filter.copy() qq.update(q) return qq else: return q def scheduler_tick(self): """ Process single scheduler tick :return: """ n = 0 try: n = self.run_pending() except Exception as e: self.logger.error("Failed to schedule next tasks: %s", e) self.apply_ops() return n def apply_ops(self): self.apply_bulk_ops() self.apply_cache_ops() @tornado.gen.coroutine def scheduler_loop(self): """ Primary scheduler loop """ while not self.to_shutdown: t0 = perf_counter() n = 0 if self.get_executor().may_submit(): try: n = yield self.executor.submit(self.scheduler_tick) except Exception as e: self.logger.error("Failed to execute scheduler tick: %s", e) dt = self.check_time - (perf_counter() - t0) * 1000 if dt > 0: if n: dt = min(dt, self.check_time / n) yield tornado.gen.sleep(dt / 1000.0) self.apply_ops() def iter_pending_jobs(self, limit): """ Yields pending jobs """ qs = (self.get_collection().find( self.get_query({ Job.ATTR_TS: { "$lte": datetime.datetime.now() + self.read_ahead_interval }, Job.ATTR_STATUS: Job.S_WAIT, })).limit(limit).sort(Job.ATTR_TS)) try: for job in qs: job[Job.ATTR_SAMPLE] = self.sample try: jcls = get_handler(job[Job.ATTR_CLASS]) yield jcls(self, job) except ImportError as e: self.logger.error("Invalid job class %s", job[Job.ATTR_CLASS]) self.logger.error("Error: %s", e) self.remove_job_by_id(job[Job.ATTR_ID]) except pymongo.errors.CursorNotFound: self.logger.info("Server cursor timed out. Waiting for next cycle") except pymongo.errors.OperationFailure as e: self.logger.error("Operation failure: %s", e) self.logger.error("Trying to recover") except pymongo.errors.AutoReconnect: self.logger.error( "Auto-reconnect detected. Waiting for next cycle") def run_pending(self): """ Read and launch all pending jobs """ executor = self.get_executor() collection = self.get_collection() n = 0 if not executor.may_submit(): return 0 jobs, self.jobs_burst = self.jobs_burst, [] burst_ids = set(j.attrs[Job.ATTR_ID] for j in jobs) if len(jobs) <= self.max_chunk // 2: jobs += [ j for j in self.iter_pending_jobs(self.max_chunk) if j.attrs[Job.ATTR_ID] not in burst_ids ] while jobs: free_workers = executor.get_free_workers() if not free_workers: self.logger.info( "All workers are busy. Sending %d jobs to burst", len(jobs)) self.jobs_burst = jobs break now = datetime.datetime.now() rl = min(sum(1 for j in jobs if j.attrs[Job.ATTR_TS] <= now), free_workers) rjobs, jobs = jobs[:rl], jobs[rl:] if rjobs: jids = [j.attrs[Job.ATTR_ID] for j in rjobs] self.logger.debug( "update({_id: {$in: %s}}, {$set: {%s: '%s'}})", jids, Job.ATTR_STATUS, Job.S_RUN) r = collection.update_many({"_id": { "$in": jids }}, {"$set": { Job.ATTR_STATUS: Job.S_RUN }}) if r.acknowledged: if r.modified_count != len(jids): self.logger.error( "Failed to update all running statuses: %d of %d", r.modified_count, len(jids), ) else: self.logger.error("Failed to update running status") # Fetch contexts # version -> key -> job cjobs = {} for j in rjobs: if not j.context_version: continue if j.context_version not in cjobs: cjobs[j.context_version] = {} cjobs[j.context_version][j.get_context_cache_key()] = j if cjobs: for v in cjobs: try: ctx = self.get_cache().get_many(cjobs[v], version=v) or {} except Exception as e: self.logger.error("Failed to restore context: %s", e) ctx = {} for k in cjobs[v]: cjobs[v][k].load_context(ctx.get(k, {})) # for job in rjobs: if job.is_retries_exceeded(): metrics["%s_jobs_retries_exceeded" % self.name] += 1 in_label = None if config.features.forensic: in_label = "%s:%s" % (job.attrs[Job.ATTR_CLASS], job.attrs[Job.ATTR_KEY]) executor.submit(job.run, _in_label=in_label) metrics["%s_jobs_started" % self.name] += 1 n += 1 if jobs: # Wait for next job within check_interval njts = jobs[0].attrs[Job.ATTR_TS] now = datetime.datetime.now() if njts > now: dt = njts - now dt = (dt.microseconds + dt.seconds * 1000000.0) / 1000000.0 dt = max(dt, self.min_sleep) time.sleep(dt) return n def apply_bulk_ops(self): if not self.bulk: return # Nothing to apply t0 = perf_counter() with self.bulk_lock: try: r = self.collection.bulk_write(self.bulk) dt = perf_counter() - t0 self.logger.info( "%d bulk operations complete in %dms: " "inserted=%d, updated=%d, removed=%d", len(self.bulk), int(dt * 1000), r.inserted_count, r.modified_count, r.deleted_count, ) except pymongo.errors.BulkWriteError as e: self.logger.error("Cannot apply bulk operations: %s [%s]", e.details, e.code) metrics["%s_bulk_failed" % self.name] += 1 return except Exception as e: self.logger.error("Cannot apply bulk operations: %s", e) metrics["%s_bulk_failed" % self.name] += 1 return finally: self.bulk = [] def remove_job(self, jcls, key=None): """ Remove job from schedule """ self.logger.info("Remove job %s(%s)", jcls, key) self.get_collection().remove({Job.ATTR_CLASS: jcls, Job.ATTR_KEY: key}) def remove_job_by_id(self, jid): """ Remove job from schedule """ self.logger.info("Remove job %s", jid) with self.bulk_lock: self.bulk += [DeleteOne({Job.ATTR_ID: jid})] def submit(self, jcls, key=None, data=None, ts=None, delta=None, keep_ts=False, max_runs=None): """ Submit new job or adjust existing one :param jcls: Job class name :param key: Job key :param data: Job data (will be passed as handler's arguments) :param ts: Set next run time (datetime) :param delta: Set next run time after delta seconds :param keep_ts: Do not touch timestamp of existing jobs, set timestamp only for created jobs :param max_runs: Limit maximum runs attempts """ now = datetime.datetime.now() data = data or {} set_op = {} iset_op = { Job.ATTR_STATUS: Job.S_WAIT, Job.ATTR_RUNS: 0, Job.ATTR_FAULTS: 0, Job.ATTR_OFFSET: random.random(), } if max_runs is not None: iset_op[Job.ATTR_MAX_RUNS] = max_runs if ts: set_op[Job.ATTR_TS] = ts elif delta: set_op[Job.ATTR_TS] = now + datetime.timedelta(seconds=delta) else: set_op[Job.ATTR_TS] = now if keep_ts: iset_op[Job.ATTR_TS] = set_op.pop(Job.ATTR_TS) if data: set_op[Job.ATTR_DATA] = data q = {Job.ATTR_CLASS: jcls, Job.ATTR_KEY: key} op = {"$setOnInsert": iset_op} if set_op: op["$set"] = set_op self.logger.info( "Submit job %s(%s, %s) at %s", jcls, key, data, set_op.get(Job.ATTR_TS) or iset_op.get(Job.ATTR_TS), ) self.logger.debug("update(%s, %s, upsert=True)", q, op) self.get_collection().update(q, op, upsert=True) def set_next_run( self, jid, status=None, ts=None, delta=None, duration=None, context=None, context_version=None, context_key=None, ): """ Reschedule job and set next run time :param jid: Job id :param status: Register run status :param ts: Set next run time (datetime) :param delta: Set next run time after delta seconds :param duration: Set last run duration (in seconds) :param context_version: Job context format vresion :param context: Stored job context :param context_key: Cache key for context """ # Build increase/set operations now = datetime.datetime.now() set_op = {Job.ATTR_STATUS: Job.S_WAIT, Job.ATTR_LAST: now} inc_op = {} if status: set_op[Job.ATTR_LAST_STATUS] = status if ts: set_op[Job.ATTR_TS] = ts elif delta: set_op[Job.ATTR_TS] = now + datetime.timedelta(seconds=delta) else: set_op[Job.ATTR_TS] = now if duration: set_op[Job.ATTR_LAST_DURATION] = duration if status: inc_op[Job.ATTR_RUNS] = 1 set_op[Job.ATTR_LAST_STATUS] = status if status == Job.E_SUCCESS: set_op[Job.ATTR_LAST_SUCCESS] = now set_op[Job.ATTR_FAULTS] = 0 elif status == Job.E_EXCEPTION: inc_op[Job.ATTR_FAULTS] = 1 if context_version is not None: self.cache_set(key=context_key, value=context, version=context_version) op = {} if set_op: op["$set"] = set_op if inc_op: op["$inc"] = inc_op if op: q = {Job.ATTR_ID: jid} self.logger.debug("update(%s, %s)", q, op) with self.bulk_lock: self.bulk += [UpdateOne(q, op)] def apply_cache_ops(self): with self.cache_lock: cache_set_ops = self.cache_set_ops self.cache_set_ops = {} if not cache_set_ops: return cache = self.get_cache() for version in cache_set_ops: metrics["%s_cache_set_requests" % self.name] += 1 try: cache.set_many(cache_set_ops[version], version=version, ttl=self.CACHE_DEFAULT_TTL) except Exception as e: self.logger.error("Error writing cache: %s", e) metrics["%s_cache_set_errors" % self.name] += 1 def cache_set(self, key, value, version): with self.cache_lock: if version not in self.cache_set_ops: self.cache_set_ops[version] = {} self.cache_set_ops[version][key] = value def apply_metrics(self, d): """ Append scheduler metrics to dictionary d :param d: :return: """ if self.executor: self.executor.apply_metrics(d) d.update({"%s_jobs_burst" % self.name: len(self.jobs_burst)}) def shutdown(self, sync=False): self.to_shutdown = True if self.executor: return self.executor.shutdown(sync) else: f = Future() f.set_result(True) return f