def setUp(self): self.registry = TaskRegistry() self.registry.register(DuePeriodicTask) self.registry.register(PendingPeriodicTask) self.scheduler = beat.Scheduler(self.registry, max_interval=0.0001, logger=log.get_default_logger())
def make_thumbnail(in_path, out_path): ''' Generate a thumbnail from a video. Returns a tuple: [duration of the video, video resolution] ''' logger = get_default_logger() ffmpeg_path = getattr(settings, "FFMPEG_PATH", 'ffmpeg') ffmpeg_cmd = '%s %s -i "%s" "%s"' % (ffmpeg_path, settings.ALBUMS_VIDEO_THUMB_ARGS, in_path, out_path) logger.info(ffmpeg_cmd) ffmpeg_output = pexpect.run(ffmpeg_cmd) logger.info(ffmpeg_output) # Extract the length of the video m = re.search("Duration:.*(\d\d):(\d\d):(\d\d\.\d\d)", ffmpeg_output) if(m is None or os.stat(out_path).st_size == 0): raise Exception("unable to find duration of video") groups = m.groups() duration = datetime.timedelta(hours=int(groups[0]), minutes=int(groups[1]), seconds=float(groups[2])).seconds m = re.search("Stream.*Video.* (\d+)x(\d+)", ffmpeg_output) if(m is None): raise Exception("Unable to determine resolution of video") resolution = [int(x) for x in m.groups()] logger.info("Resolution: %s" % resolution) return [duration, resolution]
def __init__(self, state, freq=1.0, maxrate=None, cleanup_freq=3600.0, logger=None): self.state = state self.freq = freq self.cleanup_freq = cleanup_freq self.logger = logger or log.get_default_logger(name="celery.cam") self.maxrate = maxrate and TokenBucket(rate(maxrate))
def video_convert(format, size_arg, in_path, out_path): logger = get_default_logger() format_settings = settings.ALBUMS_VIDEO_FORMATS[format] logger.info("Starting conversion of %s" % in_path) threads_arg = '-threads %s' % settings.ALBUMS_CONVERSION_THREADS ffmpeg_path = getattr(settings, "FFMPEG_PATH", 'ffmpeg') for args in format_settings['args']: command = '%s -y -i "%s" %s %s %s "%s"' % (ffmpeg_path, in_path, args, size_arg, threads_arg, out_path) logger.info(command) child = pexpect.spawn(command, maxread=1, timeout=settings.ALBUMS_CONVERSION_TIMEOUT, cwd=os.path.dirname(in_path)) child.expect(pexpect.EOF) exit_status = child.exitstatus if(exit_status == 1): raise Exception("Failed to convert %s to %s (ffmpeg exit status %s)" % (in_path, format_settings['ext'], exit_status)) elif(exit_status == 255): logger.info("Convert %s to %s with errors" % (in_file, format_settings['ext'])) logger.info("Successfully converted %s to %s" % (in_path, format_settings['ext']))
def __init__( self, task_name, task_id, args, kwargs, on_ack=noop, retries=0, delivery_info=None, hostname=None, email_subject=None, email_body=None, logger=None, eventer=None, **opts ): self.task_name = task_name self.task_id = task_id self.retries = retries self.args = args self.kwargs = kwargs self.on_ack = on_ack self.delivery_info = delivery_info or {} self.hostname = hostname or socket.gethostname() self.logger = logger or log.get_default_logger() self.eventer = eventer self.email_subject = email_subject or self.email_subject self.email_body = email_body or self.email_body self.task = tasks[self.task_name]
def __init__(self, task_name, task_id, args, kwargs, on_ack=noop, retries=0, delivery_info=None, hostname=None, email_subject=None, email_body=None, logger=None, eventer=None, **opts): self.task_name = task_name self.task_id = task_id self.retries = retries self.args = args self.kwargs = kwargs self.on_ack = on_ack self.delivery_info = delivery_info or {} self.hostname = hostname or socket.gethostname() self.logger = logger or log.get_default_logger() self.eventer = eventer self.email_subject = email_subject or self.email_subject self.email_body = email_body or self.email_body self.task = tasks[self.task_name]
def run_periodic_tasks(self): logger = get_default_logger() applied = default_periodic_status_backend.run_periodic_tasks() for task, task_id in applied: logger.debug( "PeriodicWorkController: Periodic task %s applied (%s)" % ( task.name, task_id))
def __init__(self, eta_schedule, logger=None, precision=None): super(ScheduleController, self).__init__() self.logger = logger or log.get_default_logger() self._scheduler = iter(eta_schedule) self.precision = precision or conf.CELERYD_ETA_SCHEDULER_PRECISION self.debug = log.SilenceRepeated(self.logger.debug, max_iterations=10)
def __init__(self, logger=None, hostname=None, listener=None): self.logger = logger or log.get_default_logger() self.hostname = hostname self.listener = listener self.panel = self.panel_cls(self.logger, hostname=self.hostname, listener=self.listener)
def redirect_stdouts_to_logger(self): from celery import log handled = log.setup_logging_subsystem(loglevel=self.loglevel, logfile=self.logfile) # Redirect stdout/stderr to our logger. if not handled: logger = log.get_default_logger() log.redirect_stdouts_to_logger(logger, loglevel=logging.WARNING)
def __init__(self, ready_queue, callback, logger=None): threading.Thread.__init__(self) self.logger = logger or log.get_default_logger() self.ready_queue = ready_queue self.callback = callback self._shutdown = threading.Event() self._stopped = threading.Event() self.setDaemon(True)
def setup_logging(self): from celery import log handled = log.setup_logging_subsystem(loglevel=self.loglevel, logfile=self.logfile) logger = log.get_default_logger(name="celery.beat") if self.redirect_stdouts and not handled: log.redirect_stdouts_to_logger(logger, loglevel=self.redirect_stdouts_level) return logger
def __init__(self, ready_queue, logger=None, max_interval=DEFAULT_MAX_INTERVAL): self.max_interval = float(max_interval) self.ready_queue = ready_queue self.logger = logger or log.get_default_logger() self._queue = []
def __init__(self, registry=None, schedule=None, logger=None, max_interval=None): self.registry = registry or _registry.TaskRegistry() self.data = schedule or {} self.logger = logger or log.get_default_logger() self.max_interval = max_interval or conf.CELERYBEAT_MAX_LOOP_INTERVAL self.cleanup() self.schedule_registry()
def __init__(self, schedule=None, logger=None, max_interval=None, **kwargs): UserDict.__init__(self) if schedule is None: schedule = {} self.data = schedule self.logger = logger or log.get_default_logger(name="celery.beat") self.max_interval = max_interval or conf.CELERYBEAT_MAX_LOOP_INTERVAL self.setup_schedule()
def on_iteration(self): logger = get_default_logger() logger.debug("PeriodicWorkController: Running periodic tasks...") try: self.run_periodic_tasks() except Exception, exc: logger.error( "PeriodicWorkController got exception: %s\n%s" % ( exc, traceback.format_exc()))
def __init__(self, ready_queue, callback, logger=None): threading.Thread.__init__(self) self.logger = logger or log.get_default_logger() self.ready_queue = ready_queue self.callback = callback self._shutdown = threading.Event() self._stopped = threading.Event() self.setDaemon(True) self.setName(self.__class__.__name__)
def redirect_stdouts_to_logger(self): from celery import log handled = log.setup_logging_subsystem(loglevel=self.loglevel, logfile=self.logfile) # Redirect stdout/stderr to our logger. if not handled: logger = log.get_default_logger() if self.redirect_stdouts: log.redirect_stdouts_to_logger( logger, loglevel=self.redirect_stdouts_level)
def __init__(self, schedule=None, logger=None, max_interval=None): self.data = schedule if self.data is None: self.data = {} self.logger = logger or log.get_default_logger() self.max_interval = max_interval or conf.CELERYBEAT_MAX_LOOP_INTERVAL self.cleanup() self.setup_schedule()
def __init__(self, schedule=None, logger=None, max_interval=None, lazy=False, **kwargs): UserDict.__init__(self) if schedule is None: schedule = {} self.data = schedule self.logger = logger or log.get_default_logger(name="celery.beat") self.max_interval = max_interval or conf.CELERYBEAT_MAX_LOOP_INTERVAL if not lazy: self.setup_schedule()
def __init__(self, limit, logger=None, initializer=None, maxtasksperchild=None, timeout=None, soft_timeout=None, putlocks=True): self.limit = limit self.logger = logger or log.get_default_logger() self.initializer = initializer self.maxtasksperchild = maxtasksperchild self.timeout = timeout self.soft_timeout = soft_timeout self.putlocks = putlocks self._pool = None
def test_decrement(self): consumer = self.MockConsumer() qos = QoS(consumer, 10, log.get_default_logger()) qos.update() self.assertEqual(int(qos.value), 10) self.assertEqual(consumer.prefetch_count, 10) qos.decrement() self.assertEqual(int(qos.value), 9) self.assertEqual(consumer.prefetch_count, 9) qos.decrement_eventually() self.assertEqual(int(qos.value), 8) self.assertEqual(consumer.prefetch_count, 9)
def on_iteration(self): """Get tasks from bucket queue and apply the task callback.""" logger = get_default_logger() try: # This blocks until there's a message in the queue. task = self.ready_queue.get(timeout=1) except QueueEmpty: time.sleep(1) else: logger.debug("Mediator: Running callback for task: %s[%s]" % ( task.task_name, task.task_id)) self.callback(task)
def __init__(self, limit, logger=None, initializer=None, maxtasksperchild=None, timeout=None, soft_timeout=None, putlocks=True, initargs=()): self.limit = limit self.logger = logger or log.get_default_logger() self.initializer = initializer self.maxtasksperchild = maxtasksperchild self.timeout = timeout self.soft_timeout = soft_timeout self.putlocks = putlocks self.initargs = initargs self._pool = None
def s3_delete_folder(path): logger = get_default_logger() logger.info("Deleting S3 directory %s %s" % (settings.ALBUMS_AWS_S3_BUCKET, path)) conn = S3Connection(settings.ALBUMS_AMAZON_KEY, settings.ALBUMS_AMAZON_SECRET_KEY) bucket = conn.get_bucket(settings.ALBUMS_AWS_S3_BUCKET) for key in bucket.get_all_keys(prefix=path): logger.info("Deleting key %s %s" % (settings.ALBUMS_AWS_S3_BUCKET, key.name)) key.delete()
def on_iteration(self): logger = get_default_logger() try: logger.debug("Mediator: Trying to get message from bucket_queue") # This blocks until there's a message in the queue. task = self.bucket_queue.get(timeout=1) except QueueEmpty: logger.debug("Mediator: Bucket queue is empty.") pass else: logger.debug("Mediator: Running callback for task: %s[%s]" % ( task.task_name, task.task_id)) self.callback(task)
def process_individual_file(tag_file, game_file): logger = get_default_logger() gl = GameLog.create_new(game_file, tag_file) if gl: logger.debug("Game %u created." % gl.number) else: logger.debug("Game not created.") try: os.remove(tag_file) os.remove(game_file) except: pass return bool(gl)
def albumitem_generate_thumbnails(image, sizes, send_notifications=True, delete_on_fail=True): logger = get_default_logger() try: old_dir, filename = os.path.split(image.preview.path) except ValueError: logger.info("Image ID %d has no preview associated with it." % image.id) return try: for size in sizes: sized_path = image._resized_path(size) new_file = os.path.join(settings.MEDIA_ROOT, sized_path) new_dir = os.path.dirname(new_file) if(image.preview.storage.exists(new_file) and key_on_cloudfront(sized_path)): logger.info("Skipping conversion of %s" % new_file) continue try: os.makedirs(new_dir) except OSError as e: if(e.errno != 17): raise e logger.info("About to create %s" % new_file) thumb = Image.open(image.preview.path) thumb.thumbnail([size, size], Image.ANTIALIAS) thumb.save(new_file, thumb.format, quality=90, optimize=1) logger.info("Created thumbnail %s (%d bytes)" % (new_file, os.stat(new_file).st_size)) image_to_cloudfront(new_file, sized_path) if (send_notifications and len(albums.models.Video.objects.filter(id=image.id)) == 0): notification.send([image.submitter,], "albums_conversion", {'success': True, 'object': image}) image.preview_ready = True image.save() except Exception as e: logger.info("Failed to convert image %s, %s" % (image.title, e)) if len(albums.models.Video.objects.filter(id=image.id)) == 0: if(send_notifications): notification.send([image.submitter,], "albums_conversion", {'success': False, 'title': image.title}) if(delete_on_fail): image.delete()
def __init__(self, logger=None, max_interval=conf.CELERYBEAT_MAX_LOOP_INTERVAL, schedule_filename=conf.CELERYBEAT_SCHEDULE_FILENAME): self.logger = logger or log.get_default_logger() self.max_interval = max_interval self.schedule_filename = schedule_filename self._shutdown = threading.Event() self._stopped = threading.Event() self._schedule = None self._scheduler = None self._in_sync = False silence = self.max_interval < 60 and 10 or 1 self.debug = log.SilenceRepeated(self.logger.debug, max_iterations=silence)
def __init__(self, task_name, task_id, args, kwargs, on_ack=noop, retries=0, delivery_info=None, **opts): self.task_name = task_name self.task_id = task_id self.retries = retries self.args = args self.kwargs = kwargs self.on_ack = on_ack self.delivery_info = delivery_info or {} self.task = tasks[self.task_name] for opt in ("success_msg", "fail_msg", "fail_email_subject", "fail_email_body", "logger", "eventer"): setattr(self, opt, opts.get(opt, getattr(self, opt, None))) if not self.logger: self.logger = get_default_logger()
def __init__(self, task_name, task_id, task_func, args, kwargs, on_ack=noop, retries=0, **opts): self.task_name = task_name self.task_id = task_id self.task_func = task_func self.retries = retries self.args = args self.kwargs = kwargs self.logger = kwargs.get("logger") self.on_ack = on_ack self.executed = False for opt in ("success_msg", "fail_msg", "fail_email_subject", "fail_email_body"): setattr(self, opt, opts.get(opt, getattr(self, opt, None))) if not self.logger: self.logger = get_default_logger()
def file_to_cloudfront(path_on_disk, key_name, distribution): ''' Upload the file to cloudfront. Will actually wind up in S3, but boto is smart enough to give it all the permissions we need when we do it this way. ''' logger = get_default_logger() logger.info("Uploading %s to CloudFront %s %s" % (path_on_disk, distribution.domain_name, key_name)) obj = distribution.add_object(key_name, open(path_on_disk, 'rb')) logger.info("Uploaded S3 %s %s" % (distribution.domain_name, key_name))
def __init__(self, logger=None, max_interval=conf.CELERYBEAT_MAX_LOOP_INTERVAL, schedule=conf.CELERYBEAT_SCHEDULE, schedule_filename=conf.CELERYBEAT_SCHEDULE_FILENAME, scheduler_cls=None): self.max_interval = max_interval self.scheduler_cls = scheduler_cls or self.scheduler_cls self.logger = logger or log.get_default_logger(name="celery.beat") self.schedule = schedule self.schedule_filename = schedule_filename self._scheduler = None self._shutdown = threading.Event() self._stopped = threading.Event() silence = self.max_interval < 60 and 10 or 1 self.debug = log.SilenceRepeated(self.logger.debug, max_iterations=silence)
def on_iteration(self): """Wake-up scheduler""" logger = get_default_logger() delay = self._scheduler.next() debug_log = True if delay is None: delay = 1 if self.iterations == 10: self.iterations = 0 else: debug_log = False self.iterations += 1 if debug_log: logger.debug("ScheduleController: Scheduler wake-up") logger.debug( "ScheduleController: Next wake-up eta %s seconds..." % ( delay)) time.sleep(delay)
def file_to_s3(path_on_disk, key_name): ''' Upload a file to our S3 bucket ''' logger = get_default_logger() logger.info("Uploading %s to S3 %s %s" % (path_on_disk, settings.ALBUMS_AWS_S3_BUCKET, key_name)) conn = S3Connection(settings.ALBUMS_AMAZON_KEY, settings.ALBUMS_AMAZON_SECRET_KEY) bucket = conn.get_bucket(settings.ALBUMS_AWS_S3_BUCKET) key = bucket.new_key(key_name) key.set_contents_from_filename(path_on_disk) logger.info("Uploaded S3 %s %s" % (settings.ALBUMS_AWS_S3_BUCKET, key_name))
def __init__(self, task_name, task_id, args, kwargs, on_ack=noop, retries=0, **opts): self.task_name = task_name self.task_id = task_id self.retries = retries self.args = args self.kwargs = kwargs self.logger = opts.get("logger") self.eventer = opts.get("eventer") self.on_ack = on_ack self.executed = False self.time_start = None for opt in ("success_msg", "fail_msg", "fail_email_subject", "fail_email_body"): setattr(self, opt, opts.get(opt, getattr(self, opt, None))) if not self.logger: self.logger = get_default_logger() if self.task_name not in tasks: raise NotRegistered(self.task_name) self.task = tasks[self.task_name]
def albumitem_delete_directory(directory, **kwargs): logger = get_default_logger() full_path = os.path.join(settings.MEDIA_ROOT, directory) if(os.path.exists(full_path)): logger.info("Deleting directory %s" % full_path) shutil.rmtree(full_path) else: logger.info("Skipping delete of directory %s" % full_path) album_path = full_path[:full_path.rfind(os.path.sep)] logger.info("Album path %s %s" % (album_path, os.path.exists(album_path))) if(os.path.exists(album_path) and len(os.listdir(album_path)) == 0): logger.info("Deleting album directory %s" % album_path) shutil.rmtree(album_path) if hasattr(settings, 'ALBUMS_AWS_S3_BUCKET'): s3_delete_folder(directory) else: logger.info("Skipping delete of AWS keys")
def __init__(self, task_name, task_id, args, kwargs, on_ack=noop, retries=0, delivery_info=None, hostname=None, email_subject=None, email_body=None, logger=None, eventer=None, eta=None, expires=None, **opts): self.task_name = task_name self.task_id = task_id self.retries = retries self.args = args self.kwargs = kwargs self.eta = eta self.expires = expires self.on_ack = on_ack self.delivery_info = delivery_info or {} self.hostname = hostname or socket.gethostname() self.logger = logger or log.get_default_logger() self.eventer = eventer self.email_subject = email_subject or self.email_subject self.email_body = email_body or self.email_body self.task = tasks[self.task_name] self._store_errors = True if self.task.ignore_result: self._store_errors = self.task.store_errors_even_if_ignored
def __init__(self, ready_queue, callback, logger=None): super(Mediator, self).__init__() self.logger = logger or log.get_default_logger() self.ready_queue = ready_queue self.callback = callback
from celery.task import task from celery.log import get_default_logger log = get_default_logger() from couchdb import Database from redis import StrictRedis from threading import Thread, active_count from time import sleep keys_to_remove = [ unicode(x) for x in [ 'leaf', 'dcterms_language', "text", 'dcterms_educationLevel', 'skos_exactMatch', 'dcterms_description', 'dcterms_subject', 'asn_indexingStatus', 'asn_authorityStatus', 'asn_statementLabel', 'asn_statementNotation', 'asn_altStatementNotation', 'cls', 'asn_comment' ] ] def process_doc(doc, client): doc['count'] = 0 doc['childCount'] = 0 if "asn_identifier" in doc: if 'uri' in doc['asn_identifier']: doc['id'] = doc['asn_identifier']['uri'].strip() else: doc['id'] = doc['asn_identifier'].strip() if 'id' in doc: url = doc['id'] doc['id'] = url[url.rfind("/") + 1:].lower() if "text" in doc: doc['title'] = doc['text']
def __init__(self, logger=None, hostname=None, listener=None): self.logger = logger or log.get_default_logger() self.hostname = hostname self.listener = listener self.panel = self.Panel(self.logger, self.listener, self.hostname)
def __init__(self, limit, logger=None, **kwargs): self.limit = limit self.logger = logger or log.get_default_logger() self._pool = None
def setUp(self): self.ready_queue = FastQueue() self.eta_schedule = Timer() self.logger = log.get_default_logger() self.logger.setLevel(0)
def __init__(self, *args, **kwargs): self.logger = log.get_default_logger()
def __init__(self, concurrency=None, logfile=None, loglevel=None, send_events=conf.SEND_EVENTS, hostname=None, ready_callback=noop, embed_clockservice=False, pool_cls=conf.CELERYD_POOL, listener_cls=conf.CELERYD_LISTENER, mediator_cls=conf.CELERYD_MEDIATOR, eta_scheduler_cls=conf.CELERYD_ETA_SCHEDULER, schedule_filename=conf.CELERYBEAT_SCHEDULE_FILENAME, task_time_limit=conf.CELERYD_TASK_TIME_LIMIT, task_soft_time_limit=conf.CELERYD_TASK_SOFT_TIME_LIMIT, max_tasks_per_child=conf.CELERYD_MAX_TASKS_PER_CHILD, pool_putlocks=conf.CELERYD_POOL_PUTLOCKS, disable_rate_limits=conf.DISABLE_RATE_LIMITS, db=conf.CELERYD_STATE_DB, scheduler_cls=conf.CELERYBEAT_SCHEDULER): # Options self.loglevel = loglevel or self.loglevel self.concurrency = concurrency or self.concurrency self.logfile = logfile or self.logfile self.logger = log.get_default_logger() self.hostname = hostname or socket.gethostname() self.embed_clockservice = embed_clockservice self.ready_callback = ready_callback self.send_events = send_events self.task_time_limit = task_time_limit self.task_soft_time_limit = task_soft_time_limit self.max_tasks_per_child = max_tasks_per_child self.pool_putlocks = pool_putlocks self.timer_debug = log.SilenceRepeated(self.logger.debug, max_iterations=10) self.db = db self._finalize = Finalize(self, self.stop, exitpriority=1) if self.db: persistence = state.Persistent(self.db) Finalize(persistence, persistence.save, exitpriority=5) # Queues if disable_rate_limits: self.ready_queue = FastQueue() self.ready_queue.put = self.process_task else: self.ready_queue = TaskBucket(task_registry=registry.tasks) self.logger.debug("Instantiating thread components...") # Threads + Pool + Consumer self.pool = instantiate(pool_cls, self.concurrency, logger=self.logger, initializer=process_initializer, initargs=(self.hostname, ), maxtasksperchild=self.max_tasks_per_child, timeout=self.task_time_limit, soft_timeout=self.task_soft_time_limit, putlocks=self.pool_putlocks) self.mediator = None if not disable_rate_limits: self.mediator = instantiate(mediator_cls, self.ready_queue, callback=self.process_task, logger=self.logger) self.scheduler = instantiate( eta_scheduler_cls, precision=conf.CELERYD_ETA_SCHEDULER_PRECISION, on_error=self.on_timer_error, on_tick=self.on_timer_tick) self.beat = None if self.embed_clockservice: self.beat = beat.EmbeddedService( logger=self.logger, schedule_filename=schedule_filename, scheduler_cls=scheduler_cls) prefetch_count = self.concurrency * conf.CELERYD_PREFETCH_MULTIPLIER self.listener = instantiate(listener_cls, self.ready_queue, self.scheduler, logger=self.logger, hostname=self.hostname, send_events=self.send_events, init_callback=self.ready_callback, initial_prefetch_count=prefetch_count, pool=self.pool) # The order is important here; # the first in the list is the first to start, # and they must be stopped in reverse order. self.components = filter(None, (self.pool, self.mediator, self.scheduler, self.beat, self.listener))
def set_loglevel(panel, loglevel=None): if loglevel is not None: if not isinstance(loglevel, int): loglevel = conf.LOG_LEVELS[loglevel.upper()] log.get_default_logger(loglevel=loglevel) return {"ok": loglevel}
def __init__(self, limit=None, putlocks=True, logger=None, **options): self.limit = limit self.putlocks = putlocks self.logger = logger or log.get_default_logger() self.options = options