Ejemplo n.º 1
0
def parse_url(url):
    port = path = auth = userid = password = None
    scheme = urlparse(url).scheme
    parts = urlparse(url.replace("%s://" % (scheme, ), "http://"))

    # The first pymongo.Connection() argument (host) can be
    # a mongodb connection URI. If this is the case, don't
    # use port but let pymongo get the port(s) from the URI instead.
    # This enables the use of replica sets and sharding.
    # See pymongo.Connection() for more info.
    if scheme != 'mongodb':
        netloc = parts.netloc
        if '@' in netloc:
            auth, _, netloc = partition(parts.netloc, '@')
            userid, _, password = partition(auth, ':')
        hostname, _, port = partition(netloc, ':')
        path = parts.path or ""
        if path and path[0] == '/':
            path = path[1:]
        port = port and int(port) or port
    else:
        # strip the scheme since it is appended automatically
        hostname = url[len('mongodb://'):]

    return dict(
        {
            "hostname": hostname,
            "port": port or None,
            "userid": userid or None,
            "password": password or None,
            "transport": scheme,
            "virtual_host": path or None
        }, **kwdict(dict(parse_qsl(parts.query))))
Ejemplo n.º 2
0
def parse_url(url):
    port = path = auth = userid = password = None
    scheme = urlparse(url).scheme
    parts = urlparse(url.replace("%s://" % (scheme, ), "http://"))

    # The first pymongo.Connection() argument (host) can be
    # a mongodb connection URI. If this is the case, don't
    # use port but let pymongo get the port(s) from the URI instead.
    # This enables the use of replica sets and sharding.
    # See pymongo.Connection() for more info.
    if scheme != 'mongodb':
        netloc = parts.netloc
        if '@' in netloc:
            auth, _, netloc = partition(parts.netloc, '@')
            userid, _, password = partition(auth, ':')
        hostname, _, port = partition(netloc, ':')
        path = parts.path or ""
        if path and path[0] == '/':
            path = path[1:]
        port = port and int(port) or port
    else:
        # strip the scheme since it is appended automatically
        hostname = url[len('mongodb://'):]

    return dict({"hostname": hostname,
                 "port": port or None,
                 "userid": userid or None,
                 "password": password or None,
                 "transport": scheme,
                 "virtual_host": path or None},
                **kwdict(dict(parse_qsl(parts.query))))
Ejemplo n.º 3
0
    def __init__(self, concurrency=None, loglevel=None, logfile=None,
            hostname=None, discard=False, run_clockservice=False,
            schedule=None, task_time_limit=None, task_soft_time_limit=None,
            max_tasks_per_child=None, queues=None, events=False, db=None,
            include=None, app=None, pidfile=None,
            redirect_stdouts=None, redirect_stdouts_level=None,
            autoscale=None, scheduler_cls=None, pool=None, **kwargs):
        self.app = app = app_or_default(app)
        self.concurrency = (concurrency or
                            app.conf.CELERYD_CONCURRENCY or
                            multiprocessing.cpu_count())
        self.loglevel = loglevel or app.conf.CELERYD_LOG_LEVEL
        self.logfile = logfile or app.conf.CELERYD_LOG_FILE

        self.hostname = hostname or socket.gethostname()
        self.discard = discard
        self.run_clockservice = run_clockservice
        if self.app.IS_WINDOWS and self.run_clockservice:
            self.die("-B option does not work on Windows.  "
                     "Please run celerybeat as a separate service.")
        self.schedule = schedule or app.conf.CELERYBEAT_SCHEDULE_FILENAME
        self.scheduler_cls = scheduler_cls or app.conf.CELERYBEAT_SCHEDULER
        self.events = events
        self.task_time_limit = (task_time_limit or
                                app.conf.CELERYD_TASK_TIME_LIMIT)
        self.task_soft_time_limit = (task_soft_time_limit or
                                     app.conf.CELERYD_TASK_SOFT_TIME_LIMIT)
        self.max_tasks_per_child = (max_tasks_per_child or
                                    app.conf.CELERYD_MAX_TASKS_PER_CHILD)
        self.redirect_stdouts = (redirect_stdouts or
                                 app.conf.CELERY_REDIRECT_STDOUTS)
        self.redirect_stdouts_level = (redirect_stdouts_level or
                                       app.conf.CELERY_REDIRECT_STDOUTS_LEVEL)
        self.pool = (pool or app.conf.CELERYD_POOL)
        self.db = db
        self.use_queues = queues or []
        self.queues = None
        self.include = include or []
        self.pidfile = pidfile
        self.autoscale = None
        if autoscale:
            max_c, _, min_c = partition(autoscale, ",")
            self.autoscale = [int(max_c), min_c and int(min_c) or 0]
        self._isatty = sys.stdout.isatty()

        self.colored = app.log.colored(self.logfile)

        if isinstance(self.use_queues, basestring):
            self.use_queues = self.use_queues.split(",")
        if isinstance(self.include, basestring):
            self.include = self.include.split(",")

        if not isinstance(self.loglevel, int):
            try:
                self.loglevel = LOG_LEVELS[self.loglevel.upper()]
            except KeyError:
                self.die("Unknown level %r. Please use one of %s." % (
                            self.loglevel,
                            "|".join(l for l in LOG_LEVELS.keys()
                                        if isinstance(l, basestring))))
Ejemplo n.º 4
0
 def _dispatch_event(self, event):
     self.event_count += 1
     event = kwdict(event)
     group, _, type = partition(event.pop("type"), "-")
     self.group_handlers[group](type, event)
     if self.event_callback:
         self.event_callback(self, event)
Ejemplo n.º 5
0
 def _dispatch_event(self, event):
     self.event_count += 1
     event = kwdict(event)
     group, _, type = partition(event.pop("type"), "-")
     self.group_handlers[group](type, event)
     if self.event_callback:
         self.event_callback(self, event)
Ejemplo n.º 6
0
 def completenames(self, text, *ignored):
     """Return all commands starting with `text`, for tab-completion."""
     names = self.get_names()
     first = [cmd for cmd in names
                     if cmd.startswith(text.replace("_", "."))]
     if first:
         return first
     return [cmd for cmd in names
                 if partition(cmd, ".")[2].startswith(text)]
Ejemplo n.º 7
0
def rate(rate):
    """Parses rate strings, such as `"100/m"` or `"2/h"`
    and converts them to seconds."""
    if rate:
        if isinstance(rate, basestring):
            ops, _, modifier = partition(rate, "/")
            return RATE_MODIFIER_MAP[modifier or "s"](int(ops)) or 0
        return rate or 0
    return 0
Ejemplo n.º 8
0
def rate(rate):
    """Parses rate strings, such as `"100/m"` or `"2/h"`
    and converts them to seconds."""
    if rate:
        if isinstance(rate, basestring):
            ops, _, modifier = partition(rate, "/")
            return RATE_MODIFIER_MAP[modifier or "s"](int(ops)) or 0
        return rate or 0
    return 0
Ejemplo n.º 9
0
def parse_url(url):
    auth = userid = password = None
    scheme = urlparse(url).scheme
    parts = urlparse(url.replace("%s://" % (scheme, ), "http://"))
    netloc = parts.netloc
    if '@' in netloc:
        auth, _, netloc = partition(parts.netloc, '@')
        userid, _, password = partition(auth, ':')
    hostname, _, port = partition(netloc, ':')
    path = parts.path or ""
    if path and path[0] == '/':
        path = path[path.index('/') + 1:]
    return dict({"hostname": hostname,
                 "port": port and int(port) or None,
                 "userid": userid or None,
                 "password": password or None,
                 "transport": scheme,
                 "virtual_host": path or None},
                **kwdict(dict(parse_qsl(parts.query))))
Ejemplo n.º 10
0
def parse_url(url):
    auth = userid = password = None
    scheme = urlparse(url).scheme
    parts = urlparse(url.replace("%s://" % (scheme, ), "http://"))
    netloc = parts.netloc
    if '@' in netloc:
        auth, _, netloc = partition(parts.netloc, '@')
        userid, _, password = partition(auth, ':')
    hostname, _, port = partition(netloc, ':')
    path = parts.path or ""
    if path and path[0] == '/':
        path = path[path.index('/') + 1:]
    return dict({"hostname": hostname,
                 "port": port and int(port) or None,
                 "userid": userid or None,
                 "password": password or None,
                 "transport": scheme,
                 "virtual_host": path or None},
                **kwdict(dict(parse_qsl(parts.query))))
Ejemplo n.º 11
0
 def completenames(self, text, *ignored):
     """Return all commands starting with `text`, for tab-completion."""
     names = self.get_names()
     first = [
         cmd for cmd in names if cmd.startswith(text.replace("_", "."))
     ]
     if first:
         return first
     return [
         cmd for cmd in names if partition(cmd, ".")[2].startswith(text)
     ]
Ejemplo n.º 12
0
    def __init__(self, expires=None, backend=None, options={}, **kwargs):
        super(CacheBackend, self).__init__(self, **kwargs)

        self.expires = expires or self.app.conf.CELERY_TASK_RESULT_EXPIRES
        if isinstance(self.expires, timedelta):
            self.expires = timeutils.timedelta_seconds(self.expires)
        self.options = dict(self.app.conf.CELERY_CACHE_BACKEND_OPTIONS,
                            **options)

        backend = backend or self.app.conf.CELERY_CACHE_BACKEND
        self.expires = int(self.expires)
        self.backend, _, servers = partition(backend, "://")
        self.servers = servers.split(";")
        try:
            self.Client = backends[self.backend]
        except KeyError:
            raise ImproperlyConfigured(
                    "Unknown cache backend: %s. Please use one of the "
                    "following backends: %s" % (self.backend,
                                                ", ".join(backends.keys())))
Ejemplo n.º 13
0
    def __init__(self, expires=None, backend=None, options={}, **kwargs):
        super(CacheBackend, self).__init__(self, **kwargs)

        self.expires = expires or self.app.conf.CELERY_TASK_RESULT_EXPIRES
        if isinstance(self.expires, timedelta):
            self.expires = timeutils.timedelta_seconds(self.expires)
        self.options = dict(self.app.conf.CELERY_CACHE_BACKEND_OPTIONS,
                            **options)

        backend = backend or self.app.conf.CELERY_CACHE_BACKEND
        self.expires = int(self.expires)
        self.backend, _, servers = partition(backend, "://")
        self.servers = servers.rstrip('/').split(";")
        try:
            self.Client = backends[self.backend]()
        except KeyError:
            raise ImproperlyConfigured(
                    "Unknown cache backend: %s. Please use one of the "
                    "following backends: %s" % (self.backend,
                                                ", ".join(backends.keys())))
Ejemplo n.º 14
0
    def __init__(self,
                 concurrency=None,
                 loglevel=None,
                 logfile=None,
                 hostname=None,
                 discard=False,
                 run_clockservice=False,
                 schedule=None,
                 task_time_limit=None,
                 task_soft_time_limit=None,
                 max_tasks_per_child=None,
                 queues=None,
                 events=False,
                 db=None,
                 include=None,
                 app=None,
                 pidfile=None,
                 redirect_stdouts=None,
                 redirect_stdouts_level=None,
                 autoscale=None,
                 scheduler_cls=None,
                 pool=None,
                 **kwargs):
        self.app = app = app_or_default(app)
        self.concurrency = (concurrency or app.conf.CELERYD_CONCURRENCY
                            or cpu_count())
        self.loglevel = loglevel or app.conf.CELERYD_LOG_LEVEL
        self.logfile = logfile or app.conf.CELERYD_LOG_FILE

        self.hostname = hostname or socket.gethostname()
        self.discard = discard
        self.run_clockservice = run_clockservice
        if self.app.IS_WINDOWS and self.run_clockservice:
            self.die("-B option does not work on Windows.  "
                     "Please run celerybeat as a separate service.")
        self.schedule = schedule or app.conf.CELERYBEAT_SCHEDULE_FILENAME
        self.scheduler_cls = scheduler_cls or app.conf.CELERYBEAT_SCHEDULER
        self.events = events
        self.task_time_limit = (task_time_limit
                                or app.conf.CELERYD_TASK_TIME_LIMIT)
        self.task_soft_time_limit = (task_soft_time_limit
                                     or app.conf.CELERYD_TASK_SOFT_TIME_LIMIT)
        self.max_tasks_per_child = (max_tasks_per_child
                                    or app.conf.CELERYD_MAX_TASKS_PER_CHILD)
        self.redirect_stdouts = (redirect_stdouts
                                 or app.conf.CELERY_REDIRECT_STDOUTS)
        self.redirect_stdouts_level = (redirect_stdouts_level or
                                       app.conf.CELERY_REDIRECT_STDOUTS_LEVEL)
        self.pool = (pool or app.conf.CELERYD_POOL)
        self.db = db
        self.use_queues = queues or []
        self.queues = None
        self.include = include or []
        self.pidfile = pidfile
        self.autoscale = None
        if autoscale:
            max_c, _, min_c = partition(autoscale, ",")
            self.autoscale = [int(max_c), min_c and int(min_c) or 0]
        self._isatty = sys.stdout.isatty()

        self.colored = app.log.colored(self.logfile)

        if isinstance(self.use_queues, basestring):
            self.use_queues = self.use_queues.split(",")
        if isinstance(self.include, basestring):
            self.include = self.include.split(",")

        if not isinstance(self.loglevel, int):
            try:
                self.loglevel = LOG_LEVELS[self.loglevel.upper()]
            except KeyError:
                self.die(
                    "Unknown level %r. Please use one of %s." %
                    (self.loglevel, "|".join(l for l in LOG_LEVELS.keys()
                                             if isinstance(l, basestring))))