コード例 #1
0
ファイル: core.py プロジェクト: surbas/dagobah
    def __init__(self, backend=BaseBackend(), event_handler=None):
        """ Construct a new Dagobah instance with a specified Backend. """
        self.backend = backend
        self.event_handler = event_handler
        self.dagobah_id = self.backend.get_new_dagobah_id()
        self.jobs = []
        self.created_jobs = 0
        self.scheduler = Scheduler(self)
        self.scheduler.daemon = True

        self.scheduler.start()

        self.commit()
コード例 #2
0
ファイル: core.py プロジェクト: SunnyKale/dagobah
    def __init__(self, backend=BaseBackend(), event_handler=None):
        """ Construct a new Dagobah instance with a specified Backend. """
        self.backend = backend
        self.event_handler = event_handler
        self.dagobah_id = self.backend.get_new_dagobah_id()
        self.jobs = []
        self.created_jobs = 0
        self.scheduler = Scheduler(self)
        self.scheduler.daemon = True

        self.scheduler.start()

        self.commit()
コード例 #3
0
ファイル: core.py プロジェクト: utkarsh2012/dagobah
class Dagobah(object):
    """ Top-level controller for all Dagobah usage.

    This is in control of all the jobs for a specific Dagobah
    instance, as well as top-level parameters such as the
    backend used for permanent storage.
    """

    def __init__(self, backend=BaseBackend(), event_handler=None):
        """ Construct a new Dagobah instance with a specified Backend. """
        self.backend = backend
        self.event_handler = event_handler
        self.dagobah_id = self.backend.get_new_dagobah_id()
        self.jobs = []
        self.hosts = []
        self.created_jobs = 0
        self.scheduler = Scheduler(self)
        self.scheduler.daemon = True

        self.scheduler.start()

        self.commit()

    def __repr__(self):
        return "<Dagobah with Backend %s>" % self.backend

    def set_backend(self, backend):
        """ Manually set backend after construction. """

        self.backend = backend
        self.dagobah_id = self.backend.get_new_dagobah_id()

        for job in self.jobs:
            job.backend = backend
            for task in job.tasks.values():
                task.backend = backend

        for host in self.hosts:
            host.backend = backend

        self.commit(cascade=True)

    def from_backend(self, dagobah_id):
        """ Reconstruct this Dagobah instance from the backend. """
        rec = self.backend.get_dagobah_json(dagobah_id)
        if not rec:
            raise DagobahError("dagobah with id %s does not exist " "in backend" % dagobah_id)
        self._construct_from_json(rec)

    def _construct_from_json(self, rec):
        """ Construct this Dagobah instance from a JSON document. """

        self.delete()

        for required_key in ["dagobah_id", "created_jobs"]:
            setattr(self, required_key, rec[required_key])

        for job_json in rec.get("jobs", []):
            self._add_job_from_spec(job_json)

        for host_json in rec.get("hosts", []):
            self.add_host(host_name=host_json["host_name"], host_id=host_json["host_id"])

        self.commit(cascade=True)

    def add_job_from_json(self, job_json, destructive=False):
        """ Construct a new Job from an imported JSON spec. """
        rec = self.backend.decode_import_json(job_json)
        if destructive:
            try:
                self.delete_job(rec["name"])
            except DagobahError:  # expected if no job with this name
                pass
        self._add_job_from_spec(rec, use_job_id=False)

        self.commit(cascade=True)

    def _add_job_from_spec(self, job_json, use_job_id=True):
        """ Add a single job to the Dagobah from a spec. """

        job_id = job_json["job_id"] if use_job_id else self.backend.get_new_job_id()
        self.add_job(str(job_json["name"]), job_id)
        job = self.get_job(job_json["name"])
        if job_json.get("cron_schedule", None):
            job.schedule(job_json["cron_schedule"])

        for task in job_json.get("tasks", []):
            self.add_task_to_job(
                job,
                str(task["command"]),
                str(task["name"]),
                soft_timeout=task.get("soft_timeout", 0),
                hard_timeout=task.get("hard_timeout", 0),
                host_id=task.get("host_id", None),
            )

        dependencies = job_json.get("dependencies", {})
        for from_node, to_nodes in dependencies.iteritems():
            for to_node in to_nodes:
                job.add_dependency(from_node, to_node)

    def commit(self, cascade=False):
        """ Commit this Dagobah instance to the backend.

        If cascade is True, all child Jobs are commited as well.
        """
        self.backend.commit_dagobah(self._serialize())
        if cascade:
            [job.commit() for job in self.jobs]

    def delete(self):
        """ Delete this Dagobah instance from the Backend. """
        self.jobs = []
        self.hosts = []
        self.created_jobs = 0
        self.backend.delete_dagobah(self.dagobah_id)

    def add_job(self, job_name, job_id=None):
        """ Create a new, empty Job. """
        if not self._name_is_available(job_name):
            raise DagobahError("name %s is not available" % job_name)

        if not job_id:
            job_id = self.backend.get_new_job_id()
            self.created_jobs += 1

        self.jobs.append(Job(self, self.backend, job_id, job_name))

        job = self.get_job(job_name)
        job.commit()

    def get_job(self, job_name):
        """ Returns a Job by name, or None if none exists. """
        for job in self.jobs:
            if job.name == job_name:
                return job
        return None

    def get_host(self, host_id):
        """ Returns a Host by name, or None if none exists. """
        for host in self.hosts:
            if host.id == host_id:
                return host
        return None

    def delete_job(self, job_name):
        """ Delete a job by name, or error out if no such job exists. """
        for idx, job in enumerate(self.jobs):
            if job.name == job_name:
                self.backend.delete_job(job.job_id)
                del self.jobs[idx]
                self.commit()
                return
        raise DagobahError("no job with name %s exists" % job_name)

    def add_task_to_job(self, job_or_job_name, task_command, task_name=None, **kwargs):
        """ Add a task to a job owned by the Dagobah instance. """
        if isinstance(job_or_job_name, Job):
            job = job_or_job_name
        else:
            job = self.get_job(job_or_job_name)

        if not job:
            raise DagobahError("job %s does not exist" % job_or_job_name)

        if not job.state.allow_change_graph:
            raise DagobahError("job's graph is immutable in its current state: %s" % job.state.status)

        job.add_task(task_command, task_name, **kwargs)
        job.commit()

    def add_host(self, host_name, host_id=None):
        """ Add a new host """
        if not self._host_is_added(host_name=host_name):
            raise DagobahError("Host %s is already added." % host_name)

        if not host_id:
            host_id = self.backend.get_new_host_id()

        self.hosts.append(Host(self, self.backend, host_id, host_name))

        host = self.get_host(host_id)
        host.commit()

    def delete_host(self, host_name):
        """ Delete a host """
        for idx, host in enumerate(self.hosts):
            if host.name == host_name:
                self.backend.delete_host(host.id)
                del self.hosts[idx]
                self.commit()
                return
        raise DagobahError("no host with name %s exists" % host_name)

    def _host_is_added(self, host_name=None):
        """ Returns Boolean of whether the specified host is already added. """
        return False if [host for host in self.hosts if host.name == host_name] else True

    def _name_is_available(self, job_name):
        """ Returns Boolean of whether the specified name is already in use. """
        return False if [job for job in self.jobs if job.name == job_name] else True

    def _serialize(self, include_run_logs=False, strict_json=False):
        """ Serialize a representation of this Dagobah object to JSON. """
        result = {
            "dagobah_id": self.dagobah_id,
            "created_jobs": self.created_jobs,
            "jobs": [job._serialize(include_run_logs=include_run_logs, strict_json=strict_json) for job in self.jobs],
            "hosts": [host._serialize() for host in self.hosts],
        }
        if strict_json:
            result = json.loads(json.dumps(result, cls=StrictJSONEncoder))
        return result
コード例 #4
0
ファイル: core.py プロジェクト: surbas/dagobah
class Dagobah(object):
    """ Top-level controller for all Dagobah usage.

    This is in control of all the jobs for a specific Dagobah
    instance, as well as top-level parameters such as the
    backend used for permanent storage.
    """
    def __init__(self, backend=BaseBackend(), event_handler=None):
        """ Construct a new Dagobah instance with a specified Backend. """
        self.backend = backend
        self.event_handler = event_handler
        self.dagobah_id = self.backend.get_new_dagobah_id()
        self.jobs = []
        self.created_jobs = 0
        self.scheduler = Scheduler(self)
        self.scheduler.daemon = True

        self.scheduler.start()

        self.commit()

    def __repr__(self):
        return '<Dagobah with Backend %s>' % self.backend

    def set_backend(self, backend):
        """ Manually set backend after construction. """

        self.backend = backend
        self.dagobah_id = self.backend.get_new_dagobah_id()

        for job in self.jobs:
            job.backend = backend
            for task in job.tasks.values():
                task.backend = backend

        self.commit(cascade=True)

    def from_backend(self, dagobah_id):
        """ Reconstruct this Dagobah instance from the backend. """
        rec = self.backend.get_dagobah_json(dagobah_id)
        if not rec:
            raise DagobahError('dagobah with id %s does not exist '
                               'in backend' % dagobah_id)
        self._construct_from_json(rec)

    def _construct_from_json(self, rec):
        """ Construct this Dagobah instance from a JSON document. """

        self.delete()

        for required_key in ['dagobah_id', 'created_jobs']:
            setattr(self, required_key, rec[required_key])

        for job_json in rec.get('jobs', []):
            self._add_job_from_spec(job_json)

        self.commit(cascade=True)

    def add_job_from_json(self, job_json, destructive=False):
        """ Construct a new Job from an imported JSON spec. """
        rec = self.backend.decode_import_json(job_json)
        if destructive:
            try:
                self.delete_job(rec['name'])
            except DagobahError:  # expected if no job with this name
                pass
        self._add_job_from_spec(rec, use_job_id=False)

    def _add_job_from_spec(self, job_json, use_job_id=True):
        """ Add a single job to the Dagobah from a spec. """

        job_id = (job_json['job_id']
                  if use_job_id else self.backend.get_new_job_id())
        self.add_job(str(job_json['name']), job_id)
        job = self.get_job(job_json['name'])
        if job_json.get('cron_schedule', None):
            job.schedule(job_json['cron_schedule'])

        for task in job_json.get('tasks', []):
            self.add_task_to_job(job,
                                 str(task['command']),
                                 str(task['name']),
                                 soft_timeout=task.get('soft_timeout', 0),
                                 hard_timeout=task.get('hard_timeout', 0))

        dependencies = job_json.get('dependencies', {})
        for from_node, to_nodes in dependencies.iteritems():
            for to_node in to_nodes:
                job.add_dependency(from_node, to_node)

    def commit(self, cascade=False):
        """ Commit this Dagobah instance to the backend.

        If cascade is True, all child Jobs are commited as well.
        """

        self.backend.commit_dagobah(self._serialize())
        if cascade:
            [job.commit() for job in self.jobs]

    def delete(self):
        """ Delete this Dagobah instance from the Backend. """
        self.jobs = []
        self.created_jobs = 0
        self.backend.delete_dagobah(self.dagobah_id)

    def add_job(self, job_name, job_id=None):
        """ Create a new, empty Job. """
        if not self._name_is_available(job_name):
            raise DagobahError('name %s is not available' % job_name)

        if not job_id:
            job_id = self.backend.get_new_job_id()
            self.created_jobs += 1

        self.jobs.append(Job(self, self.backend, job_id, job_name))

        job = self.get_job(job_name)
        job.commit()

    def get_job(self, job_name):
        """ Returns a Job by name, or None if none exists. """
        for job in self.jobs:
            if job.name == job_name:
                return job
        return None

    def delete_job(self, job_name):
        """ Delete a job by name, or error out if no such job exists. """
        for idx, job in enumerate(self.jobs):
            if job.name == job_name:
                self.backend.delete_job(job.job_id)
                del self.jobs[idx]
                self.commit()
                return
        raise DagobahError('no job with name %s exists' % job_name)

    def add_task_to_job(self,
                        job_or_job_name,
                        task_command,
                        task_name=None,
                        **kwargs):
        """ Add a task to a job owned by the Dagobah instance. """

        if isinstance(job_or_job_name, Job):
            job = job_or_job_name
        else:
            job = self.get_job(job_or_job_name)

        if not job:
            raise DagobahError('job %s does not exist' % job_or_job_name)

        if not job.state.allow_change_graph:
            raise DagobahError(
                "job's graph is immutable in its current state: %s" %
                job.state.status)

        job.add_task(task_command, task_name, **kwargs)
        job.commit()

    def _name_is_available(self, job_name):
        """ Returns Boolean of whether the specified name is already in use. """
        return (False if [job for job in self.jobs
                          if job.name == job_name] else True)

    def _serialize(self, include_run_logs=False, strict_json=False):
        """ Serialize a representation of this Dagobah object to JSON. """
        result = {
            'dagobah_id':
            self.dagobah_id,
            'created_jobs':
            self.created_jobs,
            'jobs': [
                job._serialize(include_run_logs=include_run_logs,
                               strict_json=strict_json) for job in self.jobs
            ]
        }
        if strict_json:
            result = json.loads(json.dumps(result, cls=StrictJSONEncoder))
        return result
コード例 #5
0
ファイル: core.py プロジェクト: markcol/dagobah
class Dagobah(object):
    """ Top-level controller for all Dagobah usage.

    This is in control of all the jobs for a specific Dagobah
    instance, as well as top-level parameters such as the
    backend used for permanent storage.
    """

    def __init__(self, backend=BaseBackend(), event_handler=None):
        """ Construct a new Dagobah instance with a specified Backend. """
        self.backend = backend
        self.event_handler = event_handler
        self.dagobah_id = self.backend.get_new_dagobah_id()
        self.jobs = []
        self.created_jobs = 0
        self.scheduler = Scheduler(self)
        self.scheduler.daemon = True

        self.scheduler.start()

        self.commit()


    def __repr__(self):
        return '<Dagobah with Backend %s>' % self.backend


    def from_backend(self, dagobah_id):
        """ Reconstruct this Dagobah instance from the backend. """

        rec = self.backend.get_dagobah_json(dagobah_id)
        if not rec:
            raise DagobahError('dagobah with id %s does not exist in backend' %
                               dagobah_id)

        # delete current version of this Dagobah instance
        self.delete()

        for required_key in ['dagobah_id', 'created_jobs']:
            setattr(self, required_key, rec[required_key])

        for job_json in rec.get('jobs', []):

            self.add_job(str(job_json['name']), job_json['job_id'])
            job = self.get_job(job_json['name'])
            if job_json.get('cron_schedule', None):
                job.schedule(job_json['cron_schedule'])

            for task in job_json.get('tasks', []):
                self.add_task_to_job(job,
                                     str(task['command']),
                                     str(task['name']))

            dependencies = job_json.get('dependencies', {})
            for from_node, to_nodes in dependencies.iteritems():
                for to_node in to_nodes:
                    job.add_dependency(from_node, to_node)

        self.commit(cascade=True)


    def commit(self, cascade=False):
        """ Commit this Dagobah instance to the backend.

        If cascade is True, all child Jobs are commited as well.
        """

        self.backend.commit_dagobah(self._serialize())
        if cascade:
            [job.commit() for job in self.jobs]


    def delete(self):
        """ Delete this Dagobah instance from the Backend. """
        self.jobs = []
        self.created_jobs = 0
        self.backend.delete_dagobah(self.dagobah_id)


    def add_job(self, job_name, job_id=None):
        """ Create a new, empty Job. """
        if not self._name_is_available(job_name):
            raise DagobahError('name %s is not available' % job_name)

        if not job_id:
            job_id = self.backend.get_new_job_id()
            self.created_jobs += 1

        self.jobs.append(Job(self,
                             self.backend,
                             job_id,
                             job_name))

        job = self.get_job(job_name)
        job.commit()


    def get_job(self, job_name):
        """ Returns a Job by name, or None if none exists. """
        for job in self.jobs:
            if job.name == job_name:
                return job
        return None


    def delete_job(self, job_name):
        """ Delete a job by name, or error out if no such job exists. """
        for idx, job in enumerate(self.jobs):
            if job.name == job_name:
                self.backend.delete_job(job.job_id)
                del self.jobs[idx]
                return
        raise DagobahError('no job with name %s exists' % job_name)


    def add_task_to_job(self, job_or_job_name, task_command, task_name=None):
        """ Add a task to a job owned by the Dagobah instance. """

        if isinstance(job_or_job_name, Job):
            job = job_or_job_name
        else:
            job = self.get_job(job_or_job_name)

        if not job:
            raise DagobahError('job %s does not exist' % job_or_job_name)

        if not job.state.allow_change_graph:
            raise DagobahError("job's graph is immutable in its current state: %s"
                               % job.state.status)

        job.add_task(task_command, task_name)
        job.commit()


    def _name_is_available(self, job_name):
        """ Returns Boolean of whether the specified name is already in use. """
        return (False
                if [job for job in self.jobs if job.name == job_name]
                else True)


    def _serialize(self, include_run_logs=False):
        """ Serialize a representation of this Dagobah object to JSON. """
        return {'dagobah_id': self.dagobah_id,
                'created_jobs': self.created_jobs,
                'jobs': [job._serialize(include_run_logs=include_run_logs)
                         for job in self.jobs]}
コード例 #6
0
ファイル: core.py プロジェクト: studer/dagobah
class Dagobah(object):
    """ Top-level controller for all Dagobah usage.

    This is in control of all the jobs for a specific Dagobah
    instance, as well as top-level parameters such as the
    backend used for permanent storage.
    """
    def __init__(self, backend=BaseBackend(), event_handler=None):
        """ Construct a new Dagobah instance with a specified Backend. """
        self.backend = backend
        self.event_handler = event_handler
        self.dagobah_id = self.backend.get_new_dagobah_id()
        self.jobs = []
        self.created_jobs = 0
        self.scheduler = Scheduler(self)
        self.scheduler.daemon = True

        self.scheduler.start()

        self.commit()

    def __repr__(self):
        return '<Dagobah with Backend %s>' % self.backend

    def from_backend(self, dagobah_id):
        """ Reconstruct this Dagobah instance from the backend. """

        rec = self.backend.get_dagobah_json(dagobah_id)
        if not rec:
            raise DagobahError('dagobah with id %s does not exist in backend' %
                               dagobah_id)

        # delete current version of this Dagobah instance
        self.delete()

        for required_key in ['dagobah_id', 'created_jobs']:
            setattr(self, required_key, rec[required_key])

        for job_json in rec.get('jobs', []):

            self.add_job(str(job_json['name']), job_json['job_id'])
            job = self.get_job(job_json['name'])
            if job_json.get('cron_schedule', None):
                job.schedule(job_json['cron_schedule'])

            for task in job_json.get('tasks', []):
                self.add_task_to_job(job, str(task['command']),
                                     str(task['name']))

            dependencies = job_json.get('dependencies', {})
            for from_node, to_nodes in dependencies.iteritems():
                for to_node in to_nodes:
                    job.add_dependency(from_node, to_node)

        self.commit(cascade=True)

    def commit(self, cascade=False):
        """ Commit this Dagobah instance to the backend.

        If cascade is True, all child Jobs are commited as well.
        """

        self.backend.commit_dagobah(self._serialize())
        if cascade:
            [job.commit() for job in self.jobs]

    def delete(self):
        """ Delete this Dagobah instance from the Backend. """
        self.jobs = []
        self.created_jobs = 0
        self.backend.delete_dagobah(self.dagobah_id)

    def add_job(self, job_name, job_id=None):
        """ Create a new, empty Job. """
        if not self._name_is_available(job_name):
            raise DagobahError('name %s is not available' % job_name)

        if not job_id:
            job_id = self.backend.get_new_job_id()
            self.created_jobs += 1

        self.jobs.append(Job(self, self.backend, job_id, job_name))

        job = self.get_job(job_name)
        job.commit()

    def get_job(self, job_name):
        """ Returns a Job by name, or None if none exists. """
        for job in self.jobs:
            if job.name == job_name:
                return job
        return None

    def delete_job(self, job_name):
        """ Delete a job by name, or error out if no such job exists. """
        for idx, job in enumerate(self.jobs):
            if job.name == job_name:
                self.backend.delete_job(job.job_id)
                del self.jobs[idx]
                return
        raise DagobahError('no job with name %s exists' % job_name)

    def add_task_to_job(self, job_or_job_name, task_command, task_name=None):
        """ Add a task to a job owned by the Dagobah instance. """

        if isinstance(job_or_job_name, Job):
            job = job_or_job_name
        else:
            job = self.get_job(job_or_job_name)

        if not job:
            raise DagobahError('job %s does not exist' % job_or_job_name)

        if not job.state.allow_change_graph:
            raise DagobahError(
                "job's graph is immutable in its current state: %s" %
                job.state.status)

        job.add_task(task_command, task_name)
        job.commit()

    def _name_is_available(self, job_name):
        """ Returns Boolean of whether the specified name is already in use. """
        return (False if [job for job in self.jobs
                          if job.name == job_name] else True)

    def _serialize(self, include_run_logs=False):
        """ Serialize a representation of this Dagobah object to JSON. """
        return {
            'dagobah_id':
            self.dagobah_id,
            'created_jobs':
            self.created_jobs,
            'jobs': [
                job._serialize(include_run_logs=include_run_logs)
                for job in self.jobs
            ]
        }