예제 #1
0
파일: jobs.py 프로젝트: philloidin/wok
    def __init__(self, name, conf):
        self._name = name
        self._conf = conf

        self._log = logger.get_logger("wok.jobs.{}".format(name))

        if "work_path" not in self._conf:
            raise MissingConfigParamError("work_path")

        self._work_path = conf["work_path"]
        if not os.path.exists(self._work_path):
            os.makedirs(self._work_path)

        db_path = os.path.join(self._work_path, "jobs.db")
        # TODO if not in recover_mode then delete jobs.db
        self._db = create_engine("sqlite:///{}".format(db_path))
        self._session_factory = create_session_factory(self._db)

        self._callbacks = CallbackManager(valid_events=[events.JOB_UPDATE])

        # Lock for database jobs state changes
        self._lock = threading.Lock()
예제 #2
0
    def __init__(self, name, conf):
        self._conf = conf
        self._name = conf.get("name", name)

        self._log = logger.get_logger("wok.platform.{}".format(name))

        if "work_path" not in self._conf:
            raise MissingConfigParamError("work_path")

        self._work_path = conf["work_path"]
        if not os.path.exists(self._work_path):
            os.makedirs(self._work_path)

        self._data = self._create_data_provider()

        self._storage = self._create_storage()

        self._job_manager = self._create_job_manager()

        self._callbacks = CallbackManager(
            delegates=[(events.JOB_UPDATE, self._job_manager.callbacks)])
예제 #3
0
파일: jobs.py 프로젝트: bbglab/wok
	def __init__(self, name, conf):
		self._name = name
		self._conf = conf

		self._log = logger.get_logger("wok.jobs.{}".format(name))

		if "work_path" not in self._conf:
			raise MissingConfigParamError("work_path")

		self._work_path = conf["work_path"]
		if not os.path.exists(self._work_path):
			os.makedirs(self._work_path)

		db_path = os.path.join(self._work_path, "jobs.db")
		# TODO if not in recover_mode then delete jobs.db
		self._db = create_engine("sqlite:///{}".format(db_path))
		self._session_factory = create_session_factory(self._db)

		self._callbacks = CallbackManager(valid_events=[events.JOB_UPDATE])

		# Lock for database jobs state changes
		self._lock = threading.Lock()
예제 #4
0
파일: jobs.py 프로젝트: philloidin/wok
class JobManager(object):
    """
	Base class for Job managers. It implements the main interface (start, submit, join, ...)
	and manages de jobs database, then it delegates core operations to the child class (_start, _submit, ...)

	Configuration namespace: **wok.jobs**
	    **work_path**: Working path for temporary files and databases.
	"""

    job_class = Job

    def __init__(self, name, conf):
        self._name = name
        self._conf = conf

        self._log = logger.get_logger("wok.jobs.{}".format(name))

        if "work_path" not in self._conf:
            raise MissingConfigParamError("work_path")

        self._work_path = conf["work_path"]
        if not os.path.exists(self._work_path):
            os.makedirs(self._work_path)

        db_path = os.path.join(self._work_path, "jobs.db")
        # TODO if not in recover_mode then delete jobs.db
        self._db = create_engine("sqlite:///{}".format(db_path))
        self._session_factory = create_session_factory(self._db)

        self._callbacks = CallbackManager(valid_events=[events.JOB_UPDATE])

        # Lock for database jobs state changes
        self._lock = threading.Lock()
        #self._lock = LogLock("lock", self._log)

    def _create_session(self):
        return self._session_factory()

    @property
    def callbacks(self):
        return self._callbacks

    def _fire_job_updated(self, job):
        self._callbacks.fire(events.JOB_UPDATE,
                             job_id=job.id,
                             workitem_cname=job.name,
                             state=job.state)

    def _get_job(self, job_id):
        session = self._create_session()
        job = session.query(
            self.job_class).filter(self.job_class.id == job_id).first()
        session.close()
        return job

    def _task_conf(self, job, key, context=None, default=None):
        task_conf = job.task_conf
        if context is not None:
            key = "{}.{}".format(context, key)
        return task_conf.get(key, default)

    def add_callback(self, event, callback):
        """
		Adds an event callback. For example whenever there are job state changes it will be called.
		:param callback: callee accepting the following parameters: event, **kargs
		:return:
		"""
        self._callbacks[event].add(callback)

    def _attach(self, session, job):
        session.delete(job)

    def start(self):
        "Start the job manager."

        self._log.info("Starting '{}' job manager ...".format(self._name))

        with self._lock:
            session = self._create_session()
            for job in session.query(self.job_class):
                self._attach(session, job)
                session.commit()
            session.close()

        self._start()

        self._log.info("Job manager '{}' started ...".format(self._name))

    def _submit(self, job):
        pass

    def submit(self, submissions):
        """
		Submit jobs.
		:param submissions: list or iterator with the job submissions.
		:return: Iterator of (job_submission, job_id) for all the tasks.
		"""

        with self._lock:
            session = self._create_session()
            try:
                for js in submissions:
                    job = self.job_class(js)
                    self._submit(job)
                    session.add(job)
                    session.commit()
                    self._lock.release()
                    yield js, job.id, job.state
                    self._lock.acquire()
            finally:
                session.close()

    def state(self, job_ids=None):
        """
		Returns an iterator of (job_id, state) for all the jobs in jobs_id. If job_ids is None then all the jobs are queried.
		:param job_ids: list of job ids or None
		:return: Iterator of (job_id, state)
		"""

        with self._lock:
            session = self._create_session()
            query = session.query(self.job_class.id, self.job_class.state)
            if job_ids is not None:
                query = query.filter(self.job_class.state.in_(job_ids))
            for job_id, job_state in query:
                self._lock.release()
                yield job_id, job_state
                self._lock.acquire()
            session.close()

    def output(self, job_id):
        "Get the output of the job"

        with self._lock:
            job = self._get_job(job_id)
            if job is None:
                return None

            self._log.debug("Retrieving output for job [{}] {} ...".format(
                job.id, job.name))

            return self._output(job)

    def join(self, job_id):
        "Retrieve the job results and delete the job from memory."

        with self._lock:
            session = self._create_session()
            job = session.query(
                self.job_class).filter(self.job_class.id == job_id).first()

            if job is None:
                session.close()
                return JobResults()

            self._log.debug("Joining job [{}] {} ...".format(job.id, job.name))

            self._join(session, job)

            session.delete(job)
            session.commit()
            session.close()

        return JobResults(job)

    def _abort(self, session, job):
        pass

    def abort(self, job_ids=None):
        "Aborts a job"

        with self._lock:
            session = self._create_session()

            abortable_states = {
                runstates.WAITING: runstates.ABORTED,
                runstates.RUNNING: runstates.ABORTING
            }

            for job_id in job_ids:
                job = session.query(self.job_class).filter(
                    self.job_class.id == job_id).first()

                if job is None or job.state not in abortable_states:
                    continue

                job.state = abortable_states[job.state]

                self._log.debug("{} job [{}] {} ...".format(
                    job.state.title.capitalize(), job.id, job.name))

                self._abort(session, job)

            session.commit()
            session.close()

    def close(self):
        "Close the job manager and free resources."

        self._log.info("Closing '{}' job manager ...".format(self._name))

        with self._lock:
            session = self._create_session()

            session.query(self.job_class).filter(
                self.job_class.state == runstates.WAITING).update(
                    {self.job_class.state: runstates.ABORTED},
                    synchronize_session=False)
            session.commit()

            self._close()

        session.close()

        self._session_factory.remove()

        self._log.info("Job manager '{}' closed".format(self._name))
예제 #5
0
파일: jobs.py 프로젝트: bbglab/wok
class JobManager(object):
	"""
	Base class for Job managers. It implements the main interface (start, submit, join, ...)
	and manages de jobs database, then it delegates core operations to the child class (_start, _submit, ...)

	Configuration namespace: **wok.jobs**
	    **work_path**: Working path for temporary files and databases.
	"""

	job_class = Job

	def __init__(self, name, conf):
		self._name = name
		self._conf = conf

		self._log = logger.get_logger("wok.jobs.{}".format(name))

		if "work_path" not in self._conf:
			raise MissingConfigParamError("work_path")

		self._work_path = conf["work_path"]
		if not os.path.exists(self._work_path):
			os.makedirs(self._work_path)

		db_path = os.path.join(self._work_path, "jobs.db")
		# TODO if not in recover_mode then delete jobs.db
		self._db = create_engine("sqlite:///{}".format(db_path))
		self._session_factory = create_session_factory(self._db)

		self._callbacks = CallbackManager(valid_events=[events.JOB_UPDATE])

		# Lock for database jobs state changes
		self._lock = threading.Lock()
		#self._lock = LogLock("lock", self._log)

	def _create_session(self):
		return self._session_factory()

	@property
	def callbacks(self):
		return self._callbacks

	def _fire_job_updated(self, job):
		self._callbacks.fire(events.JOB_UPDATE, job_id=job.id, workitem_cname=job.name, state=job.state)

	def _get_job(self, job_id):
		session = self._create_session()
		job = session.query(self.job_class).filter(self.job_class.id == job_id).first()
		session.close()
		return job

	def _task_conf(self, job, key, context=None, default=None):
		task_conf = job.task_conf
		if context is not None:
			key = "{}.{}".format(context, key)
		return task_conf.get(key, default)

	def add_callback(self, event, callback):
		"""
		Adds an event callback. For example whenever there are job state changes it will be called.
		:param callback: callee accepting the following parameters: event, **kargs
		:return:
		"""
		self._callbacks[event].add(callback)

	def _attach(self, session, job):
		session.delete(job)

	def start(self):
		"Start the job manager."

		self._log.info("Starting '{}' job manager ...".format(self._name))

		with self._lock:
			session = self._create_session()
			for job in session.query(self.job_class):
				self._attach(session, job)
				session.commit()
			session.close()

		self._start()

		self._log.info("Job manager '{}' started ...".format(self._name))

	def _submit(self, job):
		pass

	def submit(self, submissions):
		"""
		Submit jobs.
		:param submissions: list or iterator with the job submissions.
		:return: Iterator of (job_submission, job_id) for all the tasks.
		"""

		with self._lock:
			session = self._create_session()
			try:
				for js in submissions:
					job = self.job_class(js)
					self._submit(job)
					session.add(job)
					session.commit()
					self._lock.release()
					yield js, job.id, job.state
					self._lock.acquire()
			finally:
				session.close()

	def state(self, job_ids=None):
		"""
		Returns an iterator of (job_id, state) for all the jobs in jobs_id. If job_ids is None then all the jobs are queried.
		:param job_ids: list of job ids or None
		:return: Iterator of (job_id, state)
		"""

		with self._lock:
			session = self._create_session()
			query = session.query(self.job_class.id, self.job_class.state)
			if job_ids is not None:
				query = query.filter(self.job_class.state.in_(job_ids))
			for job_id, job_state in query:
				self._lock.release()
				yield job_id, job_state
				self._lock.acquire()
			session.close()

	def output(self, job_id):
		"Get the output of the job"

		with self._lock:
			job = self._get_job(job_id)
			if job is None:
				return None

			self._log.debug("Retrieving output for job [{}] {} ...".format(job.id, job.name))

			return self._output(job)

	def join(self, job_id):
		"Retrieve the job results and delete the job from memory."

		with self._lock:
			session = self._create_session()
			job = session.query(self.job_class).filter(self.job_class.id == job_id).first()

			if job is None:
				session.close()
				return JobResults()

			self._log.debug("Joining job [{}] {} ...".format(job.id, job.name))

			self._join(session, job)

			session.delete(job)
			session.commit()
			session.close()

		return JobResults(job)

	def _abort(self, session, job):
		pass

	def abort(self, job_ids=None):
		"Aborts a job"

		with self._lock:
			session = self._create_session()

			abortable_states = {
					runstates.WAITING : runstates.ABORTED,
					runstates.RUNNING : runstates.ABORTING }

			for job_id in job_ids:
				job = session.query(self.job_class).filter(self.job_class.id == job_id).first()

				if job is None or job.state not in abortable_states:
					continue

				job.state = abortable_states[job.state]

				self._log.debug("{} job [{}] {} ...".format(job.state.title.capitalize(), job.id, job.name))

				self._abort(session, job)

			session.commit()
			session.close()

	def close(self):
		"Close the job manager and free resources."

		self._log.info("Closing '{}' job manager ...".format(self._name))

		with self._lock:
			session = self._create_session()

			session.query(self.job_class).filter(
				self.job_class.state == runstates.WAITING).update(
				{self.job_class.state : runstates.ABORTED}, synchronize_session=False)
			session.commit()

			self._close()

		session.close()

		self._session_factory.remove()

		self._log.info("Job manager '{}' closed".format(self._name))