Example #1
0
	def _convert_val(self, param_type, val):
		if param_type["type"] == "native":
			switch = {
				"str"	: lambda x: str(x),
				"list"	: lambda x: list(x),
				"tuple"	: lambda x: tuple(x),
				"dict"	: lambda x: dict(x),
				"int"	: lambda x: int(x),
				"float"	: lambda x: float(x),
				"unicode"	: lambda x: unicode(x)
			}
			return switch[param_type["name"]](val)

		elif param_type["type"] == "fileset":
			val = FileSet(val)
			return val

		elif param_type["type"] == "component":
			# val should be like this:
			#	{ "class": "SpecificComponent", "params": {} }
			#
			# allow for inheritance by letting the json specify the
			# specific class that will be used
			component_cls = self._get_component_cls(val["class"])
			component_args = self._convert_params(val["params"], component_cls)

			val = component_cls(parent_log = self._log, job = self)
			val.init(**component_args)
			return val
Example #2
0
    def __init__(self, id, idx, params, tool, fileset_id, progress_callback, results_callback):
        """TODO: to be defined1.

        :idx: TODO
        :params: TODO
        :tool: TODO
        :fileset_id: The id of the default fileset that files should be added to

        """
        self._id = id
        self._idx = idx
        self._params = params
        self._tool = tool
        self._fileset_id = fileset_id
        self._fileset = FileSet(self._fileset_id)
        self._progress_callback = progress_callback
        self._results_callback = results_callback

        self._log = logging.getLogger("JOB:{}".format(self._id))
Example #3
0
	def __init__(self, id, idx, params, tool, fileset_id, progress_callback, results_callback):
		"""TODO: to be defined1.

		:idx: TODO
		:params: TODO
		:tool: TODO
		:fileset_id: The id of the default fileset that files should be added to

		"""
		self._id = id
		self._idx = idx
		self._params = params
		self._tool = tool
		self._fileset_id = fileset_id
		self._fileset = FileSet(self._fileset_id)
		self._progress_callback = progress_callback
		self._results_callback = results_callback

		self._log = logging.getLogger("JOB:{}".format(self._id))
Example #4
0
class Job(object):
    """This is the class that will run a task."""

    def __init__(self, id, idx, params, tool, fileset_id, progress_callback, results_callback):
        """TODO: to be defined1.

        :idx: TODO
        :params: TODO
        :tool: TODO
        :fileset_id: The id of the default fileset that files should be added to

        """
        self._id = id
        self._idx = idx
        self._params = params
        self._tool = tool
        self._fileset_id = fileset_id
        self._fileset = FileSet(self._fileset_id)
        self._progress_callback = progress_callback
        self._results_callback = results_callback

        self._log = logging.getLogger("JOB:{}".format(self._id))
    def add_file(self, contents, content_type="application/octet-stream", filename=None, **metadata):
        """Add a file to the default result fileset
        """
        return self._fileset.add(
            contents=contents,
            filename=filename,
            content_type=content_type,
            **metadata
        )

    def run(self):
        self._log.debug("preparing to run job")

        try:
            tool_cls = self._get_tool_cls()
            real_params = self._convert_params(self._params, tool_cls)
            tool = tool_cls(
                idx=self._idx,
                progress_cb=self._progress_callback,
                results_cb=self._results_callback,
                parent_log=self._log,
                job=self
            )

            self._log.debug("RUNNING TOOL")

            tool.run(**real_params)
        except TalusError as e:
            self._log.error(e.message)

        self._log.debug("FINISHED RUNNING TOOL")

    def _camel_to_under(self, name):
        s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name)
        return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower()

    def _get_tool_cls(self):
        mod_name = self._camel_to_under(self._tool)
        mod = __import__("talus.tools." + mod_name, globals(), locals(), fromlist=[str(self._tool)])
        return getattr(mod, self._tool)

    def _get_component_cls(self, cls_name):
        mod_name = self._camel_to_under(cls_name)
        mod_base = __import__("talus.components", globals(), locals(), fromlist=[str(mod_name)])
        mod = getattr(mod_base, mod_name)
        return getattr(mod, cls_name)

    def _convert_params(self, params, code_cls):
        filename = inspect.getfile(code_cls)
        param_types = self._get_param_types(code_cls)
        real_params = {}

        for name, val in params.iteritems():
            if name not in param_types:
                raise TalusError("unmapped argument: {!r}".format(name))

            real_params[name] = self._convert_val(param_types[name]["type"], val)

        return real_params

    def _to_bool(self, val):
        if type(val) is bool:
            return val
        if type(val) is str:
            return val.lower() == "true"
        return not (not val)
    
    def _convert_val(self, param_type, val):
        if param_type["type"] == "native":
            switch = {
                "str": lambda x: str(x),
                "list": lambda x: list(x),
                "tuple": lambda x: tuple(x),
                "dict": lambda x: dict(x),
                "int": lambda x: int(x),
                "float": lambda x: float(x),
                "unicode": lambda x: unicode(x)
                "bool"    : self._to_bool,
            }
            return switch[param_type["name"]](val)
Example #5
0
    def _convert_val(self, param_type, val):
        if param_type["type"] == "native":
            switch = {
                "str": lambda x: str(x),
                "list": lambda x: list(x),
                "tuple": lambda x: tuple(x),
                "dict": lambda x: dict(x),
                "int": lambda x: int(x),
                "float": lambda x: float(x),
                "unicode": lambda x: unicode(x)
                "bool"    : self._to_bool,
            }
            return switch[param_type["name"]](val)

        elif param_type["type"] == "fileset":
            val = FileSet(val)
            return val

        elif param_type["type"] == "component":
            # val should be like this:
            # { "class": "SpecificComponent", "params": {} }
            #
            # allow for inheritance by letting the json specify the
            # specific class that will be used
            component_cls = self._get_component_cls(val["class"])
            component_args = self._convert_params(val["params"], component_cls)

            val = component_cls(parent_log=self._log, job=self)
            val.init(**component_args)
            return val
Example #6
0
class Job(object):

	"""This is the class that will run a task."""

	def __init__(self, id, idx, params, tool, fileset_id, progress_callback, results_callback):
		"""TODO: to be defined1.

		:idx: TODO
		:params: TODO
		:tool: TODO
		:fileset_id: The id of the default fileset that files should be added to

		"""
		self._id = id
		self._idx = idx
		self._params = params
		self._tool = tool
		self._fileset_id = fileset_id
		self._fileset = FileSet(self._fileset_id)
		self._progress_callback = progress_callback
		self._results_callback = results_callback

		self._log = logging.getLogger("JOB:{}".format(self._id))
	
	def add_file(self, contents, content_type="application/octet-stream", filename=None, **metadata):
		"""Add a file to the default result fileset
		"""
		return self._fileset.add(
			contents		= contents,
			filename		= filename,
			content_type	= content_type,
			**metadata
		)
	
	def run(self):
		self._log.debug("preparing to run job")

		try:
			tool_cls = self._get_tool_cls()
			real_params = self._convert_params(self._params, tool_cls)
			tool = tool_cls(
				idx			= self._idx,
				progress_cb	= self._progress_callback,
				results_cb	= self._results_callback,
				parent_log	= self._log,
				job			= self
			)

			self._log.debug("RUNNING TOOL")

			tool.run(**real_params)
		except TalusError as e:
			self._log.error(e.message)

		self._log.debug("FINISHED RUNNING TOOL")
	
	def _camel_to_under(self, name):
		s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name)
		return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower()
	
	def _get_tool_cls(self):
		mod_name = self._camel_to_under(self._tool)
		mod = __import__("talus.tools." + mod_name, globals(), locals(), fromlist=[str(self._tool)])
		return getattr(mod, self._tool)

	def _get_component_cls(self, cls_name):
		mod_name = self._camel_to_under(cls_name)
		mod_base = __import__("talus.components", globals(), locals(), fromlist=[str(mod_name)])
		mod = getattr(mod_base, mod_name)
		return getattr(mod, cls_name)
	
	def _convert_params(self, params, code_cls):
		filename = inspect.getfile(code_cls)
		param_types = self._get_param_types(code_cls)
		real_params = {}

		for name,val in params.iteritems():
			if name not in param_types:
				raise TalusError("unmapped argument: {!r}".format(name))

			real_params[name] = self._convert_val(param_types[name]["type"], val)
		
		return real_params
	
	def _convert_val(self, param_type, val):
		if param_type["type"] == "native":
			switch = {
				"str"	: lambda x: str(x),
				"list"	: lambda x: list(x),
				"tuple"	: lambda x: tuple(x),
				"dict"	: lambda x: dict(x),
				"int"	: lambda x: int(x),
				"float"	: lambda x: float(x),
				"unicode"	: lambda x: unicode(x)
			}
			return switch[param_type["name"]](val)

		elif param_type["type"] == "fileset":
			val = FileSet(val)
			return val

		elif param_type["type"] == "component":
			# val should be like this:
			#	{ "class": "SpecificComponent", "params": {} }
			#
			# allow for inheritance by letting the json specify the
			# specific class that will be used
			component_cls = self._get_component_cls(val["class"])
			component_args = self._convert_params(val["params"], component_cls)

			val = component_cls(parent_log = self._log, job = self)
			val.init(**component_args)
			return val
	
	def _get_param_types(self, cls):
		cls_name = cls.__name__
		filename = inspect.getfile(cls)

		with open(filename, "r") as f:
			source = f.read()

		pyclass = None
		mod = ast.parse(source)
		for node in mod.body:
			if isinstance(node,ast.ClassDef):
				cls = PyClass(self._log, filename, node)
				if cls.name == cls_name:
					pyclass = cls
					break

		# make the query func always return true - this is assuming the
		# pre-receive hook has done its job and prevented invalid components
		# from slipping in
		return pyclass.get_run_params(lambda x: True)