def _do_carte(self): tempdir = tempfile.mkdtemp() write_simple_jndi_properties(self.env, tempdir) # execute transform spoon = subprocess.Popen(["/bin/sh", "./carte.sh", "127.0.0.1", "8080"], # should come from args sometime maybe... executable="/bin/sh", cwd=resource_filename(__name__, 'pentaho-data-integration'), env={'PENTAHO_DI_JAVA_OPTIONS': "-Dorg.osjava.sj.root=%s" % os.path.join(tempdir,"simple-jndi"), 'DISPLAY': os.environ['DISPLAY']}) spoon.wait() shutil.rmtree(tempdir)
def _do_spoon(self, connection_uri=None, ip=None): tempdir = tempfile.mkdtemp() write_simple_jndi_properties(self.env, tempdir, connection_uri, ip) # execute transform spoon = subprocess.Popen(["/bin/sh", "./spoon.sh"], executable="/bin/sh", cwd=resource_filename(__name__, 'pentaho-data-integration'), env={'PENTAHO_DI_JAVA_OPTIONS': "-Dfile.encoding=utf8 -Dorg.osjava.sj.root=%s" % os.path.join(tempdir,"simple-jndi"), 'DISPLAY': os.environ['DISPLAY'], 'LANG': "en_US.UTF-8", 'HOME': os.environ['HOME']}) spoon.wait() shutil.rmtree(tempdir)
def _do_jndi(self, connection_uri=None, ip=None): write_simple_jndi_properties(self.env, os.path.expanduser("~/.pentaho"), connection_uri, ip)
def _do_execute_transformation(self, transformation, transformation_id=None, store=True, return_bytes_handle=False, changecwd=False, listall=False, parameters=None): tempdir = tempfile.mkdtemp() if changecwd: os.chdir(tempdir) os.mkdir(os.path.join(tempdir, "svn")) write_simple_jndi_properties(self.env, tempdir) # execute transform transform = self._list_transformation_files(listall)[transformation] if parameters: for parameter in parameters: if parameter not in transform['parameters']: raise KeyError("%s is not valid parameter" % parameter) else: parameters = {} parameters['DefineInternal.Project.ShortName'] = os.path.split(self.env.path)[1] scriptfilename = {'transformation': 'pan.sh', 'job': 'kitchen.sh'}[transform['type']] executable = os.path.join(resource_filename(__name__, 'pentaho-data-integration'), scriptfilename) args = [ "/bin/sh", executable, "-file", transform['full_path'], "-level", "Detailed", ] for k, v in parameters.items(): if "=" in k: raise ValueError("Unable to support = symbol in parameter key named %s" % k) args.append("-param:%s=%s" % (k.encode('utf-8'), v.encode('utf-8'))) self.log.debug("Running %s with %s", executable, args) if transformation_id: # See https://d4.define.logica.com/ticket/4375#comment:7 db = self.env.get_read_db() @self.env.with_transaction() def do_insert(db): cursor = db.cursor() self.env.log.debug("Updating running_transformations - inserting new row for %s", transformation_id) cursor.execute("""INSERT INTO running_transformations (transformation_id, status, started) VALUES (%s, %s, %s)""", (transformation_id, "running", to_utimestamp(datetime.now(utc)))) # this bit of Python isn't so good :-( I'll just merge the stdout and stderr streams... # http://stackoverflow.com/questions/6809590/merging-a-python-scripts-subprocess-stdout-and-stderr-while-keeping-them-disti # http://codereview.stackexchange.com/questions/6567/how-to-redirect-a-subprocesses-output-stdout-and-stderr-to-logging-module script = subprocess.Popen(args, executable="/bin/sh", cwd=os.path.join(tempdir, "svn"), env={'PENTAHO_DI_JAVA_OPTIONS': "-Dfile.encoding=utf8 -Dnet.sf.ehcache.skipUpdateCheck=true -Djava.awt.headless=true -Dorg.osjava.sj.root=%s" % os.path.join(tempdir,"simple-jndi"), 'LANG': "en_US.UTF-8", 'KETTLE_HOME': os.path.join(tempdir,"kettle")}, stdout=subprocess.PIPE,stderr=subprocess.STDOUT) while script.poll() is None: # this can go to the database later (natively, by pdi) # keeping here, as info level for now. self.log.info("Script output: %s", script.stdout.readline()) self.log.info("Script returned %s", script.returncode) if script.returncode: # transform has failed to complete - update running_transformations table if transformation_id: @self.env.with_transaction() def do_insert(db): cursor = db.cursor() self.env.log.debug("Updating running_transformations - %s failed to complete", transformation_id) cursor.execute("""UPDATE running_transformations SET transformation_id=%s, status=%s, ended=%s WHERE transformation_id=%s""", (transformation_id, "error", to_utimestamp(datetime.now(utc)), transformation_id)) raise RuntimeError("Business Intelligence subprocess script failed") # We know assume that the transform has finished successfully # so we update the running_transformations table to represent this if transformation_id: @self.env.with_transaction() def do_insert(db): cursor = db.cursor() self.env.log.debug("Updating running_transformations - %s completed", transformation_id) cursor.execute("""UPDATE running_transformations SET transformation_id=%s, status=%s, ended=%s WHERE transformation_id=%s""", (transformation_id, "success", to_utimestamp(datetime.now(utc)), transformation_id)) if store: reponame, repos, path = RepositoryManager(self.env).get_repository_by_path('') svn_writer = SubversionWriter(self.env, repos, "reporting") revs = [] for filename_encoded in os.listdir(os.path.join(tempdir, "svn")): filename = filename_encoded.decode('utf-8') # we wrote the filename out ourselves self.log.info("Uploading %s", filename) writer = SubversionWriter(self.env, repos, "reporting") file_data = open(os.path.join(os.path.join(tempdir, "svn"), filename)).read() for path in ["define-reports", "define-reports/%s" % transformation]: try: repos.sync() repos.get_node(path) except NoSuchNode, e: self.log.warning("Creating %s for the first time", path) writer.make_dir(path, "Generated by reporting framework") repos.sync() properties = {'define:generated-by-transformation': transformation} for k, v in parameters.items(): if not k.startswith("DefineInternal"): properties[u"define:parameter:%s" % k] = v rev = writer.put_content([("define-reports/%s/%s" % (transformation, filename), file_data)], "Generated by reporting framework transformation", properties=properties, clearproperties=True) revs.append(rev)