def _do_carte(self):
tempdir = tempfile.mkdtemp()
write_simple_jndi_properties(self.env, tempdir)
# execute transform
spoon = subprocess.Popen(["/bin/sh", "./carte.sh", "127.0.0.1", "8080"], # should come from args sometime maybe...
executable="/bin/sh",
cwd=resource_filename(__name__, 'pentaho-data-integration'),
env={'PENTAHO_DI_JAVA_OPTIONS': "-Dorg.osjava.sj.root=%s" % os.path.join(tempdir,"simple-jndi"),
'DISPLAY': os.environ['DISPLAY']})
spoon.wait()
shutil.rmtree(tempdir)
def _do_spoon(self, connection_uri=None, ip=None):
tempdir = tempfile.mkdtemp()
write_simple_jndi_properties(self.env, tempdir, connection_uri, ip)
# execute transform
spoon = subprocess.Popen(["/bin/sh", "./spoon.sh"],
executable="/bin/sh",
cwd=resource_filename(__name__, 'pentaho-data-integration'),
env={'PENTAHO_DI_JAVA_OPTIONS': "-Dfile.encoding=utf8 -Dorg.osjava.sj.root=%s" % os.path.join(tempdir,"simple-jndi"),
'DISPLAY': os.environ['DISPLAY'],
'LANG': "en_US.UTF-8",
'HOME': os.environ['HOME']})
spoon.wait()
shutil.rmtree(tempdir)
def _do_jndi(self, connection_uri=None, ip=None):
write_simple_jndi_properties(self.env, os.path.expanduser("~/.pentaho"),
connection_uri, ip)
def _do_execute_transformation(self, transformation, transformation_id=None,
store=True, return_bytes_handle=False,
changecwd=False, listall=False, parameters=None):
tempdir = tempfile.mkdtemp()
if changecwd:
os.chdir(tempdir)
os.mkdir(os.path.join(tempdir, "svn"))
write_simple_jndi_properties(self.env, tempdir)
# execute transform
transform = self._list_transformation_files(listall)[transformation]
if parameters:
for parameter in parameters:
if parameter not in transform['parameters']:
raise KeyError("%s is not valid parameter" % parameter)
else:
parameters = {}
parameters['DefineInternal.Project.ShortName'] = os.path.split(self.env.path)[1]
scriptfilename = {'transformation': 'pan.sh',
'job': 'kitchen.sh'}[transform['type']]
executable = os.path.join(resource_filename(__name__, 'pentaho-data-integration'), scriptfilename)
args = [
"/bin/sh",
executable,
"-file", transform['full_path'],
"-level", "Detailed",
]
for k, v in parameters.items():
if "=" in k:
raise ValueError("Unable to support = symbol in parameter key named %s" % k)
args.append("-param:%s=%s" % (k.encode('utf-8'), v.encode('utf-8')))
self.log.debug("Running %s with %s", executable, args)
if transformation_id:
# See https://d4.define.logica.com/ticket/4375#comment:7
db = self.env.get_read_db()
@self.env.with_transaction()
def do_insert(db):
cursor = db.cursor()
self.env.log.debug("Updating running_transformations - inserting new row for %s",
transformation_id)
cursor.execute("""INSERT INTO running_transformations (transformation_id, status, started)
VALUES (%s, %s, %s)""", (transformation_id, "running", to_utimestamp(datetime.now(utc))))
# this bit of Python isn't so good :-( I'll just merge the stdout and stderr streams...
# http://stackoverflow.com/questions/6809590/merging-a-python-scripts-subprocess-stdout-and-stderr-while-keeping-them-disti
# http://codereview.stackexchange.com/questions/6567/how-to-redirect-a-subprocesses-output-stdout-and-stderr-to-logging-module
script = subprocess.Popen(args,
executable="/bin/sh",
cwd=os.path.join(tempdir, "svn"),
env={'PENTAHO_DI_JAVA_OPTIONS': "-Dfile.encoding=utf8 -Dnet.sf.ehcache.skipUpdateCheck=true -Djava.awt.headless=true -Dorg.osjava.sj.root=%s" % os.path.join(tempdir,"simple-jndi"),
'LANG': "en_US.UTF-8",
'KETTLE_HOME': os.path.join(tempdir,"kettle")},
stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
while script.poll() is None:
# this can go to the database later (natively, by pdi)
# keeping here, as info level for now.
self.log.info("Script output: %s", script.stdout.readline())
self.log.info("Script returned %s", script.returncode)
if script.returncode:
# transform has failed to complete - update running_transformations table
if transformation_id:
@self.env.with_transaction()
def do_insert(db):
cursor = db.cursor()
self.env.log.debug("Updating running_transformations - %s failed to complete",
transformation_id)
cursor.execute("""UPDATE running_transformations
SET transformation_id=%s, status=%s, ended=%s
WHERE transformation_id=%s""", (transformation_id, "error", to_utimestamp(datetime.now(utc)), transformation_id))
raise RuntimeError("Business Intelligence subprocess script failed")
# We know assume that the transform has finished successfully
# so we update the running_transformations table to represent this
if transformation_id:
@self.env.with_transaction()
def do_insert(db):
cursor = db.cursor()
self.env.log.debug("Updating running_transformations - %s completed",
transformation_id)
cursor.execute("""UPDATE running_transformations
SET transformation_id=%s, status=%s, ended=%s
WHERE transformation_id=%s""", (transformation_id, "success", to_utimestamp(datetime.now(utc)), transformation_id))
if store:
reponame, repos, path = RepositoryManager(self.env).get_repository_by_path('')
svn_writer = SubversionWriter(self.env, repos, "reporting")
revs = []
for filename_encoded in os.listdir(os.path.join(tempdir, "svn")):
filename = filename_encoded.decode('utf-8') # we wrote the filename out ourselves
self.log.info("Uploading %s", filename)
writer = SubversionWriter(self.env, repos, "reporting")
file_data = open(os.path.join(os.path.join(tempdir, "svn"), filename)).read()
for path in ["define-reports",
"define-reports/%s" % transformation]:
try:
repos.sync()
repos.get_node(path)
except NoSuchNode, e:
self.log.warning("Creating %s for the first time",
path)
writer.make_dir(path, "Generated by reporting framework")
repos.sync()
properties = {'define:generated-by-transformation': transformation}
for k, v in parameters.items():
if not k.startswith("DefineInternal"):
properties[u"define:parameter:%s" % k] = v
rev = writer.put_content([("define-reports/%s/%s" % (transformation, filename), file_data)],
"Generated by reporting framework transformation",
properties=properties,
clearproperties=True)
revs.append(rev)
