def _upload_request(self, req, handler): self.log.debug('Handling file upload for "%s"', req.authname) # Retrieve uploaded file upload_file = req.args['bsop_upload_file'] # Retrieve filename, normalize, use to check a file was uploaded # Filename checks adapted from trac/attachment.py filename = getattr(upload_file, 'filename', '') filename = unicodedata.normalize('NFC', unicode(filename, 'utf-8')) filename = filename.replace('\\', '/').replace(':', '/') filename = posixpath.basename(filename) if not filename: raise TracError('No file uploaded') # Check size of uploaded file, accepting 0 to max_upload_size bytes file_data = upload_file.value # Alternatively .file for file object file_size = len(file_data) if self.max_upload_size > 0 and file_size > self.max_upload_size: raise TracError('Uploaded file is too large, ' 'maximum upload size: %s' % pretty_size(self.max_upload_size)) self.log.debug('Received file %s with %i bytes', filename, file_size) commit_msg = req.args.get('bsop_upload_commit') self.log.debug('Opening repository for file upload') reponame, repos, path = _get_repository(self.env, req) try: repos_path = repos.normalize_path('/'.join([path, filename])) self.log.debug('Writing file %s to %s in %s', filename, repos_path, reponame) svn_writer = SubversionWriter(self.env, repos, req.authname) rev = svn_writer.put_content(repos_path, file_data, commit_msg) add_notice( req, _("Uploaded %s, creating revision %s.") % (filename, rev)) except Exception, e: self.log.exception("Failed when uploading file %s" % filename) add_warning(req, _("Failed to upload file: %s") % e)
def _upload_request(self, req, handler): self.log.debug('Handling file upload for "%s"', req.authname) # Retrieve uploaded file upload_file = req.args['bsop_upload_file'] # Retrieve filename, normalize, use to check a file was uploaded # Filename checks adapted from trac/attachment.py filename = getattr(upload_file, 'filename', '') filename = unicodedata.normalize('NFC', unicode(filename, 'utf-8')) filename = filename.replace('\\', '/').replace(':', '/') filename = posixpath.basename(filename) if not filename: raise TracError('No file uploaded') # Check size of uploaded file, accepting 0 to max_upload_size bytes file_data = upload_file.value # Alternatively .file for file object file_size = len(file_data) if self.max_upload_size > 0 and file_size > self.max_upload_size: raise TracError('Uploaded file is too large, ' 'maximum upload size: %s' % pretty_size(self.max_upload_size)) self.log.debug('Received file %s with %i bytes', filename, file_size) commit_msg = req.args.get('bsop_upload_commit') self.log.debug('Opening repository for file upload') reponame, repos, path = _get_repository(self.env, req) try: repos_path = repos.normalize_path('/'.join([path, filename])) self.log.debug('Writing file %s to %s in %s', filename, repos_path, reponame) svn_writer = SubversionWriter(self.env, repos, req.authname) rev = svn_writer.put_content(repos_path, file_data, commit_msg) add_notice(req, _("Uploaded %s, creating revision %s.") % (filename, rev)) except Exception, e: self.log.exception("Failed when uploading file %s" % filename) add_warning(req, _("Failed to upload file: %s") % e)
def test_add_changeset(self): sw = SubversionWriter(self.env, self.repos, 'kalle') new_rev = sw.put_content('/trunk/foo.txt', content='Foo Bar', commit_msg='A comment') RepositoryManager(self.env).notify('changeset_added', '', [new_rev]) # Node so = self._get_so() self.assertEquals('trac:source:trunk/foo.txt', so.doc_id) self.assertEquals('source', so.realm) self.assertEquals('trunk/foo.txt', so.id) self.assertEquals('trunk/foo.txt', so.title) self.assertEquals('kalle', so.author) self.assertEquals('Foo Bar', so.body.read()) self.assertTrue('A comment' in so.comments) # Changeset so = self._get_so(-2) self.assertEquals('trac:changeset:%i' % new_rev, so.doc_id) self.assertEquals('changeset', so.realm) self.assertEquals('%i' % new_rev, so.id) self.assertTrue(so.title.startswith('[%i]: A comment' % new_rev)) self.assertEquals('kalle', so.author) self.assertEquals('A comment', so.body)
def test_add_changeset(self): sw = SubversionWriter(self.env, self.repos, 'kalle') new_rev = sw.put_content('/trunk/foo.txt', content='Foo Bar', commit_msg='A comment') RepositoryManager(self.env).notify('changeset_added', '', [new_rev]) # Node so = self._get_so() self.assertEquals('trac:source:trunk/foo.txt', so.doc_id) self.assertEquals('source', so.realm) self.assertEquals('trunk/foo.txt', so.id) self.assertEquals('trunk/foo.txt', so.title) self.assertEquals('kalle', so.author) self.assertEquals('Foo Bar', so.body.read()) self.assertTrue('A comment' in so.comments) # Changeset so = self._get_so(-2) self.assertEquals('trac:changeset:%i' % new_rev, so.doc_id) self.assertEquals('changeset', so.realm) self.assertEquals('%i' % new_rev, so.id) self.assertTrue(so.title.startswith('[%i]: A comment' % new_rev)) self.assertEquals('kalle', so.author) self.assertEquals('A comment', so.body)
def _do_execute_transformation(self, transformation, transformation_id=None, store=True, return_bytes_handle=False, changecwd=False, listall=False, parameters=None): tempdir = tempfile.mkdtemp() if changecwd: os.chdir(tempdir) os.mkdir(os.path.join(tempdir, "svn")) write_simple_jndi_properties(self.env, tempdir) # execute transform transform = self._list_transformation_files(listall)[transformation] if parameters: for parameter in parameters: if parameter not in transform['parameters']: raise KeyError("%s is not valid parameter" % parameter) else: parameters = {} parameters['DefineInternal.Project.ShortName'] = os.path.split(self.env.path)[1] scriptfilename = {'transformation': 'pan.sh', 'job': 'kitchen.sh'}[transform['type']] executable = os.path.join(resource_filename(__name__, 'pentaho-data-integration'), scriptfilename) args = [ "/bin/sh", executable, "-file", transform['full_path'], "-level", "Detailed", ] for k, v in parameters.items(): if "=" in k: raise ValueError("Unable to support = symbol in parameter key named %s" % k) args.append("-param:%s=%s" % (k.encode('utf-8'), v.encode('utf-8'))) self.log.debug("Running %s with %s", executable, args) if transformation_id: # See https://d4.define.logica.com/ticket/4375#comment:7 db = self.env.get_read_db() @self.env.with_transaction() def do_insert(db): cursor = db.cursor() self.env.log.debug("Updating running_transformations - inserting new row for %s", transformation_id) cursor.execute("""INSERT INTO running_transformations (transformation_id, status, started) VALUES (%s, %s, %s)""", (transformation_id, "running", to_utimestamp(datetime.now(utc)))) # this bit of Python isn't so good :-( I'll just merge the stdout and stderr streams... # http://stackoverflow.com/questions/6809590/merging-a-python-scripts-subprocess-stdout-and-stderr-while-keeping-them-disti # http://codereview.stackexchange.com/questions/6567/how-to-redirect-a-subprocesses-output-stdout-and-stderr-to-logging-module script = subprocess.Popen(args, executable="/bin/sh", cwd=os.path.join(tempdir, "svn"), env={'PENTAHO_DI_JAVA_OPTIONS': "-Dfile.encoding=utf8 -Dnet.sf.ehcache.skipUpdateCheck=true -Djava.awt.headless=true -Dorg.osjava.sj.root=%s" % os.path.join(tempdir,"simple-jndi"), 'LANG': "en_US.UTF-8", 'KETTLE_HOME': os.path.join(tempdir,"kettle")}, stdout=subprocess.PIPE,stderr=subprocess.STDOUT) while script.poll() is None: # this can go to the database later (natively, by pdi) # keeping here, as info level for now. self.log.info("Script output: %s", script.stdout.readline()) self.log.info("Script returned %s", script.returncode) if script.returncode: # transform has failed to complete - update running_transformations table if transformation_id: @self.env.with_transaction() def do_insert(db): cursor = db.cursor() self.env.log.debug("Updating running_transformations - %s failed to complete", transformation_id) cursor.execute("""UPDATE running_transformations SET transformation_id=%s, status=%s, ended=%s WHERE transformation_id=%s""", (transformation_id, "error", to_utimestamp(datetime.now(utc)), transformation_id)) raise RuntimeError("Business Intelligence subprocess script failed") # We know assume that the transform has finished successfully # so we update the running_transformations table to represent this if transformation_id: @self.env.with_transaction() def do_insert(db): cursor = db.cursor() self.env.log.debug("Updating running_transformations - %s completed", transformation_id) cursor.execute("""UPDATE running_transformations SET transformation_id=%s, status=%s, ended=%s WHERE transformation_id=%s""", (transformation_id, "success", to_utimestamp(datetime.now(utc)), transformation_id)) if store: reponame, repos, path = RepositoryManager(self.env).get_repository_by_path('') svn_writer = SubversionWriter(self.env, repos, "reporting") revs = [] for filename_encoded in os.listdir(os.path.join(tempdir, "svn")): filename = filename_encoded.decode('utf-8') # we wrote the filename out ourselves self.log.info("Uploading %s", filename) writer = SubversionWriter(self.env, repos, "reporting") file_data = open(os.path.join(os.path.join(tempdir, "svn"), filename)).read() for path in ["define-reports", "define-reports/%s" % transformation]: try: repos.sync() repos.get_node(path) except NoSuchNode, e: self.log.warning("Creating %s for the first time", path) writer.make_dir(path, "Generated by reporting framework") repos.sync() properties = {'define:generated-by-transformation': transformation} for k, v in parameters.items(): if not k.startswith("DefineInternal"): properties[u"define:parameter:%s" % k] = v rev = writer.put_content([("define-reports/%s/%s" % (transformation, filename), file_data)], "Generated by reporting framework transformation", properties=properties, clearproperties=True) revs.append(rev)