def _upload_request(self, req, handler): self.log.debug('Handling file upload for "%s"', req.authname) # Retrieve uploaded file upload_file = req.args['bsop_upload_file'] # Retrieve filename, normalize, use to check a file was uploaded # Filename checks adapted from trac/attachment.py filename = getattr(upload_file, 'filename', '') filename = unicodedata.normalize('NFC', unicode(filename, 'utf-8')) filename = filename.replace('\\', '/').replace(':', '/') filename = posixpath.basename(filename) if not filename: raise TracError('No file uploaded') # Check size of uploaded file, accepting 0 to max_upload_size bytes file_data = upload_file.value # Alternatively .file for file object file_size = len(file_data) if self.max_upload_size > 0 and file_size > self.max_upload_size: raise TracError('Uploaded file is too large, ' 'maximum upload size: %s' % pretty_size(self.max_upload_size)) self.log.debug('Received file %s with %i bytes', filename, file_size) commit_msg = req.args.get('bsop_upload_commit') self.log.debug('Opening repository for file upload') reponame, repos, path = _get_repository(self.env, req) try: repos_path = repos.normalize_path('/'.join([path, filename])) self.log.debug('Writing file %s to %s in %s', filename, repos_path, reponame) svn_writer = SubversionWriter(self.env, repos, req.authname) rev = svn_writer.put_content(repos_path, file_data, commit_msg) add_notice( req, _("Uploaded %s, creating revision %s.") % (filename, rev)) except Exception, e: self.log.exception("Failed when uploading file %s" % filename) add_warning(req, _("Failed to upload file: %s") % e)
def _upload_request(self, req, handler): self.log.debug('Handling file upload for "%s"', req.authname) # Retrieve uploaded file upload_file = req.args['bsop_upload_file'] # Retrieve filename, normalize, use to check a file was uploaded # Filename checks adapted from trac/attachment.py filename = getattr(upload_file, 'filename', '') filename = unicodedata.normalize('NFC', unicode(filename, 'utf-8')) filename = filename.replace('\\', '/').replace(':', '/') filename = posixpath.basename(filename) if not filename: raise TracError('No file uploaded') # Check size of uploaded file, accepting 0 to max_upload_size bytes file_data = upload_file.value # Alternatively .file for file object file_size = len(file_data) if self.max_upload_size > 0 and file_size > self.max_upload_size: raise TracError('Uploaded file is too large, ' 'maximum upload size: %s' % pretty_size(self.max_upload_size)) self.log.debug('Received file %s with %i bytes', filename, file_size) commit_msg = req.args.get('bsop_upload_commit') self.log.debug('Opening repository for file upload') reponame, repos, path = _get_repository(self.env, req) try: repos_path = repos.normalize_path('/'.join([path, filename])) self.log.debug('Writing file %s to %s in %s', filename, repos_path, reponame) svn_writer = SubversionWriter(self.env, repos, req.authname) rev = svn_writer.put_content(repos_path, file_data, commit_msg) add_notice(req, _("Uploaded %s, creating revision %s.") % (filename, rev)) except Exception, e: self.log.exception("Failed when uploading file %s" % filename) add_warning(req, _("Failed to upload file: %s") % e)
def test_add_changeset(self): sw = SubversionWriter(self.env, self.repos, 'kalle') new_rev = sw.put_content('/trunk/foo.txt', content='Foo Bar', commit_msg='A comment') RepositoryManager(self.env).notify('changeset_added', '', [new_rev]) # Node so = self._get_so() self.assertEquals('trac:source:trunk/foo.txt', so.doc_id) self.assertEquals('source', so.realm) self.assertEquals('trunk/foo.txt', so.id) self.assertEquals('trunk/foo.txt', so.title) self.assertEquals('kalle', so.author) self.assertEquals('Foo Bar', so.body.read()) self.assertTrue('A comment' in so.comments) # Changeset so = self._get_so(-2) self.assertEquals('trac:changeset:%i' % new_rev, so.doc_id) self.assertEquals('changeset', so.realm) self.assertEquals('%i' % new_rev, so.id) self.assertTrue(so.title.startswith('[%i]: A comment' % new_rev)) self.assertEquals('kalle', so.author) self.assertEquals('A comment', so.body)
def _create_path_request(self, req, handler): self.log.debug('Handling create folder for %s', req.authname) create_name = req.args.get('bsop_create_folder_name') commit_msg = req.args.get('bsop_create_commit') self.log.debug('Opening repository to create folder') reponame, repos, path = _get_repository(self.env, req) try: create_path = repos.normalize_path('/'.join([path, create_name])) svn_writer = SubversionWriter(self.env, repos, req.authname) try: self.log.info('Creating folder %s in repository %s', create_path, reponame) svn_writer.make_dir(create_path, commit_msg) except Exception, e: self.log.exception("Failed to create directory: %s", e) add_warning(req, "Failed to create folder: %s" % e) add_notice(req, _("Folder %s created.") % create_name)
def _move_delete_request(self, req, handler): self.log.debug('Handling move/delete for %s', req.authname) operation = req.args.get('bsop_mvdel_op') #Moving or deleting? src_names = req.args.getlist('bsop_mvdel_src_name') # Items to move/del dst_name = req.args.get('bsop_mvdel_dst_name') # Destination if move commit_msg = req.args.get('bsop_mvdel_commit') # ContextMenuPlugin provides each src as [reponame/][page_path/]node_path # The Trac request path is of the form /[reponame/][page_path] # Retrieve just the portion of the src items that is relative to this # page by stripping the request path. Do this before the request path # is itself stripped of reponame when retrieving the repository path = req.args.get('path') src_names = [posixpath.join('/', src_name) for src_name in src_names] src_names = [src_name.split(path, 1)[1] for src_name in src_names] self.log.debug('Received %i source items to "%s"', len(src_names), operation) # Enforce rename_only mode if operation == 'move' and self.rename_only: # Do not allow move operation to place src in dst, if dst exists move_as_child = False # Check that move affects only one node if len(src_names) > 1: self.log.error('Attempted to rename %i nodes, only 1 node can' 'be renamed at a time', len(src_names)) raise TracError('Cannot rename multiple files or folders.') # Check that destination is just filename # i.e. it doesn't contain any path components elif '/' in dst_name: self.log.error('Rename encountered path seperator "/" in' 'destination ("%s", "%s")', src_names[0], dst_name) raise TracError(_('A filename can not contain "/"')) # Check whether the source is in a sub-folder - probably because # the user chose a node delivered by XMLHTTPRequest # Ensure destination directory is the same directory as the source elif '/' in src_names[0]: src_dir, src_base = posixpath.split(src_names[0]) dst_name = posixpath.join(src_dir, dst_name) else: # Rename only mode is not in effect, allow src to be moved inside # dst if dst exists or there is more than one src item move_as_child = True self.log.debug('Opening repository for %s', operation) reponame, repos, path = _get_repository(self.env, req) try: src_paths = [repos.normalize_path('/'.join([path, src_name])) for src_name in src_names] dst_path = repos.normalize_path('/'.join([path, dst_name])) svn_writer = SubversionWriter(self.env, repos, req.authname) try: if operation == 'delete': self.log.info('Deleting %i items in repository %s', len(src_paths), reponame) svn_writer.delete(src_paths, commit_msg) elif operation == 'move': self.log.info('Moving %i items to %s in repository %s', len(src_paths), repr(dst_path), reponame) svn_writer.move(src_paths, dst_path, move_as_child, commit_msg) else: raise TracError("Unknown operation %s" % operation) verb = _describe_op(operation, self.rename_only, 'Deleted', 'Moved', 'Renamed') add_notice(req, "%s %i item(s) successfully" % (verb, len(src_paths))) except Exception, e: self.log.exception("Failed when attempting svn operation " "%s with rename_only=%s: %s", operation, self.rename_only, e) verb = _describe_op(operation, self.rename_only, 'Delete', 'Move', 'Rename') add_warning(req, "%s failed: %s" % (verb, e)) finally: repos.sync() self.log.debug('Closing repository') repos.close() # Perform http redirect back to this page in order to rerender # template according to new repository state req.redirect(req.href(req.path_info))
class TracBrowserEdit(Component): implements(ITemplateProvider, ITemplateStreamFilter, IRequestFilter, IPermissionRequestor) max_edit_size = IntOption( 'browserops', 'max_edit_size', 262144, '''Maximum allowed file size (in bytes) for edited files. Set to 0 for unlimited editing size.''') # ITemplateProvider methods def get_htdocs_dirs(self): '''Return directories from which to serve js, css and other static files ''' return [('trac_browser_svn_ops', resource_filename(__name__, 'htdocs'))] def get_templates_dirs(self): '''Return directories from which to fetch templates for rendering ''' return [resource_filename(__name__, 'templates')] # IPermissionRequestor methods def get_permission_actions(self): return [ 'REPOSITORY_MODIFY', ('REPOSITORY_ADMIN', ['REPOSITORY_MODIFY']), ] # ITemplateStreamFilter methods def filter_stream(self, req, method, filename, stream, data): action = req.args.get('action', '') if filename == 'browser.html' and action == 'edit': req.perm.require('REPOSITORY_MODIFY') # NB TracBrowserOps already inserts javascript and css we need # So only add css/javascript needed solely by the editor if data['file'] and data['file']['preview']['rendered']: max_edit_size = self.max_edit_size data['max_edit_size'] = max_edit_size # Discard rendered table, replace with textarea of file contents # This means reading the file from the repository again # N.B. If a file is rendered as something other than a table # e.g. due to PreCodeBrowserPlugin this code won't trigger # Retrieve the same node that BrowserModule.process_request() # used to render the preview. # At this point reponame has been removed from data['path'] # and repos has already been determined repos = data['repos'] path = data['path'] rev = data['rev'] node = repos.get_node(path, rev) # If node is too large then don't allow editing, abort if max_edit_size > 0 and node.content_length > max_edit_size: return stream # Open the node and read it node_file = node.get_content() node_data = node_file.read() # Discover the mime type and character encoding of the node # Try in order # - svn:mime-type property # - detect from file name and content (BOM) # - use configured default for Trac mime_view = Mimeview(self.env) mime_type = node.content_type \ or mime_view.get_mimetype(node.name, node_data) \ or 'text/plain' encoding = mime_view.get_charset(node_data, mime_type) # Populate template data content = mime_view.to_unicode(node_data, mime_type, encoding) data['file_content'] = content data['file_encoding'] = encoding # Replace the already rendered preview with a form and textarea bsops_stream = Chrome(self.env).render_template( req, 'file_edit.html', data, fragment=True) transf = Transformer('//div[@id="preview"]' '/table[@class="code"]') stream |= transf.replace( bsops_stream.select('//div[@id="bsop_edit"]')) return stream # IRequestFilter methods def pre_process_request(self, req, handler): if req.path_info.startswith('/browser') and req.method == 'POST' \ and "bsop_edit_commit" in req.args: req.perm.require('REPOSITORY_MODIFY') self.log.debug('Intercepting browser POST for edit') # Dispatch to private edit handler method # The private handler performs a redirect, so don't return if 'bsop_edit_text' in req.args: self._edit_request(req, handler) else: return handler def post_process_request(self, req, template, data, content_type): return (template, data, content_type) # Private methods def _edit_request(self, req, handler): self.log.debug('Handling file edit for "%s"', req.authname) # Retrieve fields # Form data arrives in req.args as unicode strings so the edited text # must be encoded before writing to disk, just as paths and commit # messages are. The difference is that subversion always takes paths # in utf-8 encoding. File encoding is not prescribed, to subversion # it's all just bytes. edit_text = req.args['bsop_edit_text'] commit_msg = req.args['bsop_edit_commit_msg'] max_edit_size = self.max_edit_size self.log.debug('Received %i characters of edited text', len(edit_text)) if max_edit_size > 0 and len(edit_text) > max_edit_size: raise TracError("The edited text is too long, " "the limit is %s (%i bytes)." % (pretty_size(max_edit_size), max_edit_size)) self.log.debug('Opening repository for file edit') reponame, repos, path = _get_repository(self.env, req) try: # Determine encoding to use # Try in order # - extract charset= from svn:mimetype property # - Encoding determined earlier and sent to client # - Default encoding (in case returned by client is invalid) node = repos.get_node(path) encoding = _encoding_from_mime_type(node.content_type) \ or req.args.get('bsop_edit_encoding', '') try: text_encoded = edit_text.encode(encoding) except (LookupError, UnicodeEncodeError), e: self.log.info( 'Failed to encode edited text with encoding ' '"%s" retrying with default. Error was: %s', encoding, e) encoding = self.config.get('trac', 'default_charset') try: text_encoded = edit_text.encode(encoding) except (LookupError, UnicodeEncodeError), e: self.log.error( 'Could not encode edited text with default ' 'encoding "%s": %s', encoding, e) raise TracError('Could not commit your text because it ' 'could not be encoded.') repos_path = repos.normalize_path(path) filename = posixpath.basename(repos_path) self.log.debug('Writing file "%s" encoded as "%s" to "%s" in "%s"', filename, encoding, repos_path, reponame) svn_writer = SubversionWriter(self.env, repos, req.authname) try: rev = svn_writer.put_content(repos_path, text_encoded, commit_msg) add_notice( req, 'Committed changes to rev %i in "%s"' % (rev, filename)) except Exception, e: self.log.exception('Failed when attempting svn write: %s', e) add_warning(req, 'Failed to write edited file', 'See the Trac log file for more information')
def _move_delete_request(self, req, handler): self.log.debug('Handling move/delete for %s', req.authname) operation = req.args.get('bsop_mvdel_op') #Moving or deleting? src_names = req.args.getlist( 'bsop_mvdel_src_name') # Items to move/del dst_name = req.args.get('bsop_mvdel_dst_name') # Destination if move commit_msg = req.args.get('bsop_mvdel_commit') # ContextMenuPlugin provides each src as [reponame/][page_path/]node_path # The Trac request path is of the form /[reponame/][page_path] # Retrieve just the portion of the src items that is relative to this # page by stripping the request path. Do this before the request path # is itself stripped of reponame when retrieving the repository path = req.args.get('path') src_names = [posixpath.join('/', src_name) for src_name in src_names] src_names = [src_name.split(path, 1)[1] for src_name in src_names] self.log.debug('Received %i source items to "%s"', len(src_names), operation) # Enforce rename_only mode if operation == 'move' and self.rename_only: # Do not allow move operation to place src in dst, if dst exists move_as_child = False # Check that move affects only one node if len(src_names) > 1: self.log.error( 'Attempted to rename %i nodes, only 1 node can' 'be renamed at a time', len(src_names)) raise TracError('Cannot rename multiple files or folders.') # Check that destination is just filename # i.e. it doesn't contain any path components elif '/' in dst_name: self.log.error( 'Rename encountered path seperator "/" in' 'destination ("%s", "%s")', src_names[0], dst_name) raise TracError(_('A filename can not contain "/"')) # Check whether the source is in a sub-folder - probably because # the user chose a node delivered by XMLHTTPRequest # Ensure destination directory is the same directory as the source elif '/' in src_names[0]: src_dir, src_base = posixpath.split(src_names[0]) dst_name = posixpath.join(src_dir, dst_name) else: # Rename only mode is not in effect, allow src to be moved inside # dst if dst exists or there is more than one src item move_as_child = True self.log.debug('Opening repository for %s', operation) reponame, repos, path = _get_repository(self.env, req) try: src_paths = [ repos.normalize_path('/'.join([path, src_name])) for src_name in src_names ] dst_path = repos.normalize_path('/'.join([path, dst_name])) svn_writer = SubversionWriter(self.env, repos, req.authname) try: if operation == 'delete': self.log.info('Deleting %i items in repository %s', len(src_paths), reponame) svn_writer.delete(src_paths, commit_msg) elif operation == 'move': self.log.info('Moving %i items to %s in repository %s', len(src_paths), repr(dst_path), reponame) svn_writer.move(src_paths, dst_path, move_as_child, commit_msg) else: raise TracError("Unknown operation %s" % operation) verb = _describe_op(operation, self.rename_only, 'Deleted', 'Moved', 'Renamed') add_notice( req, "%s %i item(s) successfully" % (verb, len(src_paths))) except Exception, e: self.log.exception( "Failed when attempting svn operation " "%s with rename_only=%s: %s", operation, self.rename_only, e) verb = _describe_op(operation, self.rename_only, 'Delete', 'Move', 'Rename') add_warning(req, "%s failed: %s" % (verb, e)) finally: repos.sync() self.log.debug('Closing repository') repos.close() # Perform http redirect back to this page in order to rerender # template according to new repository state req.redirect(req.href(req.path_info))
def _do_execute_transformation(self, transformation, transformation_id=None, store=True, return_bytes_handle=False, changecwd=False, listall=False, parameters=None): tempdir = tempfile.mkdtemp() if changecwd: os.chdir(tempdir) os.mkdir(os.path.join(tempdir, "svn")) write_simple_jndi_properties(self.env, tempdir) # execute transform transform = self._list_transformation_files(listall)[transformation] if parameters: for parameter in parameters: if parameter not in transform['parameters']: raise KeyError("%s is not valid parameter" % parameter) else: parameters = {} parameters['DefineInternal.Project.ShortName'] = os.path.split(self.env.path)[1] scriptfilename = {'transformation': 'pan.sh', 'job': 'kitchen.sh'}[transform['type']] executable = os.path.join(resource_filename(__name__, 'pentaho-data-integration'), scriptfilename) args = [ "/bin/sh", executable, "-file", transform['full_path'], "-level", "Detailed", ] for k, v in parameters.items(): if "=" in k: raise ValueError("Unable to support = symbol in parameter key named %s" % k) args.append("-param:%s=%s" % (k.encode('utf-8'), v.encode('utf-8'))) self.log.debug("Running %s with %s", executable, args) if transformation_id: # See https://d4.define.logica.com/ticket/4375#comment:7 db = self.env.get_read_db() @self.env.with_transaction() def do_insert(db): cursor = db.cursor() self.env.log.debug("Updating running_transformations - inserting new row for %s", transformation_id) cursor.execute("""INSERT INTO running_transformations (transformation_id, status, started) VALUES (%s, %s, %s)""", (transformation_id, "running", to_utimestamp(datetime.now(utc)))) # this bit of Python isn't so good :-( I'll just merge the stdout and stderr streams... # http://stackoverflow.com/questions/6809590/merging-a-python-scripts-subprocess-stdout-and-stderr-while-keeping-them-disti # http://codereview.stackexchange.com/questions/6567/how-to-redirect-a-subprocesses-output-stdout-and-stderr-to-logging-module script = subprocess.Popen(args, executable="/bin/sh", cwd=os.path.join(tempdir, "svn"), env={'PENTAHO_DI_JAVA_OPTIONS': "-Dfile.encoding=utf8 -Dnet.sf.ehcache.skipUpdateCheck=true -Djava.awt.headless=true -Dorg.osjava.sj.root=%s" % os.path.join(tempdir,"simple-jndi"), 'LANG': "en_US.UTF-8", 'KETTLE_HOME': os.path.join(tempdir,"kettle")}, stdout=subprocess.PIPE,stderr=subprocess.STDOUT) while script.poll() is None: # this can go to the database later (natively, by pdi) # keeping here, as info level for now. self.log.info("Script output: %s", script.stdout.readline()) self.log.info("Script returned %s", script.returncode) if script.returncode: # transform has failed to complete - update running_transformations table if transformation_id: @self.env.with_transaction() def do_insert(db): cursor = db.cursor() self.env.log.debug("Updating running_transformations - %s failed to complete", transformation_id) cursor.execute("""UPDATE running_transformations SET transformation_id=%s, status=%s, ended=%s WHERE transformation_id=%s""", (transformation_id, "error", to_utimestamp(datetime.now(utc)), transformation_id)) raise RuntimeError("Business Intelligence subprocess script failed") # We know assume that the transform has finished successfully # so we update the running_transformations table to represent this if transformation_id: @self.env.with_transaction() def do_insert(db): cursor = db.cursor() self.env.log.debug("Updating running_transformations - %s completed", transformation_id) cursor.execute("""UPDATE running_transformations SET transformation_id=%s, status=%s, ended=%s WHERE transformation_id=%s""", (transformation_id, "success", to_utimestamp(datetime.now(utc)), transformation_id)) if store: reponame, repos, path = RepositoryManager(self.env).get_repository_by_path('') svn_writer = SubversionWriter(self.env, repos, "reporting") revs = [] for filename_encoded in os.listdir(os.path.join(tempdir, "svn")): filename = filename_encoded.decode('utf-8') # we wrote the filename out ourselves self.log.info("Uploading %s", filename) writer = SubversionWriter(self.env, repos, "reporting") file_data = open(os.path.join(os.path.join(tempdir, "svn"), filename)).read() for path in ["define-reports", "define-reports/%s" % transformation]: try: repos.sync() repos.get_node(path) except NoSuchNode, e: self.log.warning("Creating %s for the first time", path) writer.make_dir(path, "Generated by reporting framework") repos.sync() properties = {'define:generated-by-transformation': transformation} for k, v in parameters.items(): if not k.startswith("DefineInternal"): properties[u"define:parameter:%s" % k] = v rev = writer.put_content([("define-reports/%s/%s" % (transformation, filename), file_data)], "Generated by reporting framework transformation", properties=properties, clearproperties=True) revs.append(rev)