def tarball(self, commit, path=None): """ Makes a svn export at `tmpdest` then zips that into `dest/tmpfilename` then renames that to `dest/filename` """ path = self._tarball_path_clean(path, commit) if not os.path.exists(self._repo.tarball_path): os.makedirs(self._repo.tarball_path) if not os.path.exists(self._repo.tarball_tmpdir): os.makedirs(self._repo.tarball_tmpdir) archive_name = self._repo.tarball_filename(commit, path) dest = os.path.join(self._repo.tarball_path, archive_name) tmpdest = os.path.join(self._repo.tarball_tmpdir, archive_name) filename = os.path.join(self._repo.tarball_path, '%s%s' % (archive_name, '.zip')) tmpfilename = os.path.join(self._repo.tarball_path, '%s%s' % (archive_name, '.tmp')) rmtree(dest.encode('utf8'), ignore_errors=True) # must encode into bytes or it'll fail on non-ascii filenames rmtree(tmpdest.encode('utf8'), ignore_errors=True) path = os.path.join(self._url, path) try: # need to set system locale to handle all symbols in filename import locale locale.setlocale(locale.LC_ALL, 'en_US.UTF-8') self._svn.export(path, tmpdest, revision=pysvn.Revision( pysvn.opt_revision_kind.number, commit), ignore_externals=True) zipdir(tmpdest, tmpfilename) os.rename(tmpfilename, filename) finally: rmtree(dest.encode('utf8'), ignore_errors=True) rmtree(tmpdest.encode('utf8'), ignore_errors=True) if os.path.exists(tmpfilename): os.remove(tmpfilename)
def tarball(self, commit, path=None): path = self._path_to_root(path, commit) if not os.path.exists(self._repo.tarball_path): os.makedirs(self._repo.tarball_path) archive_name = self._repo.tarball_filename(commit, path) dest = os.path.join(self._repo.tarball_path, archive_name) filename = os.path.join(self._repo.tarball_path, '%s%s' % (archive_name, '.zip')) tmpfilename = os.path.join(self._repo.tarball_path, '%s%s' % (archive_name, '.tmp')) rmtree(dest, ignore_errors=True) path = os.path.join(self._url, path) try: # need to set system locale to handle all symbols in filename import locale locale.setlocale(locale.LC_ALL, 'en_US.UTF-8') self._svn.export(path, dest, revision=pysvn.Revision( pysvn.opt_revision_kind.number, commit), ignore_externals=True) zipdir(dest, tmpfilename) os.rename(tmpfilename, filename) finally: rmtree(dest, ignore_errors=True) if os.path.exists(tmpfilename): os.remove(tmpfilename)
def tarball(self, commit, path=None): """ Makes a svn export at `tmpdest` then zips that into `dest/tmpfilename` then renames that to `dest/filename` """ path = self._tarball_path_clean(path, commit) if not os.path.exists(self._repo.tarball_path): os.makedirs(self._repo.tarball_path) if not os.path.exists(self._repo.tarball_tmpdir): os.makedirs(self._repo.tarball_tmpdir) archive_name = self._repo.tarball_filename(commit, path) dest = os.path.join(self._repo.tarball_path, archive_name) tmpdest = os.path.join(self._repo.tarball_tmpdir, archive_name) filename = os.path.join(self._repo.tarball_path, '%s%s' % (archive_name, '.zip')).encode('utf-8') tmpfilename = os.path.join(self._repo.tarball_path, '%s%s' % (archive_name, '.tmp')).encode('utf-8') rmtree(dest.encode('utf8'), ignore_errors=True) # must encode into bytes or it'll fail on non-ascii filenames rmtree(tmpdest.encode('utf8'), ignore_errors=True) path = os.path.join(self._url, path) try: # need to set system locale to handle all symbols in filename import locale locale.setlocale(locale.LC_ALL, str('en_US.UTF-8')) self._svn.export(path, tmpdest, revision=pysvn.Revision( pysvn.opt_revision_kind.number, commit), ignore_externals=True) zipdir(tmpdest, tmpfilename) os.rename(tmpfilename, filename) finally: rmtree(dest.encode('utf8'), ignore_errors=True) rmtree(tmpdest.encode('utf8'), ignore_errors=True) if os.path.exists(tmpfilename): os.remove(tmpfilename)
def tarball(self, commit, path=None): path = self._path_to_root(path, commit) if not os.path.exists(self._repo.tarball_path): os.makedirs(self._repo.tarball_path) archive_name = self._repo.tarball_filename(commit, path) dest = os.path.join(self._repo.tarball_path, archive_name) filename = os.path.join(self._repo.tarball_path, '%s%s' % (archive_name, '.zip')) tmpfilename = os.path.join(self._repo.tarball_path, '%s%s' % (archive_name, '.tmp')) rmtree(dest.encode('utf8'), ignore_errors=True) # must encode into bytes or it'll fail on non-ascii filenames path = os.path.join(self._url, path) try: # need to set system locale to handle all symbols in filename import locale locale.setlocale(locale.LC_ALL, 'en_US.UTF-8') self._svn.export(path, dest, revision=pysvn.Revision( pysvn.opt_revision_kind.number, commit), ignore_externals=True) zipdir(dest, tmpfilename) os.rename(tmpfilename, filename) finally: rmtree(dest.encode('utf8'), ignore_errors=True) if os.path.exists(tmpfilename): os.remove(tmpfilename)
def process(self, project, tools, user, filename=None, send_email=True, with_attachments=False): export_filename = filename or project.bulk_export_filename() tmp_path = os.path.join( project.bulk_export_path(rootdir=tg.config.get( 'bulk_export_tmpdir', tg.config['bulk_export_path'])), os.path.splitext(export_filename) [0], # e.g. test-backup-2018-06-26-210524 without the .zip ) export_path = project.bulk_export_path( rootdir=tg.config['bulk_export_path']) export_fullpath = os.path.join(export_path, export_filename) if not os.path.exists(tmp_path): os.makedirs(tmp_path) if not os.path.exists(export_path): os.makedirs(export_path) apps = [project.app_instance(tool) for tool in tools] exportable = self.filter_exportable(apps) results = [ self.export(tmp_path, app, with_attachments) for app in exportable ] exported = self.filter_successful(results) if exported: zipdir(tmp_path, export_fullpath) shutil.rmtree( tmp_path.encode('utf8') ) # must encode into bytes or it'll fail on non-ascii filenames if not user: log.info('No user. Skipping notification.') return if not send_email: return tmpl = g.jinja2_env.get_template( 'allura:templates/mail/bulk_export.html') instructions = tg.config.get('bulk_export_download_instructions', '') instructions = instructions.format( project=project.shortname, filename=export_filename, c=c, ) exported_names = [a.config.options.mount_point for a in exported] tmpl_context = { 'instructions': instructions, 'project': project, 'tools': exported_names, 'not_exported_tools': list(set(tools) - set(exported_names)), } mail_tasks.send_system_mail_to_user( user, u'Bulk export for project %s completed' % project.shortname, tmpl.render(tmpl_context))
def process(self, project, tools, user, filename=None, send_email=True, with_attachments=False): export_filename = filename or project.bulk_export_filename() export_path = self.get_export_path(project.bulk_export_path(), export_filename) if not os.path.exists(export_path): os.makedirs(export_path) apps = [project.app_instance(tool) for tool in tools] exportable = self.filter_exportable(apps) results = [ self.export(export_path, app, with_attachments) for app in exportable ] exported = self.filter_successful(results) if exported: zipdir(export_path, os.path.join(os.path.dirname(export_path), export_filename)) shutil.rmtree(export_path) if not user: log.info('No user. Skipping notification.') return if not send_email: return tmpl = g.jinja2_env.get_template( 'allura:templates/mail/bulk_export.html') instructions = tg.config.get('bulk_export_download_instructions', '') instructions = instructions.format( project=project.shortname, filename=export_filename, c=c, ) exported_names = [a.config.options.mount_point for a in exported] tmpl_context = { 'instructions': instructions, 'project': project, 'tools': exported_names, 'not_exported_tools': list(set(tools) - set(exported_names)), } email = { 'toaddr': unicode(user._id), 'fromaddr': unicode(tg.config['forgemail.return_path']), 'sender': unicode(tg.config['forgemail.return_path']), 'reply_to': unicode(tg.config['forgemail.return_path']), 'message_id': h.gen_message_id(), 'subject': u'Bulk export for project %s completed' % project.shortname, 'text': tmpl.render(tmpl_context) } mail_tasks.sendsimplemail.post(**email)
def zip_and_cleanup(export_path, export_filename): """ Zip exported data for a given path and filename. Copy it to proper location. Remove temporary files. """ zipdir(export_path, os.path.join(os.path.dirname(export_path), export_filename)) # cleanup shutil.rmtree(export_path)
def test_zipdir(tg, popen): tg.config = {'scm.repos.tarball.zip_binary': '/bin/zip'} src = '/fake/path/to/repo' zipfile = '/fake/zip/file.tmp' zipdir(src, zipfile) popen.assert_called_once_with(['/bin/zip', '-r', zipfile, 'repo'], cwd='/fake/path/to') popen.reset_mock() src = '/fake/path/to/repo/' zipdir(src, zipfile, exclude='file.txt') popen.assert_called_once_with(['/bin/zip', '-r', zipfile, 'repo', '-x', 'file.txt'], cwd='/fake/path/to')
def test_zipdir(tg, popen): tg.config = {'scm.repos.tarball.zip_binary': '/bin/zip'} src = '/fake/path/to/repo' zipfile = '/fake/zip/file.tmp' zipdir(src, zipfile) popen.assert_called_once_with(['/bin/zip', '-r', zipfile, 'repo'], cwd='/fake/path/to') popen.reset_mock() src = '/fake/path/to/repo/' zipdir(src, zipfile, exclude='file.txt') popen.assert_called_once_with( ['/bin/zip', '-r', zipfile, 'repo', '-x', 'file.txt'], cwd='/fake/path/to')
def test_exception_logged(self, tg, popen): tg.config = {'scm.repos.tarball.zip_binary': '/bin/zip'} popen.return_value.communicate.return_value = 1, 2 popen.return_value.returncode = 1 src = '/fake/path/to/repo' zipfile = '/fake/zip/file.tmp' with self.assertRaises(Exception) as cm: zipdir(src, zipfile) emsg = str(cm.exception) self.assertTrue( "Command: " "['/bin/zip', '-y', '-q', '-r', '/fake/zip/file.tmp', 'repo'] " "returned non-zero exit code 1" in emsg) self.assertTrue("STDOUT: 1" in emsg) self.assertTrue("STDERR: 2" in emsg)
def process(self, project, tools, user, filename=None, send_email=True, with_attachments=False): export_filename = filename or project.bulk_export_filename() export_path = self.get_export_path( project.bulk_export_path(), export_filename) if not os.path.exists(export_path): os.makedirs(export_path) apps = [project.app_instance(tool) for tool in tools] exportable = self.filter_exportable(apps) results = [self.export(export_path, app, with_attachments) for app in exportable] exported = self.filter_successful(results) if exported: zipdir(export_path, os.path.join(os.path.dirname(export_path), export_filename)) shutil.rmtree(export_path) if not user: log.info('No user. Skipping notification.') return if not send_email: return tmpl = g.jinja2_env.get_template( 'allura:templates/mail/bulk_export.html') instructions = tg.config.get('bulk_export_download_instructions', '') instructions = instructions.format( project=project.shortname, filename=export_filename, c=c, ) exported_names = [a.config.options.mount_point for a in exported] tmpl_context = { 'instructions': instructions, 'project': project, 'tools': exported_names, 'not_exported_tools': list(set(tools) - set(exported_names)), } email = { 'toaddr': unicode(user._id), 'fromaddr': unicode(tg.config['forgemail.return_path']), 'sender': unicode(tg.config['forgemail.return_path']), 'reply_to': unicode(tg.config['forgemail.return_path']), 'message_id': h.gen_message_id(), 'subject': u'Bulk export for project %s completed' % project.shortname, 'text': tmpl.render(tmpl_context) } mail_tasks.sendsimplemail.post(**email)
def test_popen_called(self, tg, popen): from subprocess import PIPE popen.return_value.communicate.return_value = 1, 2 popen.return_value.returncode = 0 tg.config = {'scm.repos.tarball.zip_binary': '/bin/zip'} src = '/fake/path/to/repo' zipfile = '/fake/zip/file.tmp' zipdir(src, zipfile) popen.assert_called_once_with(['/bin/zip', '-y', '-q', '-r', zipfile, 'repo'], cwd='/fake/path/to', stdout=PIPE, stderr=PIPE) popen.reset_mock() src = '/fake/path/to/repo/' zipdir(src, zipfile, exclude='file.txt') popen.assert_called_once_with( ['/bin/zip', '-y', '-q', '-r', zipfile, 'repo', '-x', 'file.txt'], cwd='/fake/path/to', stdout=PIPE, stderr=PIPE)
def test_popen_called(self, tg, popen): from subprocess import PIPE popen.return_value.communicate.return_value = 1, 2 popen.return_value.returncode = 0 tg.config = {'scm.repos.tarball.zip_binary': '/bin/zip'} src = '/fake/path/to/repo' zipfile = '/fake/zip/file.tmp' zipdir(src, zipfile) popen.assert_called_once_with(['/bin/zip', '-r', zipfile, 'repo'], cwd='/fake/path/to', stdout=PIPE, stderr=PIPE) popen.reset_mock() src = '/fake/path/to/repo/' zipdir(src, zipfile, exclude='file.txt') popen.assert_called_once_with( ['/bin/zip', '-r', zipfile, 'repo', '-x', 'file.txt'], cwd='/fake/path/to', stdout=PIPE, stderr=PIPE)
def process(self, project, tools, user, filename=None, send_email=True, with_attachments=False): export_filename = filename or project.bulk_export_filename() tmp_path = os.path.join( project.bulk_export_path(rootdir=tg.config.get('bulk_export_tmpdir', tg.config['bulk_export_path'])), os.path.splitext(export_filename)[0], # e.g. test-backup-2018-06-26-210524 without the .zip ) export_path = project.bulk_export_path(rootdir=tg.config['bulk_export_path']) export_fullpath = os.path.join(export_path, export_filename) if not os.path.exists(tmp_path): os.makedirs(tmp_path) if not os.path.exists(export_path): os.makedirs(export_path) apps = [project.app_instance(tool) for tool in tools] exportable = self.filter_exportable(apps) results = [self.export(tmp_path, app, with_attachments) for app in exportable] exported = self.filter_successful(results) if exported: zipdir(tmp_path, export_fullpath) shutil.rmtree(tmp_path.encode('utf8')) # must encode into bytes or it'll fail on non-ascii filenames if not user: log.info('No user. Skipping notification.') return if not send_email: return tmpl = g.jinja2_env.get_template('allura:templates/mail/bulk_export.html') instructions = tg.config.get('bulk_export_download_instructions', '') instructions = instructions.format( project=project.shortname, filename=export_filename, c=c, ) exported_names = [a.config.options.mount_point for a in exported] tmpl_context = { 'instructions': instructions, 'project': project, 'tools': exported_names, 'not_exported_tools': list(set(tools) - set(exported_names)), } mail_tasks.send_system_mail_to_user(user, u'Bulk export for project %s completed' % project.shortname, tmpl.render(tmpl_context))
def tarball(self, commit, path=None): path = self._path_to_root(path, commit) if not os.path.exists(self._repo.tarball_path): os.makedirs(self._repo.tarball_path) archive_name = self._repo.tarball_filename(commit, path) dest = os.path.join(self._repo.tarball_path, archive_name) filename = os.path.join(self._repo.tarball_path, '%s%s' % (archive_name, '.zip')) tmpfilename = os.path.join(self._repo.tarball_path, '%s%s' % (archive_name, '.tmp')) rmtree(dest, ignore_errors=True) path = os.path.join(self._url, path) try: self._svn.export(path, dest, revision=pysvn.Revision(pysvn.opt_revision_kind.number, commit)) zipdir(dest, tmpfilename) os.rename(tmpfilename, filename) finally: rmtree(dest, ignore_errors=True) if os.path.exists(tmpfilename): os.remove(tmpfilename)
def tarball(self, commit, path=None): path = self._path_to_root(path, commit) if not os.path.exists(self._repo.tarball_path): os.makedirs(self._repo.tarball_path) archive_name = self._repo.tarball_filename(commit, path) dest = os.path.join(self._repo.tarball_path, archive_name) filename = os.path.join(self._repo.tarball_path, "%s%s" % (archive_name, ".zip")) tmpfilename = os.path.join(self._repo.tarball_path, "%s%s" % (archive_name, ".tmp")) rmtree(dest, ignore_errors=True) path = os.path.join(self._url, path) try: self._svn.export( path, dest, revision=pysvn.Revision(pysvn.opt_revision_kind.number, commit), ignore_externals=True ) zipdir(dest, tmpfilename) os.rename(tmpfilename, filename) finally: rmtree(dest, ignore_errors=True) if os.path.exists(tmpfilename): os.remove(tmpfilename)