def tarball(self, commit, path=None):
"""
Makes a svn export at `tmpdest`
then zips that into `dest/tmpfilename`
then renames that to `dest/filename`
"""
path = self._tarball_path_clean(path, commit)
if not os.path.exists(self._repo.tarball_path):
os.makedirs(self._repo.tarball_path)
if not os.path.exists(self._repo.tarball_tmpdir):
os.makedirs(self._repo.tarball_tmpdir)
archive_name = self._repo.tarball_filename(commit, path)
dest = os.path.join(self._repo.tarball_path, archive_name)
tmpdest = os.path.join(self._repo.tarball_tmpdir, archive_name)
filename = os.path.join(self._repo.tarball_path, '%s%s' % (archive_name, '.zip'))
tmpfilename = os.path.join(self._repo.tarball_path, '%s%s' % (archive_name, '.tmp'))
rmtree(dest.encode('utf8'), ignore_errors=True) # must encode into bytes or it'll fail on non-ascii filenames
rmtree(tmpdest.encode('utf8'), ignore_errors=True)
path = os.path.join(self._url, path)
try:
# need to set system locale to handle all symbols in filename
import locale
locale.setlocale(locale.LC_ALL, 'en_US.UTF-8')
self._svn.export(path,
tmpdest,
revision=pysvn.Revision(
pysvn.opt_revision_kind.number, commit),
ignore_externals=True)
zipdir(tmpdest, tmpfilename)
os.rename(tmpfilename, filename)
finally:
rmtree(dest.encode('utf8'), ignore_errors=True)
rmtree(tmpdest.encode('utf8'), ignore_errors=True)
if os.path.exists(tmpfilename):
os.remove(tmpfilename)
def tarball(self, commit, path=None):
path = self._path_to_root(path, commit)
if not os.path.exists(self._repo.tarball_path):
os.makedirs(self._repo.tarball_path)
archive_name = self._repo.tarball_filename(commit, path)
dest = os.path.join(self._repo.tarball_path, archive_name)
filename = os.path.join(self._repo.tarball_path, '%s%s' %
(archive_name, '.zip'))
tmpfilename = os.path.join(self._repo.tarball_path, '%s%s' %
(archive_name, '.tmp'))
rmtree(dest, ignore_errors=True)
path = os.path.join(self._url, path)
try:
# need to set system locale to handle all symbols in filename
import locale
locale.setlocale(locale.LC_ALL, 'en_US.UTF-8')
self._svn.export(path,
dest,
revision=pysvn.Revision(
pysvn.opt_revision_kind.number, commit),
ignore_externals=True)
zipdir(dest, tmpfilename)
os.rename(tmpfilename, filename)
finally:
rmtree(dest, ignore_errors=True)
if os.path.exists(tmpfilename):
os.remove(tmpfilename)
def tarball(self, commit, path=None):
"""
Makes a svn export at `tmpdest`
then zips that into `dest/tmpfilename`
then renames that to `dest/filename`
"""
path = self._tarball_path_clean(path, commit)
if not os.path.exists(self._repo.tarball_path):
os.makedirs(self._repo.tarball_path)
if not os.path.exists(self._repo.tarball_tmpdir):
os.makedirs(self._repo.tarball_tmpdir)
archive_name = self._repo.tarball_filename(commit, path)
dest = os.path.join(self._repo.tarball_path, archive_name)
tmpdest = os.path.join(self._repo.tarball_tmpdir, archive_name)
filename = os.path.join(self._repo.tarball_path, '%s%s' % (archive_name, '.zip')).encode('utf-8')
tmpfilename = os.path.join(self._repo.tarball_path, '%s%s' % (archive_name, '.tmp')).encode('utf-8')
rmtree(dest.encode('utf8'), ignore_errors=True) # must encode into bytes or it'll fail on non-ascii filenames
rmtree(tmpdest.encode('utf8'), ignore_errors=True)
path = os.path.join(self._url, path)
try:
# need to set system locale to handle all symbols in filename
import locale
locale.setlocale(locale.LC_ALL, str('en_US.UTF-8'))
self._svn.export(path,
tmpdest,
revision=pysvn.Revision(
pysvn.opt_revision_kind.number, commit),
ignore_externals=True)
zipdir(tmpdest, tmpfilename)
os.rename(tmpfilename, filename)
finally:
rmtree(dest.encode('utf8'), ignore_errors=True)
rmtree(tmpdest.encode('utf8'), ignore_errors=True)
if os.path.exists(tmpfilename):
os.remove(tmpfilename)
def tarball(self, commit, path=None):
path = self._path_to_root(path, commit)
if not os.path.exists(self._repo.tarball_path):
os.makedirs(self._repo.tarball_path)
archive_name = self._repo.tarball_filename(commit, path)
dest = os.path.join(self._repo.tarball_path, archive_name)
filename = os.path.join(self._repo.tarball_path, '%s%s' %
(archive_name, '.zip'))
tmpfilename = os.path.join(self._repo.tarball_path, '%s%s' %
(archive_name, '.tmp'))
rmtree(dest.encode('utf8'), ignore_errors=True) # must encode into bytes or it'll fail on non-ascii filenames
path = os.path.join(self._url, path)
try:
# need to set system locale to handle all symbols in filename
import locale
locale.setlocale(locale.LC_ALL, 'en_US.UTF-8')
self._svn.export(path,
dest,
revision=pysvn.Revision(
pysvn.opt_revision_kind.number, commit),
ignore_externals=True)
zipdir(dest, tmpfilename)
os.rename(tmpfilename, filename)
finally:
rmtree(dest.encode('utf8'), ignore_errors=True)
if os.path.exists(tmpfilename):
os.remove(tmpfilename)
def process(self,
project,
tools,
user,
filename=None,
send_email=True,
with_attachments=False):
export_filename = filename or project.bulk_export_filename()
tmp_path = os.path.join(
project.bulk_export_path(rootdir=tg.config.get(
'bulk_export_tmpdir', tg.config['bulk_export_path'])),
os.path.splitext(export_filename)
[0], # e.g. test-backup-2018-06-26-210524 without the .zip
)
export_path = project.bulk_export_path(
rootdir=tg.config['bulk_export_path'])
export_fullpath = os.path.join(export_path, export_filename)
if not os.path.exists(tmp_path):
os.makedirs(tmp_path)
if not os.path.exists(export_path):
os.makedirs(export_path)
apps = [project.app_instance(tool) for tool in tools]
exportable = self.filter_exportable(apps)
results = [
self.export(tmp_path, app, with_attachments) for app in exportable
]
exported = self.filter_successful(results)
if exported:
zipdir(tmp_path, export_fullpath)
shutil.rmtree(
tmp_path.encode('utf8')
) # must encode into bytes or it'll fail on non-ascii filenames
if not user:
log.info('No user. Skipping notification.')
return
if not send_email:
return
tmpl = g.jinja2_env.get_template(
'allura:templates/mail/bulk_export.html')
instructions = tg.config.get('bulk_export_download_instructions', '')
instructions = instructions.format(
project=project.shortname,
filename=export_filename,
c=c,
)
exported_names = [a.config.options.mount_point for a in exported]
tmpl_context = {
'instructions': instructions,
'project': project,
'tools': exported_names,
'not_exported_tools': list(set(tools) - set(exported_names)),
}
mail_tasks.send_system_mail_to_user(
user, u'Bulk export for project %s completed' % project.shortname,
tmpl.render(tmpl_context))
def process(self,
project,
tools,
user,
filename=None,
send_email=True,
with_attachments=False):
export_filename = filename or project.bulk_export_filename()
export_path = self.get_export_path(project.bulk_export_path(),
export_filename)
if not os.path.exists(export_path):
os.makedirs(export_path)
apps = [project.app_instance(tool) for tool in tools]
exportable = self.filter_exportable(apps)
results = [
self.export(export_path, app, with_attachments)
for app in exportable
]
exported = self.filter_successful(results)
if exported:
zipdir(export_path,
os.path.join(os.path.dirname(export_path), export_filename))
shutil.rmtree(export_path)
if not user:
log.info('No user. Skipping notification.')
return
if not send_email:
return
tmpl = g.jinja2_env.get_template(
'allura:templates/mail/bulk_export.html')
instructions = tg.config.get('bulk_export_download_instructions', '')
instructions = instructions.format(
project=project.shortname,
filename=export_filename,
c=c,
)
exported_names = [a.config.options.mount_point for a in exported]
tmpl_context = {
'instructions': instructions,
'project': project,
'tools': exported_names,
'not_exported_tools': list(set(tools) - set(exported_names)),
}
email = {
'toaddr': unicode(user._id),
'fromaddr': unicode(tg.config['forgemail.return_path']),
'sender': unicode(tg.config['forgemail.return_path']),
'reply_to': unicode(tg.config['forgemail.return_path']),
'message_id': h.gen_message_id(),
'subject':
u'Bulk export for project %s completed' % project.shortname,
'text': tmpl.render(tmpl_context)
}
mail_tasks.sendsimplemail.post(**email)
def zip_and_cleanup(export_path, export_filename):
"""
Zip exported data for a given path and filename.
Copy it to proper location. Remove temporary files.
"""
zipdir(export_path, os.path.join(os.path.dirname(export_path), export_filename))
# cleanup
shutil.rmtree(export_path)
def test_zipdir(tg, popen):
tg.config = {'scm.repos.tarball.zip_binary': '/bin/zip'}
src = '/fake/path/to/repo'
zipfile = '/fake/zip/file.tmp'
zipdir(src, zipfile)
popen.assert_called_once_with(['/bin/zip', '-r', zipfile, 'repo'], cwd='/fake/path/to')
popen.reset_mock()
src = '/fake/path/to/repo/'
zipdir(src, zipfile, exclude='file.txt')
popen.assert_called_once_with(['/bin/zip', '-r', zipfile, 'repo', '-x', 'file.txt'], cwd='/fake/path/to')
def test_zipdir(tg, popen):
tg.config = {'scm.repos.tarball.zip_binary': '/bin/zip'}
src = '/fake/path/to/repo'
zipfile = '/fake/zip/file.tmp'
zipdir(src, zipfile)
popen.assert_called_once_with(['/bin/zip', '-r', zipfile, 'repo'],
cwd='/fake/path/to')
popen.reset_mock()
src = '/fake/path/to/repo/'
zipdir(src, zipfile, exclude='file.txt')
popen.assert_called_once_with(
['/bin/zip', '-r', zipfile, 'repo', '-x', 'file.txt'],
cwd='/fake/path/to')
def test_exception_logged(self, tg, popen):
tg.config = {'scm.repos.tarball.zip_binary': '/bin/zip'}
popen.return_value.communicate.return_value = 1, 2
popen.return_value.returncode = 1
src = '/fake/path/to/repo'
zipfile = '/fake/zip/file.tmp'
with self.assertRaises(Exception) as cm:
zipdir(src, zipfile)
emsg = str(cm.exception)
self.assertTrue(
"Command: "
"['/bin/zip', '-y', '-q', '-r', '/fake/zip/file.tmp', 'repo'] "
"returned non-zero exit code 1" in emsg)
self.assertTrue("STDOUT: 1" in emsg)
self.assertTrue("STDERR: 2" in emsg)
def test_exception_logged(self, tg, popen):
tg.config = {'scm.repos.tarball.zip_binary': '/bin/zip'}
popen.return_value.communicate.return_value = 1, 2
popen.return_value.returncode = 1
src = '/fake/path/to/repo'
zipfile = '/fake/zip/file.tmp'
with self.assertRaises(Exception) as cm:
zipdir(src, zipfile)
emsg = str(cm.exception)
self.assertTrue(
"Command: "
"['/bin/zip', '-y', '-q', '-r', '/fake/zip/file.tmp', 'repo'] "
"returned non-zero exit code 1" in emsg)
self.assertTrue("STDOUT: 1" in emsg)
self.assertTrue("STDERR: 2" in emsg)
def process(self, project, tools, user, filename=None, send_email=True, with_attachments=False):
export_filename = filename or project.bulk_export_filename()
export_path = self.get_export_path(
project.bulk_export_path(), export_filename)
if not os.path.exists(export_path):
os.makedirs(export_path)
apps = [project.app_instance(tool) for tool in tools]
exportable = self.filter_exportable(apps)
results = [self.export(export_path, app, with_attachments) for app in exportable]
exported = self.filter_successful(results)
if exported:
zipdir(export_path,
os.path.join(os.path.dirname(export_path), export_filename))
shutil.rmtree(export_path)
if not user:
log.info('No user. Skipping notification.')
return
if not send_email:
return
tmpl = g.jinja2_env.get_template(
'allura:templates/mail/bulk_export.html')
instructions = tg.config.get('bulk_export_download_instructions', '')
instructions = instructions.format(
project=project.shortname,
filename=export_filename,
c=c,
)
exported_names = [a.config.options.mount_point for a in exported]
tmpl_context = {
'instructions': instructions,
'project': project,
'tools': exported_names,
'not_exported_tools': list(set(tools) - set(exported_names)),
}
email = {
'toaddr': unicode(user._id),
'fromaddr': unicode(tg.config['forgemail.return_path']),
'sender': unicode(tg.config['forgemail.return_path']),
'reply_to': unicode(tg.config['forgemail.return_path']),
'message_id': h.gen_message_id(),
'subject': u'Bulk export for project %s completed' % project.shortname,
'text': tmpl.render(tmpl_context)
}
mail_tasks.sendsimplemail.post(**email)
def test_popen_called(self, tg, popen):
from subprocess import PIPE
popen.return_value.communicate.return_value = 1, 2
popen.return_value.returncode = 0
tg.config = {'scm.repos.tarball.zip_binary': '/bin/zip'}
src = '/fake/path/to/repo'
zipfile = '/fake/zip/file.tmp'
zipdir(src, zipfile)
popen.assert_called_once_with(['/bin/zip', '-y', '-q', '-r', zipfile, 'repo'],
cwd='/fake/path/to', stdout=PIPE, stderr=PIPE)
popen.reset_mock()
src = '/fake/path/to/repo/'
zipdir(src, zipfile, exclude='file.txt')
popen.assert_called_once_with(
['/bin/zip', '-y', '-q', '-r', zipfile, 'repo', '-x', 'file.txt'],
cwd='/fake/path/to', stdout=PIPE, stderr=PIPE)
def test_popen_called(self, tg, popen):
from subprocess import PIPE
popen.return_value.communicate.return_value = 1, 2
popen.return_value.returncode = 0
tg.config = {'scm.repos.tarball.zip_binary': '/bin/zip'}
src = '/fake/path/to/repo'
zipfile = '/fake/zip/file.tmp'
zipdir(src, zipfile)
popen.assert_called_once_with(['/bin/zip', '-r', zipfile, 'repo'],
cwd='/fake/path/to', stdout=PIPE, stderr=PIPE)
popen.reset_mock()
src = '/fake/path/to/repo/'
zipdir(src, zipfile, exclude='file.txt')
popen.assert_called_once_with(
['/bin/zip', '-r', zipfile, 'repo', '-x', 'file.txt'],
cwd='/fake/path/to', stdout=PIPE, stderr=PIPE)
def process(self, project, tools, user, filename=None, send_email=True, with_attachments=False):
export_filename = filename or project.bulk_export_filename()
tmp_path = os.path.join(
project.bulk_export_path(rootdir=tg.config.get('bulk_export_tmpdir', tg.config['bulk_export_path'])),
os.path.splitext(export_filename)[0], # e.g. test-backup-2018-06-26-210524 without the .zip
)
export_path = project.bulk_export_path(rootdir=tg.config['bulk_export_path'])
export_fullpath = os.path.join(export_path, export_filename)
if not os.path.exists(tmp_path):
os.makedirs(tmp_path)
if not os.path.exists(export_path):
os.makedirs(export_path)
apps = [project.app_instance(tool) for tool in tools]
exportable = self.filter_exportable(apps)
results = [self.export(tmp_path, app, with_attachments) for app in exportable]
exported = self.filter_successful(results)
if exported:
zipdir(tmp_path, export_fullpath)
shutil.rmtree(tmp_path.encode('utf8')) # must encode into bytes or it'll fail on non-ascii filenames
if not user:
log.info('No user. Skipping notification.')
return
if not send_email:
return
tmpl = g.jinja2_env.get_template('allura:templates/mail/bulk_export.html')
instructions = tg.config.get('bulk_export_download_instructions', '')
instructions = instructions.format(
project=project.shortname,
filename=export_filename,
c=c,
)
exported_names = [a.config.options.mount_point for a in exported]
tmpl_context = {
'instructions': instructions,
'project': project,
'tools': exported_names,
'not_exported_tools': list(set(tools) - set(exported_names)),
}
mail_tasks.send_system_mail_to_user(user,
u'Bulk export for project %s completed' % project.shortname,
tmpl.render(tmpl_context))
def tarball(self, commit, path=None):
path = self._path_to_root(path, commit)
if not os.path.exists(self._repo.tarball_path):
os.makedirs(self._repo.tarball_path)
archive_name = self._repo.tarball_filename(commit, path)
dest = os.path.join(self._repo.tarball_path, archive_name)
filename = os.path.join(self._repo.tarball_path, '%s%s' % (archive_name, '.zip'))
tmpfilename = os.path.join(self._repo.tarball_path, '%s%s' % (archive_name, '.tmp'))
rmtree(dest, ignore_errors=True)
path = os.path.join(self._url, path)
try:
self._svn.export(path,
dest,
revision=pysvn.Revision(pysvn.opt_revision_kind.number, commit))
zipdir(dest, tmpfilename)
os.rename(tmpfilename, filename)
finally:
rmtree(dest, ignore_errors=True)
if os.path.exists(tmpfilename):
os.remove(tmpfilename)
def tarball(self, commit, path=None):
path = self._path_to_root(path, commit)
if not os.path.exists(self._repo.tarball_path):
os.makedirs(self._repo.tarball_path)
archive_name = self._repo.tarball_filename(commit, path)
dest = os.path.join(self._repo.tarball_path, archive_name)
filename = os.path.join(self._repo.tarball_path, "%s%s" % (archive_name, ".zip"))
tmpfilename = os.path.join(self._repo.tarball_path, "%s%s" % (archive_name, ".tmp"))
rmtree(dest, ignore_errors=True)
path = os.path.join(self._url, path)
try:
self._svn.export(
path, dest, revision=pysvn.Revision(pysvn.opt_revision_kind.number, commit), ignore_externals=True
)
zipdir(dest, tmpfilename)
os.rename(tmpfilename, filename)
finally:
rmtree(dest, ignore_errors=True)
if os.path.exists(tmpfilename):
os.remove(tmpfilename)
