def create_new_file(cls, filename, code, base):
"""Handles creation of new translation file."""
with open(base, 'r') as handle:
data = handle.read()
# Assume input is UTF-8 if not specified
if b'Content-Type: text/plain; charset=CHARSET' in data:
data = data.replace(
b'Content-Type: text/plain; charset=CHARSET',
b'Content-Type: text/plain; charset=UTF-8'
)
process = subprocess.Popen(
[
'msginit',
'--input', '-',
'--output', filename,
'--no-translator',
'--locale', code
],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=get_clean_env(),
)
output, output_err = process.communicate(input=data)
retcode = process.poll()
if retcode:
raise ValueError(output_err if output_err else output)
def generate_ssh_key(request):
"""
Generates SSH key.
"""
# Create directory if it does not exist
key_dir = os.path.dirname(RSA_KEY_FILE)
# Try generating key
try:
if not os.path.exists(key_dir):
os.makedirs(key_dir)
subprocess.check_output(
[
'ssh-keygen', '-q',
'-N', '',
'-C', 'Weblate',
'-t', 'rsa',
'-f', RSA_KEY_FILE[:-4]
],
stderr=subprocess.STDOUT,
env=get_clean_env(),
)
messages.success(request, _('Created new SSH key.'))
except (subprocess.CalledProcessError, OSError) as exc:
messages.error(
request,
_('Failed to generate key: %s') %
getattr(exc, 'output', str(exc))
)
def generate_ssh_key(request):
"""
Generates SSH key.
"""
try:
# Create directory if it does not exist
ensure_ssh_dir()
# Actually generate the key
subprocess.check_output(
[
'ssh-keygen', '-q',
'-N', '',
'-C', 'Weblate',
'-t', 'rsa',
'-f', ssh_file(RSA_KEY)
],
stderr=subprocess.STDOUT,
env=get_clean_env(),
)
messages.success(request, _('Created new SSH key.'))
except (subprocess.CalledProcessError, OSError) as exc:
messages.error(
request,
_('Failed to generate key: %s') %
getattr(exc, 'output', str(exc))
)
def run_hook(component, script, *args):
"""
Generic script hook executor.
"""
if script:
command = [script]
if args:
command.extend(args)
environment = get_clean_env()
if component.is_repo_link:
target = component.linked_subproject
else:
target = component
environment['WL_VCS'] = target.vcs
environment['WL_REPO'] = target.repo
environment['WL_PATH'] = target.get_path()
environment['WL_FILEMASK'] = component.filemask
environment['WL_FILE_FORMAT'] = component.file_format
try:
subprocess.check_call(
command,
env=environment,
cwd=component.get_path(),
)
return True
except (OSError, subprocess.CalledProcessError) as err:
component.log_error(
'failed to run hook script %s: %s',
script,
err
)
return False
def add_host_key(request):
"""
Adds host key for a host.
"""
ensure_ssh_dir()
host = request.POST.get('host', '')
port = request.POST.get('port', '')
if len(host) == 0:
messages.error(request, _('Invalid host name given!'))
else:
cmdline = ['ssh-keyscan']
if port:
cmdline.extend(['-p', port])
cmdline.append(host)
try:
output = subprocess.check_output(
cmdline,
stderr=subprocess.STDOUT,
env=get_clean_env(),
)
keys = []
for key in output.splitlines():
key = key.strip()
if not is_key_line(key):
continue
keys.append(key)
host, keytype, fingerprint = parse_hosts_line(key)
messages.warning(
request,
_(
'Added host key for %(host)s with fingerprint '
'%(fingerprint)s (%(keytype)s), '
'please verify that it is correct.'
) % {
'host': host,
'fingerprint': fingerprint,
'keytype': keytype,
}
)
if len(keys) == 0:
messages.error(
request,
_('Failed to fetch public key for a host!')
)
with open(ssh_file(KNOWN_HOSTS), 'a') as handle:
for key in keys:
handle.write('%s\n' % key)
except subprocess.CalledProcessError as exc:
messages.error(
request,
_('Failed to get host key: %s') % exc.output
)
except OSError as exc:
messages.error(
request,
_('Failed to get host key: %s') % str(exc)
)
def _getenv():
"""Generates environment for process execution."""
env = {"GIT_SSH": ssh_file(SSH_WRAPPER)}
# Add path to config if it exists
userconfig = os.path.expanduser("~/.config/hub")
if os.path.exists(userconfig):
env["HUB_CONFIG"] = userconfig
return get_clean_env(env)
def _getenv():
"""Generate environment for process execution."""
env = {'GIT_SSH': get_wrapper_filename()}
# Add path to config if it exists
userconfig = os.path.expanduser('~/.config/hub')
if os.path.exists(userconfig):
env['HUB_CONFIG'] = userconfig
return get_clean_env(env)
def add_language(filename, code, base):
"""
Adds new language file.
"""
subprocess.check_call(
["msginit", "-i", base, "-o", filename, "--no-translator", "-l", code],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
env=get_clean_env(),
)
def can_generate_key():
"""
Checks whether we can generate key.
"""
try:
ret = subprocess.check_call(
['which', 'ssh-keygen'],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
env=get_clean_env(),
)
if ret == 0 and not os.path.exists(RSA_KEY_FILE):
return is_home_writable()
return False
except subprocess.CalledProcessError:
return False
def add_language(filename, code, base):
'''
Adds new language file.
'''
subprocess.check_call(
[
'msginit',
'-i', base,
'-o', filename,
'--no-translator',
'-l', code
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
env=get_clean_env(),
)
def supports_new_language(cls):
'''
Checks whether we can create new language file.
'''
if cls.msginit_found is None:
try:
ret = subprocess.check_call(
['msginit', '--help'],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
env=get_clean_env(),
)
cls.msginit_found = (ret == 0)
except (subprocess.CalledProcessError, OSError):
cls.msginit_found = False
return cls.msginit_found
def ssh(request):
"""
Show information and manipulate with SSH key.
"""
# Check whether we can generate SSH key
try:
ret = subprocess.check_call(
['which', 'ssh-keygen'],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
env=get_clean_env(),
)
can_generate = (ret == 0 and not os.path.exists(RSA_KEY_FILE))
except subprocess.CalledProcessError:
can_generate = False
if not os.access(os.path.expanduser('~'), os.W_OK):
can_generate = False
messages.error(
request,
_('Can not write to home directory, please check documentation.')
)
# Grab action type
action = request.POST.get('action', None)
# Generate key if it does not exist yet
if can_generate and action == 'generate':
generate_ssh_key(request)
# Read key data if it exists
key = get_key_data()
# Add host key
if action == 'add-host':
add_host_key(request)
return render(
request,
"admin/ssh.html",
{
'public_key': key,
'can_generate': can_generate,
'host_keys': get_host_keys(),
'ssh_docs': weblate.get_doc_url('admin/projects', 'private'),
}
)
def __git_commit(self, author, timestamp, sync=False):
'''
Commits translation to git.
'''
# Format commit message
msg = self.get_commit_message()
# Pre commit hook
if self.subproject.pre_commit_script != '':
try:
subprocess.check_call(
[
self.subproject.pre_commit_script,
self.get_filename()
],
env=get_clean_env(),
)
except (OSError, subprocess.CalledProcessError) as err:
weblate.logger.error(
'Failed to run pre commit script %s: %s',
self.subproject.pre_commit_script,
err
)
# Create list of files to commit
files = [self.filename]
if self.subproject.extra_commit_file != '':
extra_file = self.subproject.extra_commit_file % {
'language': self.language_code,
}
full_path_extra = os.path.join(
self.subproject.get_path(),
extra_file
)
if os.path.exists(full_path_extra):
files.append(extra_file)
# Do actual commit
self.repository.commit(
msg, author, timestamp, files
)
# Optionally store updated hash
if sync:
self.store_hash()
def add_host_key(request):
"""
Adds host key for a host.
"""
host = request.POST.get('host', '')
port = request.POST.get('port', '')
if len(host) == 0:
messages.error(request, _('Invalid host name given!'))
else:
cmdline = ['ssh-keyscan']
if port:
cmdline.extend(['-p', port])
cmdline.append(host)
try:
output = subprocess.check_output(
cmdline,
stderr=subprocess.STDOUT,
env=get_clean_env(),
)
keys = [
line
for line in output.splitlines()
if ' ssh-rsa ' in line or ' ecdsa-sha2-nistp256 ' in line
]
for key in keys:
host, keytype, fingerprint = parse_hosts_line(key)
messages.warning(
request,
_(
'Added host key for %(host)s with fingerprint '
'%(fingerprint)s (%(keytype)s), '
'please verify that it is correct.'
) % {
'host': host,
'fingerprint': fingerprint,
'keytype': keytype,
}
)
with open(KNOWN_HOSTS_FILE, 'a') as handle:
for key in keys:
handle.write('%s\n' % key)
except (subprocess.CalledProcessError, OSError) as exc:
messages.error(
request,
_('Failed to get host key: %s') % exc.output
)
def run_hook(self, script, *args):
"""
Generic script hook executor.
"""
if script:
command = [script]
if args:
command.extend(args)
try:
subprocess.check_call(
command,
env=get_clean_env(),
)
except (OSError, subprocess.CalledProcessError) as err:
self.log_error(
'failed to run hook script %s: %s',
script,
err
)
def _popen(cls, args, cwd=None):
"""
Executes the command using popen.
"""
if args is None:
raise RepositoryException(0, "Not supported functionality", "")
args = [cls._cmd] + args
process = subprocess.Popen(
args,
cwd=cwd,
env=get_clean_env({"GIT_SSH": ssh_file(SSH_WRAPPER)}),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
output, output_err = process.communicate()
retcode = process.poll()
if retcode:
raise RepositoryException(retcode, output_err, output)
return output
def _popen(cls, args, cwd=None):
if args is None:
raise RepositoryException('Not supported functionality')
args = [cls._cmd] + args
process = subprocess.Popen(
args,
cwd=cwd,
env=get_clean_env(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
output, output_err = process.communicate()
retcode = process.poll()
if retcode:
message = output_err.strip()
if not message:
message = output.strip()
raise RepositoryException(message)
return output
def can_generate_key():
"""
Checks whether we can generate key.
"""
try:
ensure_ssh_dir()
except OSError:
return False
try:
ret = subprocess.check_call(
['which', 'ssh-keygen'],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
env=get_clean_env(),
)
return ret == 0
except subprocess.CalledProcessError:
return False
def _popen(cls, args, cwd=None):
'''
Executes the command using popen.
'''
if args is None:
raise RepositoryException(0, 'Not supported functionality', '')
args = [cls._cmd] + args
process = subprocess.Popen(
args,
cwd=cwd,
env=get_clean_env(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
output, output_err = process.communicate()
retcode = process.poll()
if retcode:
raise RepositoryException(retcode, output_err, output)
return output
def generate_gpg_key():
try:
subprocess.check_output(
[
'gpg',
'--batch',
'--pinentry-mode', 'loopback',
'--passphrase', '',
'--quick-generate-key',
settings.WEBLATE_GPG_IDENTITY,
settings.WEBLATE_GPG_ALGO,
'default', 'never',
],
stderr=subprocess.STDOUT,
env=get_clean_env(),
)
return get_gpg_key()
except (subprocess.CalledProcessError, OSError) as exc:
add_configuration_error('GPG key generating', force_text(exc))
return None
def execute_process(self, component, cmd, env=None):
component.log_debug('%s addon exec: %s', self.name, repr(cmd))
try:
output = subprocess.check_output(
cmd,
env=get_clean_env(env),
cwd=component.full_path,
stderr=subprocess.STDOUT,
)
component.log_debug('exec result: %s', repr(output))
except (OSError, subprocess.CalledProcessError) as err:
output = getattr(err, 'output', '').decode('utf-8')
component.log_error('failed to exec %s: %s', repr(cmd), err)
for line in output.splitlines():
component.log_error('program output: %s', line)
self.alerts.append({
'addon': self.name,
'command': ' '.join(cmd),
'output': output,
'error': str(err),
})
def get_gpg_public_key() -> Optional[str]:
key = get_gpg_sign_key()
if key is None:
return None
cache_key = gpg_cache_key("public")
data = cache.get(cache_key)
if not data:
try:
result = subprocess.run(
["gpg", "--batch", "-armor", "--export", key],
env=get_clean_env(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True,
check=True,
)
data = result.stdout
cache.set(cache_key, data, 7 * 86400)
except (subprocess.CalledProcessError, OSError) as error:
gpg_error("GPG key public", error)
return None
return data
def get_gpg_key(silent=False):
try:
output = subprocess.check_output(
[
'gpg',
'--batch',
'--with-colons',
'--list-secret-keys',
settings.WEBLATE_GPG_IDENTITY,
],
stderr=subprocess.STDOUT,
env=get_clean_env(),
).decode('utf-8')
for line in output.splitlines():
if not line.startswith('fpr:'):
continue
return line.split(':')[9]
return None
except (subprocess.CalledProcessError, OSError) as exc:
if not silent:
add_configuration_error('GPG key listing', force_text(exc))
return None
def run_hook(component, translation, script, env=None, *args):
"""
Generic script hook executor.
"""
if script:
command = [script]
if args:
command.extend(args)
if component.is_repo_link:
target = component.linked_subproject
else:
target = component
environment = {
'WL_VCS': target.vcs,
'WL_REPO': target.repo,
'WL_PATH': target.get_path(),
'WL_FILEMASK': component.filemask,
'WL_TEMPLATE': component.template,
'WL_FILE_FORMAT': component.file_format,
'WL_BRANCH': component.branch,
}
if translation:
environment['WL_LANGUAGE'] = translation.language_code
if env is not None:
environment.update(env)
try:
subprocess.check_call(
command,
env=get_clean_env(environment),
cwd=component.get_path(),
)
return True
except (OSError, subprocess.CalledProcessError) as err:
component.log_error(
'failed to run hook script %s: %s',
script,
err
)
return False
def create_new_file(cls, filename, code, base):
"""Handles creation of new translation file."""
with open(base, 'r') as handle:
data = handle.read()
# Assume input is UTF-8 if not specified
if 'Content-Type: text/plain; charset=CHARSET' in data:
data = data.replace('Content-Type: text/plain; charset=CHARSET',
'Content-Type: text/plain; charset=UTF-8')
process = subprocess.Popen(
[
'msginit', '--input', '-', '--output', filename,
'--no-translator', '--locale', code
],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=get_clean_env(),
)
output, output_err = process.communicate(input=data)
retcode = process.poll()
if retcode:
raise ValueError(output_err if output_err else output)
def database_backup():
ensure_backup_dir()
database = settings.DATABASES["default"]
if database["ENGINE"] != "django.db.backends.postgresql":
return
cmd = [
"pg_dump",
"--file",
data_dir("backups", "database.sql"),
"--dbname",
database["NAME"],
]
if database["HOST"]:
cmd += ["--host", database["HOST"]]
if database["PORT"]:
cmd += ["--port", database["PORT"]]
if database["USER"]:
cmd += ["--username", database["USER"]]
subprocess.check_call(cmd,
env=get_clean_env(
{"PGPASSWORD": database["PASSWORD"]}))
def execute_process(self, component, cmd, env=None):
component.log_debug("%s addon exec: %s", self.name, " ".join(cmd))
try:
output = subprocess.check_output(
cmd,
env=get_clean_env(env),
cwd=component.full_path,
stderr=subprocess.STDOUT,
universal_newlines=True,
)
component.log_debug("exec result: %s", output)
except (OSError, subprocess.CalledProcessError) as err:
output = getattr(err, "output", "")
component.log_error("failed to exec %s: %s", repr(cmd), err)
for line in output.splitlines():
component.log_error("program output: %s", line)
self.alerts.append({
"addon": self.name,
"command": " ".join(cmd),
"output": output,
"error": str(err),
})
def get_gpg_key(silent=False):
try:
output = subprocess.check_output(
[
'gpg',
'--batch',
'--with-colons',
'--list-secret-keys',
settings.WEBLATE_GPG_IDENTITY,
],
stderr=subprocess.STDOUT,
env=get_clean_env(),
).decode('utf-8')
for line in output.splitlines():
if not line.startswith('fpr:'):
continue
return line.split(':')[9]
return None
except (subprocess.CalledProcessError, OSError) as exc:
if not silent:
add_configuration_error('GPG key listing', force_text(exc))
return None
def generate_ssh_key(request):
"""Generate SSH key."""
keyfile = ssh_file(RSA_KEY)
pubkeyfile = ssh_file(RSA_KEY_PUB)
try:
# Actually generate the key
subprocess.run(
[
"ssh-keygen",
"-q",
"-b",
"4096",
"-N",
"",
"-C",
"Weblate",
"-t",
"rsa",
"-f",
keyfile,
],
universal_newlines=True,
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=get_clean_env(),
)
except (subprocess.CalledProcessError, OSError) as exc:
messages.error(
request,
_("Failed to generate key: %s") % getattr(exc, "output", str(exc)))
return
# Fix key permissions
os.chmod(keyfile, stat.S_IWUSR | stat.S_IRUSR)
os.chmod(pubkeyfile,
stat.S_IWUSR | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)
messages.success(request, _("Created new SSH key."))
def execute_process(self, component, cmd, env=None):
component.log_debug('%s addon exec: %s', self.name, repr(cmd))
try:
output = subprocess.check_output(
cmd,
env=get_clean_env(env),
cwd=component.full_path,
stderr=subprocess.STDOUT,
)
component.log_debug('exec result: %s', repr(output))
except (OSError, subprocess.CalledProcessError) as err:
component.log_error('failed to exec %s: %s', repr(cmd), err)
self.alerts.append({
'addon':
self.name,
'command':
' '.join(cmd),
'output':
getattr(err, 'output', '').decode('utf-8'),
'error':
str(err),
})
def add_host_key(request, host, port=''):
"""Add host key for a host."""
if not host:
messages.error(request, _('Invalid host name given!'))
else:
cmdline = ['ssh-keyscan']
if port:
cmdline.extend(['-p', str(port)])
cmdline.append(host)
try:
output = subprocess.check_output(
cmdline, stderr=subprocess.STDOUT, env=get_clean_env()
)
keys = []
for key in output.decode('utf-8').splitlines():
key = key.strip()
if not is_key_line(key):
continue
keys.append(key)
host, keytype, fingerprint = parse_hosts_line(key)
messages.warning(
request,
_(
'Added host key for %(host)s with fingerprint '
'%(fingerprint)s (%(keytype)s), '
'please verify that it is correct.'
)
% {'host': host, 'fingerprint': fingerprint, 'keytype': keytype},
)
if not keys:
messages.error(request, _('Failed to fetch public key for a host!'))
with open(ssh_file(KNOWN_HOSTS), 'a') as handle:
for key in keys:
handle.write('{0}\n'.format(key))
except subprocess.CalledProcessError as exc:
messages.error(request, _('Failed to get host key: %s') % exc.output)
except OSError as exc:
messages.error(request, _('Failed to get host key: %s') % str(exc))
def run_hook(component, translation, script, env=None, *args):
"""Generic script hook executor."""
if script:
command = [script]
if args:
command.extend(args)
if component.is_repo_link:
target = component.linked_subproject
else:
target = component
environment = {
'WL_VCS': target.vcs,
'WL_REPO': target.repo,
'WL_PATH': target.get_path(),
'WL_FILEMASK': component.filemask,
'WL_TEMPLATE': component.template,
'WL_NEW_BASE': component.new_base,
'WL_FILE_FORMAT': component.file_format,
'WL_BRANCH': component.branch,
}
if translation:
environment['WL_LANGUAGE'] = translation.language_code
if env is not None:
environment.update(env)
try:
subprocess.check_call(
command,
env=get_clean_env(environment),
cwd=component.get_path(),
)
return True
except (OSError, subprocess.CalledProcessError) as err:
component.log_error(
'failed to run hook script %s: %s',
script,
err
)
return False
def generate_gpg_key():
try:
subprocess.check_output(
[
'gpg',
'--batch',
'--pinentry-mode',
'loopback',
'--passphrase',
'',
'--quick-generate-key',
settings.WEBLATE_GPG_IDENTITY,
settings.WEBLATE_GPG_ALGO,
'default',
'never',
],
stderr=subprocess.STDOUT,
env=get_clean_env(),
)
return get_gpg_key()
except (subprocess.CalledProcessError, OSError) as exc:
add_configuration_error('GPG key generating', force_text(exc))
return None
def get_gpg_public_key():
key = get_gpg_sign_key()
if key is None:
return None
data = cache.get("gpg-key-public")
if not data:
try:
result = subprocess.run(
["gpg", "--batch", "-armor", "--export", key],
env=get_clean_env(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True,
check=True,
)
data = result.stdout
cache.set("gpg-key-public", data, 7 * 86400)
delete_configuration_error("GPG key public")
except (subprocess.CalledProcessError, OSError) as error:
report_error(cause="GPG key public")
add_configuration_error("GPG key public", force_str(error))
return None
return data
def get_gpg_key(silent=False) -> Optional[str]:
try:
result = subprocess.run(
[
"gpg",
"--batch",
"--with-colons",
"--list-secret-keys",
settings.WEBLATE_GPG_IDENTITY,
],
capture_output=True,
env=get_clean_env(),
text=True,
check=True,
)
for line in result.stdout.splitlines():
if not line.startswith("fpr:"):
continue
return line.split(":")[9]
return None
except (subprocess.CalledProcessError, OSError) as error:
gpg_error("GPG key listing", error, silent)
return None
def generate_ssh_key(request):
"""
Generates SSH key.
"""
try:
# Create directory if it does not exist
ensure_ssh_dir()
# Actually generate the key
subprocess.check_output(
[
'ssh-keygen', '-q', '-N', '', '-C', 'Weblate', '-t', 'rsa',
'-f',
ssh_file(RSA_KEY)
],
stderr=subprocess.STDOUT,
env=get_clean_env(),
)
messages.success(request, _('Created new SSH key.'))
except (subprocess.CalledProcessError, OSError) as exc:
messages.error(
request,
_('Failed to generate key: %s') % getattr(exc, 'output', str(exc)))
def get_gpg_public_key():
key = get_gpg_sign_key()
if key is None:
return None
data = cache.get('gpg-key-public')
if not data:
try:
data = subprocess.check_output(
[
'gpg',
'--batch',
'-armor',
'--export',
key,
],
stderr=subprocess.STDOUT,
env=get_clean_env(),
).decode('utf-8')
cache.set('gpg-key-public', data, 7 * 86400)
except (subprocess.CalledProcessError, OSError) as exc:
add_configuration_error('GPG key public', force_text(exc))
return None
return data
def get_gpg_public_key():
key = get_gpg_sign_key()
if key is None:
return None
data = cache.get('gpg-key-public')
if not data:
try:
data = subprocess.check_output(
[
'gpg',
'--batch',
'-armor',
'--export',
key,
],
stderr=subprocess.STDOUT,
env=get_clean_env(),
).decode('utf-8')
cache.set('gpg-key-public', data, 7 * 86400)
except (subprocess.CalledProcessError, OSError) as exc:
add_configuration_error('GPG key public', force_text(exc))
return None
return data
def _popen(cls, args, cwd=None):
'''
Executes the command using popen.
'''
if args is None:
raise RepositoryException(0, 'Not supported functionality', '')
args = [cls._cmd] + args
process = subprocess.Popen(
args,
cwd=cwd,
env=get_clean_env({'GIT_SSH': ssh_file(SSH_WRAPPER)}),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
output, output_err = process.communicate()
retcode = process.poll()
if retcode:
raise RepositoryException(
retcode,
output_err.decode('utf-8'),
output.decode('utf-8')
)
return output.decode('utf-8')
def __git_commit(self, author, timestamp, sync=False):
'''
Commits translation to git.
'''
# Format commit message
msg = self.get_commit_message()
# Pre commit hook
if self.subproject.pre_commit_script != '':
try:
subprocess.check_call(
[self.subproject.pre_commit_script,
self.get_filename()],
env=get_clean_env(),
)
except (OSError, subprocess.CalledProcessError) as err:
self.log_error('failed to run pre commit script %s: %s',
self.subproject.pre_commit_script, err)
# Create list of files to commit
files = [self.filename]
if self.subproject.extra_commit_file != '':
extra_file = self.subproject.extra_commit_file % {
'language': self.language_code,
}
full_path_extra = os.path.join(self.subproject.get_path(),
extra_file)
if os.path.exists(full_path_extra):
files.append(extra_file)
# Do actual commit
self.repository.commit(msg, author, timestamp, files)
# Optionally store updated hash
if sync:
self.store_hash()
def run_script(self, component=None, translation=None, env=None):
command = [self.script]
if translation:
component = translation.component
command.extend(translation.get_filename())
if component.is_repo_link:
target = component.linked_component
else:
target = component
environment = {
'WL_VCS': target.vcs,
'WL_REPO': target.repo,
'WL_PATH': target.full_path,
'WL_FILEMASK': component.filemask,
'WL_TEMPLATE': component.template,
'WL_NEW_BASE': component.new_base,
'WL_FILE_FORMAT': component.file_format,
'WL_BRANCH': component.branch,
}
if translation:
environment['WL_LANGUAGE'] = translation.language_code
if env is not None:
environment.update(env)
try:
subprocess.check_call(
command,
env=get_clean_env(environment),
cwd=component.full_path,
)
except (OSError, subprocess.CalledProcessError) as err:
component.log_error(
'failed to run hook script %s: %s',
self.script,
err
)
raise
def get_gpg_key(silent=False):
try:
output = subprocess.check_output(
[
"gpg",
"--batch",
"--with-colons",
"--list-secret-keys",
settings.WEBLATE_GPG_IDENTITY,
],
stderr=subprocess.STDOUT,
env=get_clean_env(),
).decode("utf-8")
for line in output.splitlines():
if not line.startswith("fpr:"):
continue
delete_configuration_error("GPG key listing")
return line.split(":")[9]
return None
except (subprocess.CalledProcessError, OSError) as exc:
report_error(exc, prefix="GPG key listing")
if not silent:
add_configuration_error("GPG key listing", force_text(exc))
return None
def database_backup():
if settings.DATABASE_BACKUP == "none":
return
ensure_backup_dir()
database = settings.DATABASES["default"]
if database["ENGINE"] != "django.db.backends.postgresql":
return
cmd = ["pg_dump", "--dbname", database["NAME"]]
if database["HOST"]:
cmd += ["--host", database["HOST"]]
if database["PORT"]:
cmd += ["--port", database["PORT"]]
if database["USER"]:
cmd += ["--username", database["USER"]]
if settings.DATABASE_BACKUP == "compressed":
cmd += ["--file", data_dir("backups", "database.sql.gz")]
cmd += ["--compress", "6"]
else:
cmd += ["--file", data_dir("backups", "database.sql")]
try:
subprocess.run(
cmd,
env=get_clean_env({"PGPASSWORD": database["PASSWORD"]}),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stdin=subprocess.DEVNULL,
check=True,
universal_newlines=True,
)
except subprocess.CalledProcessError as error:
report_error(extra_data={
"stdout": error.stdout,
"stderr": error.stderr
})
raise
def _getenv():
"""Generate environment for process execution."""
return get_clean_env({
"GIT_SSH": SSH_WRAPPER.filename,
"GIT_TERMINAL_PROMPT": "0"
})
def _getenv():
"""Generate environment for process execution."""
return get_clean_env({
'GIT_SSH': SSH_WRAPPER.filename,
'GIT_TERMINAL_PROMPT': '0',
})
def database_backup():
if settings.DATABASE_BACKUP == "none":
return
with backup_lock():
database = settings.DATABASES["default"]
env = get_clean_env()
compress = settings.DATABASE_BACKUP == "compressed"
out_compressed = data_dir("backups", "database.sql.gz")
out_plain = data_dir("backups", "database.sql")
if using_postgresql():
cmd = ["pg_dump", "--dbname", database["NAME"]]
if database["HOST"]:
cmd.extend(["--host", database["HOST"]])
if database["PORT"]:
cmd.extend(["--port", database["PORT"]])
if database["USER"]:
cmd.extend(["--username", database["USER"]])
if settings.DATABASE_BACKUP == "compressed":
cmd.extend(["--file", out_compressed])
cmd.extend(["--compress", "6"])
compress = False
else:
cmd.extend(["--file", out_plain])
env["PGPASSWORD"] = database["PASSWORD"]
else:
cmd = [
"mysqldump",
"--result-file",
out_plain,
"--single-transaction",
"--skip-lock-tables",
]
if database["HOST"]:
cmd.extend(["--host", database["HOST"]])
if database["PORT"]:
cmd.extend(["--port", database["PORT"]])
if database["USER"]:
cmd.extend(["--user", database["USER"]])
cmd.extend(["--databases", database["NAME"]])
env["MYSQL_PWD"] = database["PASSWORD"]
try:
subprocess.run(
cmd,
env=env,
capture_output=True,
stdin=subprocess.DEVNULL,
check=True,
text=True,
)
except subprocess.CalledProcessError as error:
add_breadcrumb(
category="backup",
message="database dump output",
stdout=error.stdout,
stderr=error.stderr,
)
report_error()
raise
if compress:
with open(out_plain, "rb") as f_in:
with gzip.open(out_compressed, "wb") as f_out:
shutil.copyfileobj(f_in, f_out)
os.unlink(out_plain)
def _getenv():
"""Generate environment for process execution."""
return get_clean_env({
'GIT_SSH': SSH_WRAPPER.filename,
'GIT_TERMINAL_PROMPT': '0',
})
def _getenv():
"""Generates environment for process execution."""
return get_clean_env({'GIT_SSH': ssh_file(SSH_WRAPPER)})
def _getenv():
"""Generate environment for process execution."""
return get_clean_env({'GIT_SSH': get_wrapper_filename()})
def _getenv():
"""Generates environment for process execution."""
return get_clean_env({"GIT_SSH": ssh_file(SSH_WRAPPER)})
def _getenv():
"""Generate environment for process execution."""
return get_clean_env({'GIT_SSH': get_wrapper_filename()})
def supports_new_language(cls):
"""
Checks whether we can create new language file.
"""
if cls.msginit_found is None:
try:
ret = subprocess.check_call(
["msginit", "--help"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=get_clean_env()
)
cls.msginit_found = ret == 0
except (subprocess.CalledProcessError, OSError):
cls.msginit_found = False
return cls.msginit_found
def database_backup():
if settings.DATABASE_BACKUP == "none":
return
ensure_backup_dir()
database = settings.DATABASES["default"]
env = get_clean_env()
compress = settings.DATABASE_BACKUP == "compressed"
out_compressed = data_dir("backups", "database.sql.gz")
out_plain = data_dir("backups", "database.sql")
if database["ENGINE"] == "django.db.backends.postgresql":
cmd = ["pg_dump", "--dbname", database["NAME"]]
if database["HOST"]:
cmd.extend(["--host", database["HOST"]])
if database["PORT"]:
cmd.extend(["--port", database["PORT"]])
if database["USER"]:
cmd.extend(["--username", database["USER"]])
if settings.DATABASE_BACKUP == "compressed":
cmd.extend(["--file", out_compressed])
cmd.extend(["--compress", "6"])
compress = False
else:
cmd.extend(["--file", out_plain])
env["PGPASSWORD"] = database["PASSWORD"]
elif database["ENGINE"] == "django.db.backends.mysql":
cmd = [
"mysqldump",
"--result-file",
out_plain,
"--single-transaction",
"--skip-lock-tables",
]
if database["HOST"]:
cmd.extend(["--host", database["HOST"]])
if database["PORT"]:
cmd.extend(["--port", database["PORT"]])
if database["USER"]:
cmd.extend(["--user", database["USER"]])
cmd.extend(["--databases", database["NAME"]])
env["MYSQL_PWD"] = database["PASSWORD"]
else:
return
try:
subprocess.run(
cmd,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stdin=subprocess.DEVNULL,
check=True,
universal_newlines=True,
)
except subprocess.CalledProcessError as error:
report_error(extra_data={
"stdout": error.stdout,
"stderr": error.stderr
})
raise
if compress:
with open(out_plain, "rb") as f_in:
with gzip.open(out_compressed, "wb") as f_out:
shutil.copyfileobj(f_in, f_out)
os.unlink(out_plain)
def _getenv():
"""Generate environment for process execution."""
return get_clean_env({'GIT_SSH': SSH_WRAPPER.filename})
