def bitbucket_hook_helper(data):
"""API to handle service hooks from Bitbucket."""
if 'push' in data:
return bitbucket_webhook_helper(data)
# Parse owner, branch and repository name
owner = data['repository']['owner']
slug = data['repository']['slug']
if data['commits']:
branch = data['commits'][-1]['branch']
else:
branch = None
params = {'owner': owner, 'slug': slug}
# Construct possible repository URLs
if data['repository']['scm'] == 'git':
repos = [repo % params for repo in BITBUCKET_GIT_REPOS]
elif data['repository']['scm'] == 'hg':
repos = [repo % params for repo in BITBUCKET_HG_REPOS]
else:
LOGGER.error(
'unsupported repository: %s',
repr(data['repository'])
)
raise ValueError('unsupported repository')
return {
'service_long_name': 'Bitbucket',
'repo_url': ''.join([
data['canon_url'], data['repository']['absolute_url']
]),
'repos': repos,
'branch': branch,
}
def get_avatar_image(request, user, size):
"""Return avatar image from cache (if available) or download it."""
cache_key = '-'.join((
'avatar-img',
user.username,
str(size)
))
# Try using avatar specific cache if available
try:
cache = caches['avatar']
except InvalidCacheBackendError:
cache = caches['default']
image = cache.get(cache_key)
if image is None:
try:
image = download_avatar_image(user, size)
cache.set(cache_key, image)
except (IOError, CertificateError) as error:
report_error(
error, sys.exc_info(), request,
extra_data={'avatar': user.username},
level='debug',
)
LOGGER.error(
'Failed to fetch avatar for %s: %s',
user.username,
str(error)
)
return get_fallback_avatar(size)
return image
def mail_admins_contact(request, subject, message, context, sender, to):
"""Send a message to the admins, as defined by the ADMINS setting."""
LOGGER.info("contact form from %s", sender)
if not to and settings.ADMINS:
to = [a[1] for a in settings.ADMINS]
elif not settings.ADMINS:
messages.error(request, _("Could not send message to administrator."))
LOGGER.error("ADMINS not configured, cannot send message")
return
mail = EmailMultiAlternatives(
subject="{}{}".format(settings.EMAIL_SUBJECT_PREFIX,
subject % context),
body="{}\n{}".format(
message % context,
TEMPLATE_FOOTER.format(
address=get_ip_address(request),
agent=get_user_agent(request),
username=request.user.username,
),
),
to=to,
from_email=sender,
)
mail.send(fail_silently=False)
messages.success(
request,
_("Your request has been sent, you will shortly hear from us."))
def get_plural_type(code, pluralequation):
'''
Gets correct plural type for language.
'''
# Remove not needed parenthesis
if pluralequation[-1] == ';':
pluralequation = pluralequation[:-1]
if pluralequation[0] == '(' and pluralequation[-1] == ')':
pluralequation = pluralequation[1:-1]
# Get base language code
base_code = code.replace('_', '-').split('-')[0]
# No plural
if pluralequation == '0':
return data.PLURAL_NONE
# Standard plural equations
for mapping in data.PLURAL_MAPPINGS:
if pluralequation in mapping[0]:
return mapping[1]
# Arabic special case
if base_code in ('ar',):
return data.PLURAL_ARABIC
# Log error in case of uknown mapping
LOGGER.error(
'Can not guess type of plural for %s: %s', code, pluralequation
)
return data.PLURAL_UNKNOWN
def bitbucket_hook_helper(data):
"""API to handle service hooks from Bitbucket."""
if 'diagnostics' in data:
return None
if 'push' in data:
return bitbucket_webhook_helper(data)
# Parse owner, branch and repository name
owner = data['repository']['owner']
slug = data['repository']['slug']
if data['commits']:
branch = data['commits'][-1]['branch']
else:
branch = None
params = {'owner': owner, 'slug': slug}
# Construct possible repository URLs
if data['repository']['scm'] == 'git':
repos = [repo % params for repo in BITBUCKET_GIT_REPOS]
elif data['repository']['scm'] == 'hg':
repos = [repo % params for repo in BITBUCKET_HG_REPOS]
else:
LOGGER.error('unsupported repository: %s', repr(data['repository']))
raise ValueError('unsupported repository')
return {
'service_long_name':
'Bitbucket',
'repo_url':
''.join([data['canon_url'], data['repository']['absolute_url']]),
'repos':
repos,
'branch':
branch,
}
def mail_admins_contact(request, subject, message, context, sender, to):
"""Send a message to the admins, as defined by the ADMINS setting."""
LOGGER.info(
'contact form from %s',
sender,
)
if not to and settings.ADMINS:
to = [a[1] for a in settings.ADMINS]
elif not settings.ADMINS:
messages.error(
request,
_('Message could not be sent to administrator!')
)
LOGGER.error(
'ADMINS not configured, can not send message!'
)
return
mail = EmailMultiAlternatives(
'{0}{1}'.format(settings.EMAIL_SUBJECT_PREFIX, subject % context),
message % context,
to=to,
headers={'Reply-To': sender},
)
mail.send(fail_silently=False)
messages.success(
request,
_('Message has been sent to administrator.')
)
def get_plural_type(base_code, pluralequation):
"""Get correct plural type for language."""
# Remove not needed parenthesis
if pluralequation[-1] == ';':
pluralequation = pluralequation[:-1]
# No plural
if pluralequation == '0':
return data.PLURAL_NONE
# Standard plural equations
for mapping in data.PLURAL_MAPPINGS:
if pluralequation in mapping[0]:
return mapping[1]
# Arabic special case
if base_code in ('ar',):
return data.PLURAL_ARABIC
# Log error in case of uknown mapping
LOGGER.error(
'Can not guess type of plural for %s: %s', base_code, pluralequation
)
return data.PLURAL_UNKNOWN
def mail_admins_contact(request, subject, message, context, sender, to):
"""Send a message to the admins, as defined by the ADMINS setting."""
LOGGER.info(
'contact form from %s',
sender,
)
if not to and settings.ADMINS:
to = [a[1] for a in settings.ADMINS]
elif not settings.ADMINS:
messages.error(
request,
_('Message could not be sent to administrator!')
)
LOGGER.error(
'ADMINS not configured, can not send message!'
)
return
mail = EmailMultiAlternatives(
'{0}{1}'.format(settings.EMAIL_SUBJECT_PREFIX, subject % context),
message % context,
to=to,
headers={'Reply-To': sender},
)
mail.send(fail_silently=False)
messages.success(
request,
_('Message has been sent to administrator.')
)
def unit_post_save(self, unit, created):
if created or unit.state != STATE_APPROVED:
return
stats = unit.translation.get_stats()
is_approved = int(unit.translation.stats.approved_percent) == 100
is_translated = int(stats.get('translated_percent', 0)) == 100
is_fuzzy = int(stats.get('fuzzy_percent', 0)) == 100
if is_approved and is_translated and not is_fuzzy:
site_id, _ = os.path.splitext(
os.path.split(unit.translation.filename)[-1]
)
try:
r = requests.post(
self.PLATFORM_NOTIFY_URL,
json={'site_id': site_id},
# WANT: Optionally add extra headers to check
# coming from Weblate. (e.g. X-WEBLATE: <val>)
)
r.raise_for_status()
except requests.exceptions.HTTPError:
logger.exception(
'Call to UWAI Platform failed: %s', r.text
)
return
def get_plural_type(base_code, pluralequation):
"""Get correct plural type for language."""
# Remove not needed parenthesis
if pluralequation[-1] == ';':
pluralequation = pluralequation[:-1]
# No plural
if pluralequation == '0':
return data.PLURAL_NONE
# Standard plural equations
for mapping in data.PLURAL_MAPPINGS:
if pluralequation in mapping[0]:
return mapping[1]
# Arabic special case
if base_code in ('ar',):
return data.PLURAL_ARABIC
# Log error in case of uknown mapping
LOGGER.error(
'Can not guess type of plural for %s: %s', base_code, pluralequation
)
return data.PLURAL_UNKNOWN
def report_error(error,
request=None,
extra_data=None,
level='warning',
prefix='Handled exception',
skip_raven=False,
print_tb=False):
"""Wrapper for error reporting
This can be used for store exceptions in error reporting solutions as
rollbar while handling error gracefully and giving user cleaner message.
"""
if HAS_ROLLBAR and hasattr(settings, 'ROLLBAR'):
rollbar.report_exc_info(request=request,
extra_data=extra_data,
level=level)
if not skip_raven and HAS_RAVEN and hasattr(settings, 'RAVEN_CONFIG'):
raven_client.captureException(request=request,
extra=extra_data,
level=level)
LOGGER.error('%s: %s: %s', prefix, error.__class__.__name__,
force_text(error))
if print_tb:
LOGGER.exception(prefix)
def check_rate_limit(scope, request):
"""Check authentication rate limit."""
if request.user.is_superuser:
return True
key = get_cache_key(scope, request)
try:
# Try to increase cache key
attempts = cache.incr(key)
except ValueError:
# No such key, so set it
cache.set(key, 1, get_rate_setting(scope, "WINDOW"))
attempts = 1
if attempts > get_rate_setting(scope, "ATTEMPTS"):
# Set key to longer expiry for lockout period
cache.set(key, attempts, get_rate_setting(scope, "LOCKOUT"))
LOGGER.info(
"rate-limit lockout for %s in %s scope from %s",
key,
scope,
get_ip_address(request),
)
return False
return True
def get_avatar_image(user, size):
"""
Returns avatar image from cache (if available) or downloads it.
"""
cache_key = 'avatar-img-{0}-{1}'.format(
user.username,
size
)
# Try using avatar specific cache if available
try:
cache = caches['avatar']
except InvalidCacheBackendError:
cache = caches['default']
image = cache.get(cache_key)
if image is None:
try:
image = download_avatar_image(user, size)
cache.set(cache_key, image)
except IOError as error:
report_error(
error, sys.exc_info(),
extra_data={'avatar': user.username}
)
LOGGER.error(
'Failed to fetch avatar for %s: %s',
user.username,
str(error)
)
return get_fallback_avatar(size)
return image
def send_mails(mails):
"""Sends multiple mails in single connection."""
try:
connection = get_connection()
connection.send_messages([mail for mail in mails if mail is not None])
except SMTPException as error:
LOGGER.error('Failed to send email: %s', error)
report_error(error, sys.exc_info())
def report_error(self, exc, request, message):
"""Wrapper for handling error situations"""
extra = {'mt_url': self.request_url, 'mt_params': self.request_params}
report_error(exc, request, extra, prefix='Machine translation error')
LOGGER.error(message, self.name)
LOGGER.info(
'Last URL: %s, params: %s', extra['mt_url'], extra['mt_params']
)
def send_mails(mails):
"""Sends multiple mails in single connection."""
try:
connection = get_connection()
connection.send_messages(mails)
except SMTPException as error:
LOGGER.error('Failed to send email: %s', error)
report_error(error, sys.exc_info())
def report_error(self, exc, request, message):
"""Wrapper for handling error situations"""
extra = {'mt_url': self.request_url, 'mt_params': self.request_params}
report_error(exc, request, extra)
LOGGER.error(message, self.name)
LOGGER.info(
'Last URL: %s, params: %s', extra['mt_url'], extra['mt_params']
)
def get_notification_email(language, email, notification,
context=None, info=None):
"""Render notification email."""
context = context or {}
headers = {}
LOGGER.info(
'sending notification %s on %s to %s',
notification,
info,
email
)
with override('en' if language is None else language):
# Template name
context['subject_template'] = 'mail/{0}_subject.txt'.format(
notification
)
context['LANGUAGE_CODE'] = get_language()
context['LANGUAGE_BIDI'] = get_language_bidi()
# Adjust context
context['current_site_url'] = get_site_url()
context['site_title'] = settings.SITE_TITLE
# Render subject
subject = render_to_string(
context['subject_template'],
context
).strip()
# Render body
html_body = render_to_string(
'mail/{0}.html'.format(notification),
context
)
body = html2text(html_body)
# Define headers
headers['Auto-Submitted'] = 'auto-generated'
headers['X-AutoGenerated'] = 'yes'
headers['Precedence'] = 'bulk'
headers['X-Mailer'] = 'Weblate {0}'.format(VERSION)
# List of recipients
if email == 'ADMINS':
emails = [a[1] for a in settings.ADMINS]
else:
emails = [email]
# Return the mail content
return {
'subject': subject,
'body': body,
'to': emails,
'headers': headers,
'html_body': html_body,
}
def get_notification_email(language, email, notification,
context=None, info=None):
"""Render notification email."""
context = context or {}
headers = {}
LOGGER.info(
'sending notification %s on %s to %s',
notification,
info,
email
)
with override('en' if language is None else language):
# Template name
context['subject_template'] = 'mail/{0}_subject.txt'.format(
notification
)
context['LANGUAGE_CODE'] = get_language()
context['LANGUAGE_BIDI'] = get_language_bidi()
# Adjust context
context['current_site_url'] = get_site_url()
context['site_title'] = settings.SITE_TITLE
# Render subject
subject = render_to_string(
context['subject_template'],
context
).strip()
# Render body
html_body = render_to_string(
'mail/{0}.html'.format(notification),
context
)
body = html2text(html_body)
# Define headers
headers['Auto-Submitted'] = 'auto-generated'
headers['X-AutoGenerated'] = 'yes'
headers['Precedence'] = 'bulk'
headers['X-Mailer'] = 'Weblate {0}'.format(VERSION)
# List of recipients
if email == 'ADMINS':
emails = [a[1] for a in settings.ADMINS]
else:
emails = [email]
# Return the mail content
return {
'subject': subject,
'body': body,
'to': emails,
'headers': headers,
'html_body': html_body,
}
def clean_captcha(self):
"""
Validation for captcha.
"""
if self.tampering or not self.captcha.validate(self.cleaned_data["captcha"]):
raise forms.ValidationError(_("Please check your math and try again."))
mail = self.cleaned_data.get("email", "NONE")
LOGGER.info("Passed captcha for %s (%s = %s)", mail, self.captcha.question, self.cleaned_data["captcha"])
def clean_captcha(self):
"""Validation for captcha."""
if (self.tampering
or not self.captcha.validate(self.cleaned_data['captcha'])):
raise forms.ValidationError(
_('Please check your math and try again.'))
mail = self.cleaned_data.get('email', 'NONE')
LOGGER.info('Passed captcha for %s (%s = %s)', mail,
self.captcha.question, self.cleaned_data['captcha'])
def clean_captcha(self):
"""Validation for captcha."""
if (self.fresh
or not self.captcha.validate(self.cleaned_data['captcha'])):
self.generate_captcha()
raise forms.ValidationError(
_('Please check your math and try again with new expression.'))
mail = self.cleaned_data.get('email', 'NONE')
LOGGER.info('Passed captcha for %s (%s = %s)', mail,
self.captcha.question, self.cleaned_data['captcha'])
def create_component(self, main, match, **params):
max_length = settings.COMPONENT_NAME_LENGTH
def get_val(key, extra=0):
result = match[key]
if len(result) > max_length - extra:
result = result[:max_length - extra]
return result
name = get_val('name')
slug = get_val('slug')
simple_keys = (
'project',
'branch',
'vcs',
'push_on_commit',
'license_url',
'license',
)
for key in simple_keys:
if key not in params:
params[key] = getattr(main, key)
if 'repo' not in params:
params['repo'] = main.get_repo_link_url()
components = Component.objects.filter(project=params['project'])
if components.filter(Q(slug=slug) | Q(name=name)).exists():
base_name = get_val('name', 4)
base_slug = get_val('slug', 4)
for i in range(1, 1000):
name = '{} {}'.format(base_name, i)
slug = '{}-{}'.format(base_slug, i)
if components.filter(Q(slug=slug) | Q(name=name)).exists():
continue
break
if self.component:
self.component.log_info('Creating component %s', name)
else:
LOGGER.info('Creating component %s', name)
return Component.objects.create(name=name,
slug=slug,
template=match['base_file'],
filemask=match['mask'],
file_format=self.file_format,
language_regex=self.language_re,
**params)
def report_error(error, exc_info, request=None, extra_data=None):
"""Wrapper for error reporting
This can be used for store exceptions in error reporting solutions as
rollbar while handling error gracefully and giving user cleaner message.
"""
if HAS_ROLLBAR and hasattr(settings, 'ROLLBAR'):
rollbar.report_exc_info(exc_info, request, extra_data=extra_data)
LOGGER.error(
'Handled exception %s: %s',
error.__class__.__name__,
str(error)
)
def bitbucket_hook_helper(data, request):
"""API to handle service hooks from Bitbucket."""
# Bitbucket ping event
if request and request.META.get("HTTP_X_EVENT_KEY") not in (
"repo:push",
"repo:refs_changed",
"pullrequest:fulfilled",
"pr:merged",
):
return None
if "pullRequest" in data:
# The pr:merged event
repository = data["pullRequest"]["fromRef"]["repository"]
else:
repository = data["repository"]
full_name = bitbucket_extract_full_name(repository)
repo_url = bitbucket_extract_repo_url(data, repository)
# Extract repository links
if "links" in repository and "clone" in repository["links"]:
repos = [val["href"] for val in repository["links"]["clone"]]
else:
repo_servers = {"bitbucket.org", urlparse(repo_url).hostname}
repos = []
if "scm" not in data["repository"]:
templates = BITBUCKET_GIT_REPOS + BITBUCKET_HG_REPOS
elif data["repository"]["scm"] == "hg":
templates = BITBUCKET_HG_REPOS
else:
templates = BITBUCKET_GIT_REPOS
# Construct possible repository URLs
for repo in templates:
repos.extend(
repo.format(full_name=full_name, server=server)
for server in repo_servers
)
if not repos:
LOGGER.error("unsupported repository: %s", repr(data["repository"]))
raise ValueError("unsupported repository")
return {
"service_long_name": "Bitbucket",
"repo_url": repo_url,
"repos": repos,
"branch": bitbucket_extract_branch(data),
"full_name": f"{full_name}.git",
}
def send_immediate(self, language, email, change):
with override('en' if language is None else language):
context = self.get_context(change)
subject = self.render_template('_subject.txt', context)
context['subject'] = subject
LOGGER.info(
'sending notification %s on %s to %s',
self.get_name(), context['component'], email,
)
self.send(
email,
subject,
self.render_template('.html', context),
self.get_headers(context),
)
def report_error(error, exc_info, request=None, extra_data=None):
"""Wrapper for error reporting
This can be used for store exceptions in error reporting solutions as
rollbar while handling error gracefully and giving user cleaner message.
"""
if HAS_ROLLBAR and hasattr(settings, 'ROLLBAR'):
rollbar.report_exc_info(exc_info, request, extra_data=extra_data)
LOGGER.error('Handled exception %s: %s', error.__class__.__name__,
force_text(error).encode('utf-8'))
# Print error when running testsuite
if sys.argv[1:2] == ['test']:
traceback.print_exc()
def report_error(self, exc, message):
"""Wrapper for handling error situations"""
report_error(
exc, sys.exc_info(),
{'mt_url': self.request_url, 'mt_params': self.request_params}
)
LOGGER.error(
message,
self.name,
)
LOGGER.error(
'Last fetched URL: %s, params: %s',
self.request_url,
self.request_params,
)
def report_error(self, exc, message):
"""Wrapper for handling error situations"""
report_error(
exc, sys.exc_info(),
{'mt_url': self.request_url, 'mt_params': self.request_params}
)
LOGGER.error(
message,
self.name,
)
LOGGER.error(
'Last fetched URL: %s, params: %s',
self.request_url,
self.request_params,
)
def clean_checksum(self):
"""Validate whether checksum is valid and fetches unit for it."""
if 'checksum' not in self.cleaned_data:
return
unit_set = self.translation.unit_set
try:
self.cleaned_data['unit'] = unit_set.filter(
id_hash=self.cleaned_data['checksum'])[0]
except (Unit.DoesNotExist, IndexError):
LOGGER.error('message %s disappeared!',
self.cleaned_data['checksum'])
raise ValidationError(
_('Message you wanted to translate is no longer available!'))
def send_immediate(self, language, email, change):
with override('en' if language is None else language):
context = self.get_context(change)
subject = self.render_template('_subject.txt', context)
context['subject'] = subject
LOGGER.info(
'sending notification %s on %s to %s',
self.get_name(), context['component'], email,
)
self.send(
email,
subject,
self.render_template('.html', context),
self.get_headers(context),
)
def vcs_service_hook(request, service):
'''
Shared code between VCS service hooks.
Currently used for bitbucket_hook, github_hook and gitlab_hook, but should
be usable for other VCS services (Google Code, custom coded sites, etc.)
too.
'''
# We support only post methods
if not appsettings.ENABLE_HOOKS:
return HttpResponseNotAllowed(())
# Check if we got payload
try:
# GitLab sends json as application/json
if request.META['CONTENT_TYPE'] == 'application/json':
data = json.loads(request.body)
# Bitbucket and GitHub sends json as x-www-form-data
else:
data = json.loads(request.POST['payload'])
except (ValueError, KeyError):
return HttpResponseBadRequest('Could not parse JSON payload!')
# Get service helper
if service not in HOOK_HANDLERS:
LOGGER.error('service %s is not supported', service)
return HttpResponseBadRequest('invalid service')
hook_helper = HOOK_HANDLERS[service]
# Send the request data to the service handler.
try:
service_data = hook_helper(data)
except KeyError:
LOGGER.error('failed to parse service %s data', service)
return HttpResponseBadRequest('Invalid data in json payload!')
# Log data
service_long_name = service_data['service_long_name']
repos = service_data['repos']
repo_url = service_data['repo_url']
branch = service_data['branch']
LOGGER.info('received %s notification on repository %s, branch %s',
service_long_name, repo_url, branch)
subprojects = SubProject.objects.filter(repo__in=repos)
if branch is not None:
subprojects = subprojects.filter(branch=branch)
# Trigger updates
for obj in subprojects:
if not obj.project.enable_hooks:
continue
LOGGER.info('%s notification will update %s', service_long_name, obj)
perform_update(obj)
return hook_response()
def clean_captcha(self):
"""Validation for captcha."""
if (self.fresh
or not self.captcha.validate(self.cleaned_data['captcha'])):
self.generate_captcha()
rotate_token(self.request)
raise forms.ValidationError(
_('Please check your math and try again with new expression.'))
if self.form.is_valid():
mail = self.form.cleaned_data['email']
else:
mail = 'NONE'
LOGGER.info('Passed captcha for %s (%s = %s)', mail,
self.captcha.question, self.cleaned_data['captcha'])
def clean_captcha(self):
"""Validation for captcha."""
if (self.fresh or
not self.captcha.validate(self.cleaned_data['captcha'])):
raise forms.ValidationError(
_('Please check your math and try again.')
)
mail = self.cleaned_data.get('email', 'NONE')
LOGGER.info(
'Passed captcha for %s (%s = %s)',
mail,
self.captcha.question,
self.cleaned_data['captcha']
)
def get_uri_error(uri):
"""Return error for fetching the URL or None if it works."""
if uri.startswith("https://nonexisting.weblate.org/"):
return "Non existing test URL"
cache_key = f"uri-check-{uri}"
cached = cache.get(cache_key)
if cached:
LOGGER.debug("URL check for %s, cached success", uri)
return None
try:
with request("get", uri, stream=True):
cache.set(cache_key, True, 3600)
LOGGER.debug("URL check for %s, tested success", uri)
return None
except requests.exceptions.RequestException as error:
report_error(cause="URL check failed")
return str(error)
def send_digest(self, language, email, changes, subscription=None):
with override("en" if language is None else language):
context = self.get_context(subscription=subscription, changes=changes)
subject = self.render_template("_subject.txt", context, digest=True)
context["subject"] = subject
LOGGER.info(
"sending digest notification %s on %d changes to %s",
self.get_name(),
len(changes),
email,
)
self.send(
email,
subject,
self.render_template(".html", context, digest=True),
self.get_headers(context),
)
def clean_captcha(self):
"""Validation for CAPTCHA."""
if (self.fresh
or not self.captcha.validate(self.cleaned_data['captcha'])):
self.generate_captcha()
rotate_token(self.request)
raise forms.ValidationError(
# Translators: 'Shown on wrong answer to the mathematics-based CAPTCHA',
_('That was not correct, please try again.'))
if self.form.is_valid():
mail = self.form.cleaned_data['email']
else:
mail = 'NONE'
LOGGER.info('Passed captcha for %s (%s = %s)', mail,
self.captcha.question, self.cleaned_data['captcha'])
def bitbucket_hook_helper(data, request):
"""API to handle service hooks from Bitbucket."""
# Bitbucket ping event
if request and request.META.get('HTTP_X_EVENT_KEY') not in (
'repo:push',
'repo:refs_changed',
):
return None
repository = data['repository']
full_name = bitbucket_extract_full_name(repository)
repo_url = bitbucket_extract_repo_url(data, repository)
# Extract repository links
if 'links' in repository and 'clone' in repository['links']:
repos = [val['href'] for val in repository['links']['clone']]
else:
repo_servers = {'bitbucket.org', urlparse(repo_url).hostname}
repos = []
if 'scm' not in data['repository']:
templates = BITBUCKET_GIT_REPOS + BITBUCKET_HG_REPOS
elif data['repository']['scm'] == 'hg':
templates = BITBUCKET_HG_REPOS
else:
templates = BITBUCKET_GIT_REPOS
# Construct possible repository URLs
for repo in templates:
repos.extend(
(
repo.format(full_name=full_name, server=server)
for server in repo_servers
)
)
if not repos:
LOGGER.error('unsupported repository: %s', repr(data['repository']))
raise ValueError('unsupported repository')
return {
'service_long_name': 'Bitbucket',
'repo_url': repo_url,
'repos': repos,
'branch': bitbucket_extract_branch(data),
'full_name': '{}.git'.format(full_name),
}
def clean_checksum(self):
"""Validate whether checksum is valid and fetches unit for it."""
if 'checksum' not in self.cleaned_data:
return
unit_set = self.translation.unit_set
try:
self.cleaned_data['unit'] = unit_set.filter(
id_hash=self.cleaned_data['checksum']
)[0]
except (Unit.DoesNotExist, IndexError):
LOGGER.error(
'string %s disappeared!', self.cleaned_data['checksum']
)
raise ValidationError(_(
'The string you wanted to translate is no longer available!'
))
def send_digest(self, language, email, changes):
with override('en' if language is None else language):
context = self.get_context()
context['changes'] = changes
subject = self.render_template(
'_subject.txt', context, digest=True
)
context['subject'] = subject
LOGGER.info(
'sending digest notification %s on %d changes to %s',
self.get_name(), len(changes), email,
)
self.send(
email,
subject,
self.render_template('.html', context, digest=True),
self.get_headers(context),
)
def report_error(error, exc_info, request=None, extra_data=None):
"""Wrapper for error reporting
This can be used for store exceptions in error reporting solutions as
rollbar while handling error gracefully and giving user cleaner message.
"""
if HAS_ROLLBAR and hasattr(settings, 'ROLLBAR'):
rollbar.report_exc_info(exc_info, request, extra_data=extra_data)
LOGGER.error(
'Handled exception %s: %s',
error.__class__.__name__,
force_text(error).encode('utf-8')
)
# Print error when running testsuite
if sys.argv[1:2] == ['test']:
traceback.print_exc()
def send_digest(self, language, email, changes):
with override('en' if language is None else language):
context = self.get_context()
context['changes'] = changes
subject = self.render_template(
'_subject.txt', context, digest=True
)
context['subject'] = subject
LOGGER.info(
'sending digest notification %s on %d changes to %s',
self.get_name(), len(changes), email,
)
self.send(
email,
subject,
self.render_template('.html', context, digest=True),
self.get_headers(context),
)
def report_error(error, request=None, extra_data=None):
"""Wrapper for error reporting
This can be used for store exceptions in error reporting solutions as
rollbar while handling error gracefully and giving user cleaner message.
"""
if HAS_ROLLBAR and hasattr(settings, 'ROLLBAR'):
rollbar.report_exc_info(request=request,
extra_data=extra_data,
level='warning')
if HAS_RAVEN and hasattr(settings, 'RAVEN_CONFIG'):
raven_client.captureException(request=request,
extra_data=extra_data,
level='warning')
LOGGER.error('Handled exception %s: %s', error.__class__.__name__,
force_text(error).encode('utf-8'))
def get_notification_emails(language,
recipients,
notification,
context=None,
info=None):
"""Render notification email."""
context = context or {}
headers = {}
LOGGER.info('sending notification %s on %s to %s', notification, info,
', '.join(recipients))
with override('en' if language is None else language):
# Template name
context['subject_template'] = 'mail/{0}_subject.txt'.format(
notification)
context['LANGUAGE_CODE'] = get_language()
context['LANGUAGE_BIDI'] = get_language_bidi()
# Adjust context
context['current_site_url'] = get_site_url()
context['site_title'] = settings.SITE_TITLE
# Render subject
subject = render_to_string(context['subject_template'],
context).strip()
context['subject'] = subject
# Render body
body = render_to_string('mail/{0}.html'.format(notification), context)
# Define headers
headers['Auto-Submitted'] = 'auto-generated'
headers['X-AutoGenerated'] = 'yes'
headers['Precedence'] = 'bulk'
headers['X-Mailer'] = 'Weblate {0}'.format(VERSION)
# Return the mail content
return [{
'subject': subject,
'body': body,
'address': address,
'headers': headers
} for address in recipients]
def get_notification_emails(language,
recipients,
notification,
context=None,
info=None):
"""Render notification email."""
context = context or {}
headers = {}
LOGGER.info("sending notification %s on %s to %s", notification, info,
", ".join(recipients))
with override("en" if language is None else language):
# Template name
context["subject_template"] = "mail/{0}_subject.txt".format(
notification)
context["LANGUAGE_CODE"] = get_language()
context["LANGUAGE_BIDI"] = get_language_bidi()
# Adjust context
context["current_site_url"] = get_site_url()
context["site_title"] = settings.SITE_TITLE
# Render subject
subject = render_to_string(context["subject_template"],
context).strip()
context["subject"] = subject
# Render body
body = render_to_string("mail/{0}.html".format(notification), context)
# Define headers
headers["Auto-Submitted"] = "auto-generated"
headers["X-AutoGenerated"] = "yes"
headers["Precedence"] = "bulk"
headers["X-Mailer"] = USER_AGENT
# Return the mail content
return [{
"subject": subject,
"body": body,
"address": address,
"headers": headers
} for address in recipients]
def send_immediate(
self, language, email, change, extracontext=None, subscription=None
):
with override("en" if language is None else language):
context = self.get_context(change, subscription, extracontext)
subject = self.render_template("_subject.txt", context)
context["subject"] = subject
LOGGER.info(
"sending notification %s on %s to %s",
self.get_name(),
context["component"],
email,
)
self.send(
email,
subject,
self.render_template(".html", context),
self.get_headers(context),
)
def translate(self, language, text, unit, user):
'''
Returns list of machine translations.
'''
language = self.convert_language(language)
if text == '':
return []
if not self.is_supported(language):
return []
try:
translations = self.download_translations(
language, text, unit, user
)
return [
{
'text': trans[0],
'quality': trans[1],
'service': trans[2],
'source': trans[3]
}
for trans in translations
]
except Exception as exc:
report_error(
exc, sys.exc_info(),
{'mt_url': self.request_url, 'mt_params': self.request_params}
)
LOGGER.error(
'Failed to fetch translations from %s',
self.name,
)
LOGGER.error(
'Last fetched URL: %s, params: %s',
self.request_url,
self.request_params,
)
raise MachineTranslationError('{}: {}'.format(
exc.__class__.__name__,
str(exc)
))
def translate(self, language, text, unit, user):
'''
Returns list of machine translations.
'''
language = self.convert_language(language)
if text == '':
return []
if not self.is_supported(language):
return []
try:
translations = self.download_translations(
language, text, unit, user
)
return [
{
'text': trans[0],
'quality': trans[1],
'service': trans[2],
'source': trans[3]
}
for trans in translations
]
except Exception as exc:
report_error(
exc, sys.exc_info(),
{'mt_url': self.request_url, 'mt_params': self.request_params}
)
LOGGER.error(
'Failed to fetch translations from %s',
self.name,
)
LOGGER.error(
'Last fetched URL: %s, params: %s',
self.request_url,
self.request_params,
)
raise MachineTranslationError('{}: {}'.format(
exc.__class__.__name__,
str(exc)
))
def mail_admins_contact(request, subject, message, context, sender):
"""
Sends a message to the admins, as defined by the ADMINS setting.
"""
LOGGER.info("contact form from %s", sender)
if not settings.ADMINS:
messages.error(request, _("Message could not be sent to administrator!"))
LOGGER.error("ADMINS not configured, can not send message!")
return
mail = EmailMultiAlternatives(
"%s%s" % (settings.EMAIL_SUBJECT_PREFIX, subject % context),
message % context,
to=[a[1] for a in settings.ADMINS],
headers={"Reply-To": sender},
)
mail.send(fail_silently=False)
messages.success(request, _("Message has been sent to administrator."))
def clean_checksum(self):
'''
Validates whether checksum is valid and fetches unit for it.
'''
if 'checksum' not in self.cleaned_data:
return
unit_set = self.translation.unit_set
try:
self.cleaned_data['unit'] = unit_set.filter(
checksum=self.cleaned_data['checksum'],
)[0]
except (Unit.DoesNotExist, IndexError):
LOGGER.error(
'message %s disappeared!',
self.cleaned_data['checksum']
)
raise ValidationError(
_('Message you wanted to translate is no longer available!')
)
def clean_captcha(self):
"""Validation for CAPTCHA."""
if (self.fresh or
not self.captcha.validate(self.cleaned_data['captcha'])):
self.generate_captcha()
rotate_token(self.request)
raise forms.ValidationError(
_('Please check your math and try again with new expression.')
)
if self.form.is_valid():
mail = self.form.cleaned_data['email']
else:
mail = 'NONE'
LOGGER.info(
'Passed captcha for %s (%s = %s)',
mail,
self.captcha.question,
self.cleaned_data['captcha']
)
def clean_captcha(self):
'''
Validation for captcha.
'''
if (self.tampering or
not self.captcha.validate(self.cleaned_data['captcha'])):
raise forms.ValidationError(
_('Please check your math and try again.')
)
if 'email' in self.cleaned_data:
mail = self.cleaned_data['email']
else:
mail = 'NONE'
LOGGER.info(
'Passed captcha for %s (%s = %s)',
mail,
self.captcha.question,
self.cleaned_data['captcha']
)
def report_error(error, request=None, extra_data=None):
"""Wrapper for error reporting
This can be used for store exceptions in error reporting solutions as
rollbar while handling error gracefully and giving user cleaner message.
"""
if HAS_ROLLBAR and hasattr(settings, 'ROLLBAR'):
rollbar.report_exc_info(
request=request, extra_data=extra_data, level='warning'
)
if HAS_RAVEN and hasattr(settings, 'RAVEN_CONFIG'):
raven_client.captureException(
request=request, extra=extra_data, level='warning'
)
LOGGER.error(
'Handled exception %s: %s',
error.__class__.__name__,
force_text(error)
)
def supported_languages(self):
'''
Returns list of supported languages.
'''
cache_key = '%s-languages' % self.mtid
# Try using list from cache
languages = cache.get(cache_key)
if languages is not None:
return languages
# Download
try:
languages = self.download_languages()
except Exception as exc:
report_error(
exc, sys.exc_info(),
{'mt_url': self.request_url, 'mt_params': self.request_params}
)
LOGGER.error(
'Failed to fetch languages from %s, using defaults',
self.name,
)
LOGGER.error(
'Last fetched URL: %s, params: %s',
self.request_url,
self.request_params,
)
if settings.DEBUG:
raise
return self.default_languages
# Update cache
cache.set(cache_key, languages, 3600 * 48)
return languages
def execute(self, *args, **options):
"""Wrapper to configure logging prior execution."""
verbosity = int(options['verbosity'])
if verbosity > 1:
LOGGER.setLevel(logging.DEBUG)
elif verbosity == 1:
LOGGER.setLevel(logging.INFO)
else:
LOGGER.setLevel(logging.ERROR)
super(WeblateCommand, self).execute(*args, **options)
def log_info(self, msg, *args):
return LOGGER.info(
': '.join((self.full_slug, msg)), *args
)
def log_warning(self, msg, *args):
return LOGGER.warning(
': '.join((self.full_slug, msg)), *args
)
def vcs_service_hook(request, service):
"""Shared code between VCS service hooks.
Currently used for bitbucket_hook, github_hook and gitlab_hook, but should
be usable for other VCS services (Google Code, custom coded sites, etc.)
too.
"""
# We support only post methods
if not settings.ENABLE_HOOKS:
return HttpResponseNotAllowed(())
# Check if we got payload
try:
data = parse_hook_payload(request)
except (ValueError, KeyError, UnicodeError):
return HttpResponseBadRequest('Could not parse JSON payload!')
# Get service helper
hook_helper = HOOK_HANDLERS[service]
# Send the request data to the service handler.
try:
service_data = hook_helper(data)
except Exception as error:
LOGGER.error('failed to parse service %s data', service)
report_error(error, sys.exc_info())
return HttpResponseBadRequest('Invalid data in json payload!')
# This happens on ping request upon installation
if service_data is None:
return hook_response('Hook working')
# Log data
service_long_name = service_data['service_long_name']
repos = service_data['repos']
repo_url = service_data['repo_url']
branch = service_data['branch']
full_name = service_data['full_name']
# Generate filter
spfilter = Q(repo__in=repos) | Q(repo__iendswith=full_name)
# We need to match also URLs which include username and password
for repo in repos:
if not repo.startswith('https://'):
continue
spfilter = spfilter | (
Q(repo__startswith='https://') &
Q(repo__endswith='@{0}'.format(repo[8:]))
)
all_components = Component.objects.filter(spfilter)
if branch is not None:
all_components = all_components.filter(branch=branch)
components = all_components.filter(project__enable_hooks=True)
LOGGER.info(
'received %s notification on repository %s, branch %s, '
'%d matching components, %d to process',
service_long_name, repo_url, branch,
all_components.count(), components.count(),
)
# Trigger updates
updates = 0
for obj in components:
updates += 1
LOGGER.info(
'%s notification will update %s',
service_long_name,
obj
)
perform_update(obj)
if updates == 0:
return hook_response('No matching repositories found!', 'failure')
return hook_response('Update triggered: {}'.format(
', '.join([obj.log_prefix for obj in components])
))
def vcs_service_hook(request, service):
'''
Shared code between VCS service hooks.
Currently used for bitbucket_hook, github_hook and gitlab_hook, but should
be usable for other VCS services (Google Code, custom coded sites, etc.)
too.
'''
# Check for enabled hooks
if appsettings.ENABLE_HOOKS:
allowed_methods = ('POST',)
else:
allowed_methods = ()
# We support only post methods
if not appsettings.ENABLE_HOOKS or request.method not in allowed_methods:
return HttpResponseNotAllowed(allowed_methods)
# Check if we got payload
try:
# GitLab sends json as application/json
if request.META['CONTENT_TYPE'] == 'application/json':
data = json.loads(request.body)
# Bitbucket and GitHub sends json as x-www-form-data
else:
data = json.loads(request.POST['payload'])
except (ValueError, KeyError):
return HttpResponseBadRequest('Could not parse JSON payload!')
# Get service helper
if service not in HOOK_HANDLERS:
LOGGER.error('service %s is not supported', service)
return HttpResponseBadRequest('invalid service')
hook_helper = HOOK_HANDLERS[service]
# Send the request data to the service handler.
try:
service_data = hook_helper(data)
except KeyError:
LOGGER.error('failed to parse service %s data', service)
return HttpResponseBadRequest('Invalid data in json payload!')
# Log data
service_long_name = service_data['service_long_name']
repos = service_data['repos']
repo_url = service_data['repo_url']
branch = service_data['branch']
LOGGER.info(
'received %s notification on repository %s, branch %s',
service_long_name, repo_url, branch
)
subprojects = SubProject.objects.filter(repo__in=repos)
if branch is not None:
subprojects = subprojects.filter(branch=branch)
# Trigger updates
for obj in subprojects:
if not obj.project.enable_hooks:
continue
LOGGER.info(
'%s notification will update %s',
service_long_name,
obj
)
perform_update(obj)
return hook_response()
def get_notification_email(language, email, notification,
translation_obj=None, context=None, headers=None,
user=None, info=None):
'''
Renders notification email.
'''
cur_language = django_translation.get_language()
context = context or {}
headers = headers or {}
references = None
if 'unit' in context:
unit = context['unit']
references = '{0}/{1}/{2}/{3}'.format(
unit.translation.subproject.project.slug,
unit.translation.subproject.slug,
unit.translation.language.code,
unit.id
)
if references is not None:
references = '<{0}@{1}>'.format(references, get_site_domain())
headers['In-Reply-To'] = references
headers['References'] = references
try:
if info is None:
info = translation_obj.__unicode__()
LOGGER.info(
'sending notification %s on %s to %s',
notification,
info,
email
)
# Load user language
if language is not None:
django_translation.activate(language)
# Template name
context['subject_template'] = 'mail/{}_subject.txt'.format(
notification
)
# Adjust context
context['current_site_url'] = get_site_url()
if translation_obj is not None:
context['translation'] = translation_obj
context['translation_url'] = get_site_url(
translation_obj.get_absolute_url()
)
context['site_title'] = SITE_TITLE
# Render subject
subject = render_to_string(
context['subject_template'],
context
).strip()
# Render body
body = render_to_string(
'mail/{}.txt'.format(notification),
context
)
html_body = render_to_string(
'mail/{}.html'.format(notification),
context
)
# Define headers
headers['Auto-Submitted'] = 'auto-generated'
headers['X-AutoGenerated'] = 'yes'
headers['Precedence'] = 'bulk'
headers['X-Mailer'] = 'Weblate {}'.format(VERSION)
# Reply to header
if user is not None:
headers['Reply-To'] = user.email
# List of recipients
if email == 'ADMINS':
emails = [a[1] for a in settings.ADMINS]
else:
emails = [email]
# Create message
email = EmailMultiAlternatives(
settings.EMAIL_SUBJECT_PREFIX + subject,
body,
to=emails,
headers=headers,
)
email.attach_alternative(
html_body,
'text/html'
)
# Return the mail
return email
finally:
django_translation.activate(cur_language)