def get_avatar_image(request, user, size):
"""Return avatar image from cache (if available) or download it."""
cache_key = '-'.join(('avatar-img', user.username, str(size)))
# Try using avatar specific cache if available
try:
cache = caches['avatar']
except InvalidCacheBackendError:
cache = caches['default']
image = cache.get(cache_key)
if image is None:
try:
image = download_avatar_image(user, size)
cache.set(cache_key, image)
except IOError as error:
report_error(
error,
sys.exc_info(),
request,
extra_data={'avatar': user.username},
level='debug',
)
LOGGER.error('Failed to fetch avatar for %s: %s', user.username,
str(error))
return get_fallback_avatar(size)
return image
def mail_admins_contact(request, subject, message, context, sender, to):
"""Send a message to the admins, as defined by the ADMINS setting."""
LOGGER.info(
'contact form from %s',
sender,
)
if not to and settings.ADMINS:
to = [a[1] for a in settings.ADMINS]
elif not settings.ADMINS:
messages.error(
request,
_('Message could not be sent to administrator!')
)
LOGGER.error(
'ADMINS not configured, can not send message!'
)
return
mail = EmailMultiAlternatives(
'{0}{1}'.format(settings.EMAIL_SUBJECT_PREFIX, subject % context),
message % context,
to=to,
headers={'Reply-To': sender},
)
mail.send(fail_silently=False)
messages.success(
request,
_('Message has been sent to administrator.')
)
def mail_admins_contact(request, subject, message, context, sender, to):
"""Send a message to the admins, as defined by the ADMINS setting."""
LOGGER.info('contact form from %s', sender)
if not to and settings.ADMINS:
to = [a[1] for a in settings.ADMINS]
elif not settings.ADMINS:
messages.error(request,
_('Message could not be sent to administrator!'))
LOGGER.error('ADMINS not configured, can not send message!')
return
mail = EmailMultiAlternatives(
'{0}{1}'.format(settings.EMAIL_SUBJECT_PREFIX, subject % context),
'{}\n{}'.format(
message % context,
TEMPLATE_FOOTER.format(
address=get_ip_address(request),
agent=get_user_agent(request),
username=request.user.username,
),
),
to=to,
headers={'Reply-To': sender},
)
mail.send(fail_silently=False)
messages.success(request, _('Message has been sent to administrator.'))
def vcs_service_hook(request, service):
'''
Shared code between VCS service hooks.
Currently used for bitbucket_hook, github_hook and gitlab_hook, but should
be usable for other VCS services (Google Code, custom coded sites, etc.)
too.
'''
# We support only post methods
if not appsettings.ENABLE_HOOKS:
return HttpResponseNotAllowed(())
# Check if we got payload
try:
# GitLab sends json as application/json
if request.META['CONTENT_TYPE'] == 'application/json':
data = json.loads(request.body.decode('utf-8'))
# Bitbucket and GitHub sends json as x-www-form-data
else:
data = json.loads(request.POST['payload'])
except (ValueError, KeyError, UnicodeError):
return HttpResponseBadRequest('Could not parse JSON payload!')
# Get service helper
hook_helper = HOOK_HANDLERS[service]
# Send the request data to the service handler.
try:
service_data = hook_helper(data)
except KeyError:
LOGGER.error('failed to parse service %s data', service)
return HttpResponseBadRequest('Invalid data in json payload!')
# Log data
service_long_name = service_data['service_long_name']
repos = service_data['repos']
repo_url = service_data['repo_url']
branch = service_data['branch']
LOGGER.info('received %s notification on repository %s, branch %s',
service_long_name, repo_url, branch)
subprojects = SubProject.objects.filter(repo__in=repos)
if branch is not None:
subprojects = subprojects.filter(branch=branch)
# Trigger updates
updates = 0
for obj in subprojects:
if not obj.project.enable_hooks:
continue
updates += 1
LOGGER.info('%s notification will update %s', service_long_name, obj)
perform_update(obj)
if updates == 0:
return hook_response('No matching repositories found!', 'failure')
return hook_response()
def get_plural_type(code, pluralequation):
"""Get correct plural type for language."""
# Remove not needed parenthesis
if pluralequation[-1] == ';':
pluralequation = pluralequation[:-1]
# Get base language code
base_code = code.replace('_', '-').split('-')[0]
# No plural
if pluralequation == '0':
return data.PLURAL_NONE
# Standard plural equations
for mapping in data.PLURAL_MAPPINGS:
if pluralequation in mapping[0]:
return mapping[1]
# Arabic special case
if base_code in ('ar',):
return data.PLURAL_ARABIC
# Log error in case of uknown mapping
LOGGER.error(
'Can not guess type of plural for %s: %s', code, pluralequation
)
return data.PLURAL_UNKNOWN
def report_error(self, exc, request, message):
"""Wrapper for handling error situations"""
extra = {'mt_url': self.request_url, 'mt_params': self.request_params}
report_error(exc, request, extra)
LOGGER.error(message, self.name)
LOGGER.info('Last URL: %s, params: %s', extra['mt_url'],
extra['mt_params'])
def get_plural_type(base_code, pluralequation):
"""Get correct plural type for language."""
# Remove not needed parenthesis
if pluralequation[-1] == ';':
pluralequation = pluralequation[:-1]
# No plural
if pluralequation == '0':
return data.PLURAL_NONE
# Standard plural equations
for mapping in data.PLURAL_MAPPINGS:
if pluralequation in mapping[0]:
return mapping[1]
# Arabic special case
if base_code in ('ar',):
return data.PLURAL_ARABIC
# Log error in case of uknown mapping
LOGGER.error(
'Can not guess type of plural for %s: %s', base_code, pluralequation
)
return data.PLURAL_UNKNOWN
def get_plural_type(code, pluralequation):
'''
Gets correct plural type for language.
'''
# Remove not needed parenthesis
if pluralequation[-1] == ';':
pluralequation = pluralequation[:-1]
if pluralequation[0] == '(' and pluralequation[-1] == ')':
pluralequation = pluralequation[1:-1]
# Get base language code
base_code = code.replace('_', '-').split('-')[0]
# No plural
if pluralequation == '0':
return data.PLURAL_NONE
# Standard plural equations
for mapping in data.PLURAL_MAPPINGS:
if pluralequation in mapping[0]:
return mapping[1]
# Arabic special case
if base_code in ('ar',):
return data.PLURAL_ARABIC
# Log error in case of uknown mapping
LOGGER.error(
'Can not guess type of plural for %s: %s', code, pluralequation
)
return data.PLURAL_UNKNOWN
def get_plural_type(base_code, plural_formula):
"""Get correct plural type for language."""
# Remove not needed parenthesis
if plural_formula[-1] == ";":
plural_formula = plural_formula[:-1]
# No plural
if plural_formula == "0":
return data.PLURAL_NONE
# Remove whitespace
formula = plural_formula.replace(" ", "")
# Standard plural formulas
for mapping in data.PLURAL_MAPPINGS:
if formula in mapping[0]:
return mapping[1]
# Arabic special case
if base_code in ("ar", ):
return data.PLURAL_ARABIC
# Log error in case of uknown mapping
LOGGER.error("Can not guess type of plural for %s: %s", base_code,
plural_formula)
return data.PLURAL_UNKNOWN
def get_avatar_image(request, user, size):
"""Return avatar image from cache (if available) or download it."""
cache_key = '-'.join((
'avatar-img',
user.username,
str(size)
))
# Try using avatar specific cache if available
try:
cache = caches['avatar']
except InvalidCacheBackendError:
cache = caches['default']
image = cache.get(cache_key)
if image is None:
try:
image = download_avatar_image(user, size)
cache.set(cache_key, image)
except (IOError, CertificateError) as error:
report_error(
error, sys.exc_info(), request,
extra_data={'avatar': user.username},
level='debug',
)
LOGGER.error(
'Failed to fetch avatar for %s: %s',
user.username,
str(error)
)
return get_fallback_avatar(size)
return image
def bitbucket_hook_helper(data):
'''
API to handle service hooks from Bitbucket.
'''
if 'push' in data:
return bitbucket_webhook_helper(data)
# Parse owner, branch and repository name
owner = data['repository']['owner']
slug = data['repository']['slug']
if data['commits']:
branch = data['commits'][-1]['branch']
else:
branch = None
params = {'owner': owner, 'slug': slug}
# Construct possible repository URLs
if data['repository']['scm'] == 'git':
repos = [repo % params for repo in BITBUCKET_GIT_REPOS]
elif data['repository']['scm'] == 'hg':
repos = [repo % params for repo in BITBUCKET_HG_REPOS]
else:
LOGGER.error('unsupported repository: %s', repr(data['repository']))
raise ValueError('unsupported repository')
return {
'service_long_name':
'Bitbucket',
'repo_url':
''.join([data['canon_url'], data['repository']['absolute_url']]),
'repos':
repos,
'branch':
branch,
}
def mail_admins_contact(request, subject, message, context, sender):
"""Send a message to the admins, as defined by the ADMINS setting."""
LOGGER.info(
'contact form from %s',
sender,
)
if not settings.ADMINS:
messages.error(
request,
_('Message could not be sent to administrator!')
)
LOGGER.error(
'ADMINS not configured, can not send message!'
)
return
mail = EmailMultiAlternatives(
'{0}{1}'.format(settings.EMAIL_SUBJECT_PREFIX, subject % context),
message % context,
to=[a[1] for a in settings.ADMINS],
headers={'Reply-To': sender},
)
mail.send(fail_silently=False)
messages.success(
request,
_('Message has been sent to administrator.')
)
def report_error(
error,
request=None,
extra_data=None,
level='warning',
prefix='Handled exception',
skip_sentry=False,
print_tb=False,
):
"""Wrapper for error reporting
This can be used for store exceptions in error reporting solutions as
rollbar while handling error gracefully and giving user cleaner message.
"""
if HAS_ROLLBAR and hasattr(settings, 'ROLLBAR'):
rollbar.report_exc_info(request=request,
extra_data=extra_data,
level=level)
if not skip_sentry and settings.SENTRY_DSN:
sentry_sdk.capture_exception()
LOGGER.error('%s: %s: %s', prefix, error.__class__.__name__,
force_text(error))
if print_tb:
LOGGER.exception(prefix)
def get_avatar_image(user, size):
"""
Returns avatar image from cache (if available) or downloads it.
"""
cache_key = 'avatar-img-{0}-{1}'.format(
user.username,
size
)
# Try using avatar specific cache if available
try:
cache = caches['avatar']
except InvalidCacheBackendError:
cache = caches['default']
image = cache.get(cache_key)
if image is None:
try:
image = download_avatar_image(user, size)
cache.set(cache_key, image)
except IOError as error:
report_error(
error, sys.exc_info(),
extra_data={'avatar': user.username}
)
LOGGER.error(
'Failed to fetch avatar for %s: %s',
user.username,
str(error)
)
return get_fallback_avatar(size)
return image
def mail_admins_contact(request, subject, message, context, sender, to):
"""Send a message to the admins, as defined by the ADMINS setting."""
LOGGER.info("contact form from %s", sender)
if not to and settings.ADMINS:
to = [a[1] for a in settings.ADMINS]
elif not settings.ADMINS:
messages.error(request, _("Could not send message to administrator."))
LOGGER.error("ADMINS not configured, cannot send message")
return
mail = EmailMultiAlternatives(
"{}{}".format(settings.EMAIL_SUBJECT_PREFIX, subject % context),
"{}\n{}".format(
message % context,
TEMPLATE_FOOTER.format(
address=get_ip_address(request),
agent=get_user_agent(request),
username=request.user.username,
),
),
to=to,
headers={"Reply-To": sender},
)
mail.send(fail_silently=False)
messages.success(
request,
_("Your request has been sent, you will shortly hear from us."))
def bitbucket_hook_helper(data):
"""API to handle service hooks from Bitbucket."""
if 'push' in data:
return bitbucket_webhook_helper(data)
# Parse owner, branch and repository name
owner = data['repository']['owner']
slug = data['repository']['slug']
if data['commits']:
branch = data['commits'][-1]['branch']
else:
branch = None
params = {'owner': owner, 'slug': slug}
# Construct possible repository URLs
if data['repository']['scm'] == 'git':
repos = [repo % params for repo in BITBUCKET_GIT_REPOS]
elif data['repository']['scm'] == 'hg':
repos = [repo % params for repo in BITBUCKET_HG_REPOS]
else:
LOGGER.error(
'unsupported repository: %s',
repr(data['repository'])
)
raise ValueError('unsupported repository')
return {
'service_long_name': 'Bitbucket',
'repo_url': ''.join([
data['canon_url'], data['repository']['absolute_url']
]),
'repos': repos,
'branch': branch,
}
def send_mails(mails):
"""Sends multiple mails in single connection."""
try:
connection = get_connection()
connection.send_messages([mail for mail in mails if mail is not None])
except SMTPException as error:
LOGGER.error('Failed to send email: %s', error)
report_error(error, sys.exc_info())
def send_mails(mails):
"""Sends multiple mails in single connection."""
try:
connection = get_connection()
connection.send_messages(mails)
except SMTPException as error:
LOGGER.error('Failed to send email: %s', error)
report_error(error, sys.exc_info())
def report_error(self, exc, request, message):
"""Wrapper for handling error situations"""
extra = {'mt_url': self.request_url, 'mt_params': self.request_params}
report_error(exc, request, extra, prefix='Machine translation error')
LOGGER.error(message, self.name)
LOGGER.info(
'Last URL: %s, params: %s', extra['mt_url'], extra['mt_params']
)
def report_error(error, exc_info, request=None, extra_data=None):
"""Wrapper for error reporting
This can be used for store exceptions in error reporting solutions as
rollbar while handling error gracefully and giving user cleaner message.
"""
if HAS_ROLLBAR and hasattr(settings, 'ROLLBAR'):
rollbar.report_exc_info(exc_info, request, extra_data=extra_data)
LOGGER.error(
'Handled exception %s: %s',
error.__class__.__name__,
str(error)
)
def bitbucket_hook_helper(data, request):
"""API to handle service hooks from Bitbucket."""
# Bitbucket ping event
if request and request.META.get("HTTP_X_EVENT_KEY") not in (
"repo:push",
"repo:refs_changed",
"pullrequest:fulfilled",
"pr:merged",
):
return None
if "pullRequest" in data:
# The pr:merged event
repository = data["pullRequest"]["fromRef"]["repository"]
else:
repository = data["repository"]
full_name = bitbucket_extract_full_name(repository)
repo_url = bitbucket_extract_repo_url(data, repository)
# Extract repository links
if "links" in repository and "clone" in repository["links"]:
repos = [val["href"] for val in repository["links"]["clone"]]
else:
repo_servers = {"bitbucket.org", urlparse(repo_url).hostname}
repos = []
if "scm" not in data["repository"]:
templates = BITBUCKET_GIT_REPOS + BITBUCKET_HG_REPOS
elif data["repository"]["scm"] == "hg":
templates = BITBUCKET_HG_REPOS
else:
templates = BITBUCKET_GIT_REPOS
# Construct possible repository URLs
for repo in templates:
repos.extend(
repo.format(full_name=full_name, server=server)
for server in repo_servers
)
if not repos:
LOGGER.error("unsupported repository: %s", repr(data["repository"]))
raise ValueError("unsupported repository")
return {
"service_long_name": "Bitbucket",
"repo_url": repo_url,
"repos": repos,
"branch": bitbucket_extract_branch(data),
"full_name": f"{full_name}.git",
}
def report_error(self, exc, message):
"""Wrapper for handling error situations"""
report_error(
exc, sys.exc_info(),
{'mt_url': self.request_url, 'mt_params': self.request_params}
)
LOGGER.error(
message,
self.name,
)
LOGGER.error(
'Last fetched URL: %s, params: %s',
self.request_url,
self.request_params,
)
def clean_checksum(self):
"""Validate whether checksum is valid and fetches unit for it."""
if 'checksum' not in self.cleaned_data:
return
unit_set = self.translation.unit_set
try:
self.cleaned_data['unit'] = unit_set.filter(
id_hash=self.cleaned_data['checksum'])[0]
except (Unit.DoesNotExist, IndexError):
LOGGER.error('message %s disappeared!',
self.cleaned_data['checksum'])
raise ValidationError(
_('Message you wanted to translate is no longer available!'))
def report_error(error, exc_info, request=None, extra_data=None):
"""Wrapper for error reporting
This can be used for store exceptions in error reporting solutions as
rollbar while handling error gracefully and giving user cleaner message.
"""
if HAS_ROLLBAR and hasattr(settings, 'ROLLBAR'):
rollbar.report_exc_info(exc_info, request, extra_data=extra_data)
LOGGER.error('Handled exception %s: %s', error.__class__.__name__,
force_text(error).encode('utf-8'))
# Print error when running testsuite
if sys.argv[1:2] == ['test']:
traceback.print_exc()
def report_error(self, exc, message):
"""Wrapper for handling error situations"""
report_error(
exc, sys.exc_info(),
{'mt_url': self.request_url, 'mt_params': self.request_params}
)
LOGGER.error(
message,
self.name,
)
LOGGER.error(
'Last fetched URL: %s, params: %s',
self.request_url,
self.request_params,
)
def bitbucket_hook_helper(data, request):
"""API to handle service hooks from Bitbucket."""
# Bitbucket ping event
if request and request.META.get('HTTP_X_EVENT_KEY') not in (
'repo:push',
'repo:refs_changed',
):
return None
repository = data['repository']
full_name = bitbucket_extract_full_name(repository)
repo_url = bitbucket_extract_repo_url(data, repository)
# Extract repository links
if 'links' in repository and 'clone' in repository['links']:
repos = [val['href'] for val in repository['links']['clone']]
else:
repo_servers = {'bitbucket.org', urlparse(repo_url).hostname}
repos = []
if 'scm' not in data['repository']:
templates = BITBUCKET_GIT_REPOS + BITBUCKET_HG_REPOS
elif data['repository']['scm'] == 'hg':
templates = BITBUCKET_HG_REPOS
else:
templates = BITBUCKET_GIT_REPOS
# Construct possible repository URLs
for repo in templates:
repos.extend(
(
repo.format(full_name=full_name, server=server)
for server in repo_servers
)
)
if not repos:
LOGGER.error('unsupported repository: %s', repr(data['repository']))
raise ValueError('unsupported repository')
return {
'service_long_name': 'Bitbucket',
'repo_url': repo_url,
'repos': repos,
'branch': bitbucket_extract_branch(data),
'full_name': '{}.git'.format(full_name),
}
def report_error(error, exc_info, request=None, extra_data=None):
"""Wrapper for error reporting
This can be used for store exceptions in error reporting solutions as
rollbar while handling error gracefully and giving user cleaner message.
"""
if HAS_ROLLBAR and hasattr(settings, 'ROLLBAR'):
rollbar.report_exc_info(exc_info, request, extra_data=extra_data)
LOGGER.error(
'Handled exception %s: %s',
error.__class__.__name__,
force_text(error).encode('utf-8')
)
# Print error when running testsuite
if sys.argv[1:2] == ['test']:
traceback.print_exc()
def report_error(error, request=None, extra_data=None):
"""Wrapper for error reporting
This can be used for store exceptions in error reporting solutions as
rollbar while handling error gracefully and giving user cleaner message.
"""
if HAS_ROLLBAR and hasattr(settings, 'ROLLBAR'):
rollbar.report_exc_info(request=request,
extra_data=extra_data,
level='warning')
if HAS_RAVEN and hasattr(settings, 'RAVEN_CONFIG'):
raven_client.captureException(request=request,
extra=extra_data,
level='warning')
LOGGER.error('Handled exception %s: %s', error.__class__.__name__,
force_text(error))
def clean_checksum(self):
"""Validate whether checksum is valid and fetches unit for it."""
if 'checksum' not in self.cleaned_data:
return
unit_set = self.translation.unit_set
try:
self.cleaned_data['unit'] = unit_set.filter(
id_hash=self.cleaned_data['checksum']
)[0]
except (Unit.DoesNotExist, IndexError):
LOGGER.error(
'string %s disappeared!', self.cleaned_data['checksum']
)
raise ValidationError(_(
'The string you wanted to translate is no longer available!'
))
def translate(self, language, text, unit, user):
'''
Returns list of machine translations.
'''
language = self.convert_language(language)
if text == '':
return []
if not self.is_supported(language):
return []
try:
translations = self.download_translations(
language, text, unit, user
)
return [
{
'text': trans[0],
'quality': trans[1],
'service': trans[2],
'source': trans[3]
}
for trans in translations
]
except Exception as exc:
report_error(
exc, sys.exc_info(),
{'mt_url': self.request_url, 'mt_params': self.request_params}
)
LOGGER.error(
'Failed to fetch translations from %s',
self.name,
)
LOGGER.error(
'Last fetched URL: %s, params: %s',
self.request_url,
self.request_params,
)
raise MachineTranslationError('{}: {}'.format(
exc.__class__.__name__,
str(exc)
))
def translate(self, language, text, unit, user):
'''
Returns list of machine translations.
'''
language = self.convert_language(language)
if text == '':
return []
if not self.is_supported(language):
return []
try:
translations = self.download_translations(
language, text, unit, user
)
return [
{
'text': trans[0],
'quality': trans[1],
'service': trans[2],
'source': trans[3]
}
for trans in translations
]
except Exception as exc:
report_error(
exc, sys.exc_info(),
{'mt_url': self.request_url, 'mt_params': self.request_params}
)
LOGGER.error(
'Failed to fetch translations from %s',
self.name,
)
LOGGER.error(
'Last fetched URL: %s, params: %s',
self.request_url,
self.request_params,
)
raise MachineTranslationError('{}: {}'.format(
exc.__class__.__name__,
str(exc)
))
def mail_admins_contact(request, subject, message, context, sender):
"""
Sends a message to the admins, as defined by the ADMINS setting.
"""
LOGGER.info("contact form from %s", sender)
if not settings.ADMINS:
messages.error(request, _("Message could not be sent to administrator!"))
LOGGER.error("ADMINS not configured, can not send message!")
return
mail = EmailMultiAlternatives(
"%s%s" % (settings.EMAIL_SUBJECT_PREFIX, subject % context),
message % context,
to=[a[1] for a in settings.ADMINS],
headers={"Reply-To": sender},
)
mail.send(fail_silently=False)
messages.success(request, _("Message has been sent to administrator."))
def report_error(error, request=None, extra_data=None):
"""Wrapper for error reporting
This can be used for store exceptions in error reporting solutions as
rollbar while handling error gracefully and giving user cleaner message.
"""
if HAS_ROLLBAR and hasattr(settings, 'ROLLBAR'):
rollbar.report_exc_info(
request=request, extra_data=extra_data, level='warning'
)
if HAS_RAVEN and hasattr(settings, 'RAVEN_CONFIG'):
raven_client.captureException(
request=request, extra=extra_data, level='warning'
)
LOGGER.error(
'Handled exception %s: %s',
error.__class__.__name__,
force_text(error)
)
def clean_checksum(self):
'''
Validates whether checksum is valid and fetches unit for it.
'''
if 'checksum' not in self.cleaned_data:
return
unit_set = self.translation.unit_set
try:
self.cleaned_data['unit'] = unit_set.filter(
checksum=self.cleaned_data['checksum'],
)[0]
except (Unit.DoesNotExist, IndexError):
LOGGER.error(
'message %s disappeared!',
self.cleaned_data['checksum']
)
raise ValidationError(
_('Message you wanted to translate is no longer available!')
)
def supported_languages(self):
'''
Returns list of supported languages.
'''
cache_key = '%s-languages' % self.mtid
# Try using list from cache
languages = cache.get(cache_key)
if languages is not None:
return languages
# Download
try:
languages = self.download_languages()
except Exception as exc:
report_error(
exc, sys.exc_info(),
{'mt_url': self.request_url, 'mt_params': self.request_params}
)
LOGGER.error(
'Failed to fetch languages from %s, using defaults',
self.name,
)
LOGGER.error(
'Last fetched URL: %s, params: %s',
self.request_url,
self.request_params,
)
if settings.DEBUG:
raise
return self.default_languages
# Update cache
cache.set(cache_key, languages, 3600 * 48)
return languages
def supported_languages(self):
'''
Returns list of supported languages.
'''
cache_key = '%s-languages' % self.mtid
# Try using list from cache
languages = cache.get(cache_key)
if languages is not None:
return languages
# Download
try:
languages = self.download_languages()
except Exception as exc:
report_error(
exc, sys.exc_info(),
{'mt_url': self.request_url, 'mt_params': self.request_params}
)
LOGGER.error(
'Failed to fetch languages from %s, using defaults',
self.name,
)
LOGGER.error(
'Last fetched URL: %s, params: %s',
self.request_url,
self.request_params,
)
if settings.DEBUG:
raise
return self.default_languages
# Update cache
cache.set(cache_key, languages, 3600 * 48)
return languages
def log_error(self, msg, *args):
self.log_hook('ERROR', msg, *args)
return LOGGER.error(': '.join((self.full_slug, msg)), *args)
def log_error(self, msg, *args):
return LOGGER.error(': '.join((self.full_slug, msg)), *args)
def vcs_service_hook(request, service):
"""Shared code between VCS service hooks.
Currently used for bitbucket_hook, github_hook and gitlab_hook, but should
be usable for other VCS services (Google Code, custom coded sites, etc.)
too.
"""
# We support only post methods
if not settings.ENABLE_HOOKS:
return HttpResponseNotAllowed(())
# Check if we got payload
try:
data = parse_hook_payload(request)
except (ValueError, KeyError, UnicodeError):
return HttpResponseBadRequest('Could not parse JSON payload!')
# Get service helper
hook_helper = HOOK_HANDLERS[service]
# Send the request data to the service handler.
try:
service_data = hook_helper(data)
except Exception as error:
LOGGER.error('failed to parse service %s data', service)
report_error(error, sys.exc_info())
return HttpResponseBadRequest('Invalid data in json payload!')
# Log data
service_long_name = service_data['service_long_name']
repos = service_data['repos']
repo_url = service_data['repo_url']
branch = service_data['branch']
# Generate filter
spfilter = Q(repo__in=repos)
# We need to match also URLs which include username and password
for repo in repos:
if not repo.startswith('https://'):
continue
spfilter = spfilter | (Q(repo__startswith='https://')
& Q(repo__endswith='@{0}'.format(repo[8:])))
all_subprojects = SubProject.objects.filter(spfilter)
if branch is not None:
all_subprojects = all_subprojects.filter(branch=branch)
subprojects = all_subprojects.filter(project__enable_hooks=True)
LOGGER.info(
'received %s notification on repository %s, branch %s,'
'%d matching components, %d to process',
service_long_name,
repo_url,
branch,
all_subprojects.count(),
subprojects.count(),
)
# Trigger updates
updates = 0
for obj in subprojects:
updates += 1
LOGGER.info('%s notification will update %s', service_long_name, obj)
perform_update(obj)
if updates == 0:
return hook_response('No matching repositories found!', 'failure')
return hook_response()
def log_error(self, msg, *args):
return LOGGER.error(self.log_prefix + msg, *args)
def log_error(self, msg, *args):
self.log_hook('ERROR', msg, *args)
return LOGGER.error(
': '.join((self.full_slug, msg)), *args
)
def log_error(self, msg, *args):
return LOGGER.error(
self.log_prefix + msg, *args
)
def report_error(self, message):
"""Wrapper for handling error situations."""
report_error(cause="Machinery error")
LOGGER.error(message, self.name)
def vcs_service_hook(request, service):
"""Shared code between VCS service hooks.
Currently used for bitbucket_hook, github_hook and gitlab_hook, but should
be usable for other VCS services (Google Code, custom coded sites, etc.)
too.
"""
# We support only post methods
if not settings.ENABLE_HOOKS:
return HttpResponseNotAllowed(())
# Check if we got payload
try:
data = parse_hook_payload(request)
except (ValueError, KeyError, UnicodeError):
return HttpResponseBadRequest('Could not parse JSON payload!')
# Get service helper
hook_helper = HOOK_HANDLERS[service]
# Send the request data to the service handler.
try:
service_data = hook_helper(data)
except Exception as error:
LOGGER.error('failed to parse service %s data', service)
report_error(error, sys.exc_info())
return HttpResponseBadRequest('Invalid data in json payload!')
# This happens on ping request upon installation
if service_data is None:
return hook_response('Hook working')
# Log data
service_long_name = service_data['service_long_name']
repos = service_data['repos']
repo_url = service_data['repo_url']
branch = service_data['branch']
full_name = service_data['full_name']
# Generate filter
spfilter = Q(repo__in=repos) | Q(repo__iendswith=full_name)
# We need to match also URLs which include username and password
for repo in repos:
if not repo.startswith('https://'):
continue
spfilter = spfilter | (
Q(repo__startswith='https://') &
Q(repo__endswith='@{0}'.format(repo[8:]))
)
all_components = Component.objects.filter(spfilter)
if branch is not None:
all_components = all_components.filter(branch=branch)
components = all_components.filter(project__enable_hooks=True)
LOGGER.info(
'received %s notification on repository %s, branch %s, '
'%d matching components, %d to process',
service_long_name, repo_url, branch,
all_components.count(), components.count(),
)
# Trigger updates
updates = 0
for obj in components:
updates += 1
LOGGER.info(
'%s notification will update %s',
service_long_name,
obj
)
perform_update(obj)
if updates == 0:
return hook_response('No matching repositories found!', 'failure')
return hook_response('Update triggered: {}'.format(
', '.join([obj.log_prefix for obj in components])
))
def log_error(self, msg, *args):
return LOGGER.error(
': '.join((self.full_slug, msg)), *args
)
def log_error(self, msg, *args):
self.log_hook("ERROR", msg, *args)
return LOGGER.error(": ".join((self.full_slug, msg)), *args)
def log_error(self, msg, *args):
return LOGGER.error(
': '.join((self.log_prefix, msg)), *args
)
def report_error(self, exc, message):
"""Wrapper for handling error situations."""
report_error(exc, prefix='Machinery error')
LOGGER.error(message, self.name)
def report_error(self, exc, message):
"""Wrapper for handling error situations"""
report_error(exc, prefix='Machinery error')
LOGGER.error(message, self.name)
LOGGER.info('Last URL: %s, params: %s', self.request_url,
self.request_params)
def vcs_service_hook(request, service):
"""Shared code between VCS service hooks.
Currently used for bitbucket_hook, github_hook and gitlab_hook, but should be usable
for other VCS services (Google Code, custom coded sites, etc.) too.
"""
# We support only post methods
if not settings.ENABLE_HOOKS:
return HttpResponseNotAllowed(())
# Check if we got payload
try:
data = parse_hook_payload(request)
except (ValueError, KeyError):
return HttpResponseBadRequest("Could not parse JSON payload!")
if not data:
return HttpResponseBadRequest("Invalid data in json payload!")
# Get service helper
hook_helper = HOOK_HANDLERS[service]
# Send the request data to the service handler.
try:
service_data = hook_helper(data, request)
except Exception as error:
LOGGER.error("failed to parse service %s data", service)
report_error(error, request)
return HttpResponseBadRequest("Invalid data in json payload!")
# This happens on ping request upon installation
if service_data is None:
return hook_response("Hook working", status=201)
# Log data
service_long_name = service_data["service_long_name"]
repos = service_data["repos"]
repo_url = service_data["repo_url"]
branch = service_data["branch"]
full_name = service_data["full_name"]
# Generate filter
spfilter = Q(repo__in=repos) | Q(repo__iendswith=full_name)
# We need to match also URLs which include username and password
for repo in repos:
if repo.startswith("http://"):
spfilter = spfilter | (
Q(repo__startswith="http://")
& Q(repo__endswith="@{0}".format(repo[7:]))
)
elif repo.startswith("https://"):
spfilter = spfilter | (
Q(repo__startswith="https://")
& Q(repo__endswith="@{0}".format(repo[8:]))
)
all_components = Component.objects.filter(spfilter)
if branch is not None:
all_components = all_components.filter(branch=branch)
components = all_components.filter(project__enable_hooks=True)
LOGGER.info(
"received %s notification on repository %s, branch %s, "
"%d matching components, %d to process",
service_long_name,
repo_url,
branch,
all_components.count(),
components.count(),
)
# Trigger updates
updates = 0
for obj in components:
updates += 1
LOGGER.info("%s notification will update %s", service_long_name, obj)
Change.objects.create(
component=obj, action=Change.ACTION_HOOK, details=service_data
)
perform_update.delay("Component", obj.pk)
if updates == 0:
return hook_response("No matching repositories found!", "failure", status=202)
return hook_response(
"Update triggered: {}".format(", ".join(obj.full_slug for obj in components))
)
def log_error(self, msg, *args):
return LOGGER.error(": ".join((self.log_prefix, msg)), *args)
def vcs_service_hook(request, service):
'''
Shared code between VCS service hooks.
Currently used for bitbucket_hook, github_hook and gitlab_hook, but should
be usable for other VCS services (Google Code, custom coded sites, etc.)
too.
'''
# Check for enabled hooks
if appsettings.ENABLE_HOOKS:
allowed_methods = ('POST',)
else:
allowed_methods = ()
# We support only post methods
if not appsettings.ENABLE_HOOKS or request.method not in allowed_methods:
return HttpResponseNotAllowed(allowed_methods)
# Check if we got payload
try:
# GitLab sends json as application/json
if request.META['CONTENT_TYPE'] == 'application/json':
data = json.loads(request.body)
# Bitbucket and GitHub sends json as x-www-form-data
else:
data = json.loads(request.POST['payload'])
except (ValueError, KeyError):
return HttpResponseBadRequest('Could not parse JSON payload!')
# Get service helper
if service not in HOOK_HANDLERS:
LOGGER.error('service %s is not supported', service)
return HttpResponseBadRequest('invalid service')
hook_helper = HOOK_HANDLERS[service]
# Send the request data to the service handler.
try:
service_data = hook_helper(data)
except KeyError:
LOGGER.error('failed to parse service %s data', service)
return HttpResponseBadRequest('Invalid data in json payload!')
# Log data
service_long_name = service_data['service_long_name']
repos = service_data['repos']
repo_url = service_data['repo_url']
branch = service_data['branch']
LOGGER.info(
'received %s notification on repository %s, branch %s',
service_long_name, repo_url, branch
)
subprojects = SubProject.objects.filter(repo__in=repos)
if branch is not None:
subprojects = subprojects.filter(branch=branch)
# Trigger updates
for obj in subprojects:
if not obj.project.enable_hooks:
continue
LOGGER.info(
'%s notification will update %s',
service_long_name,
obj
)
perform_update(obj)
return hook_response()
