def get_avatar_image(request, user, size): """Return avatar image from cache (if available) or download it.""" cache_key = '-'.join(('avatar-img', user.username, str(size))) # Try using avatar specific cache if available try: cache = caches['avatar'] except InvalidCacheBackendError: cache = caches['default'] image = cache.get(cache_key) if image is None: try: image = download_avatar_image(user, size) cache.set(cache_key, image) except IOError as error: report_error( error, sys.exc_info(), request, extra_data={'avatar': user.username}, level='debug', ) LOGGER.error('Failed to fetch avatar for %s: %s', user.username, str(error)) return get_fallback_avatar(size) return image
def mail_admins_contact(request, subject, message, context, sender, to): """Send a message to the admins, as defined by the ADMINS setting.""" LOGGER.info( 'contact form from %s', sender, ) if not to and settings.ADMINS: to = [a[1] for a in settings.ADMINS] elif not settings.ADMINS: messages.error( request, _('Message could not be sent to administrator!') ) LOGGER.error( 'ADMINS not configured, can not send message!' ) return mail = EmailMultiAlternatives( '{0}{1}'.format(settings.EMAIL_SUBJECT_PREFIX, subject % context), message % context, to=to, headers={'Reply-To': sender}, ) mail.send(fail_silently=False) messages.success( request, _('Message has been sent to administrator.') )
def mail_admins_contact(request, subject, message, context, sender, to): """Send a message to the admins, as defined by the ADMINS setting.""" LOGGER.info('contact form from %s', sender) if not to and settings.ADMINS: to = [a[1] for a in settings.ADMINS] elif not settings.ADMINS: messages.error(request, _('Message could not be sent to administrator!')) LOGGER.error('ADMINS not configured, can not send message!') return mail = EmailMultiAlternatives( '{0}{1}'.format(settings.EMAIL_SUBJECT_PREFIX, subject % context), '{}\n{}'.format( message % context, TEMPLATE_FOOTER.format( address=get_ip_address(request), agent=get_user_agent(request), username=request.user.username, ), ), to=to, headers={'Reply-To': sender}, ) mail.send(fail_silently=False) messages.success(request, _('Message has been sent to administrator.'))
def vcs_service_hook(request, service): ''' Shared code between VCS service hooks. Currently used for bitbucket_hook, github_hook and gitlab_hook, but should be usable for other VCS services (Google Code, custom coded sites, etc.) too. ''' # We support only post methods if not appsettings.ENABLE_HOOKS: return HttpResponseNotAllowed(()) # Check if we got payload try: # GitLab sends json as application/json if request.META['CONTENT_TYPE'] == 'application/json': data = json.loads(request.body.decode('utf-8')) # Bitbucket and GitHub sends json as x-www-form-data else: data = json.loads(request.POST['payload']) except (ValueError, KeyError, UnicodeError): return HttpResponseBadRequest('Could not parse JSON payload!') # Get service helper hook_helper = HOOK_HANDLERS[service] # Send the request data to the service handler. try: service_data = hook_helper(data) except KeyError: LOGGER.error('failed to parse service %s data', service) return HttpResponseBadRequest('Invalid data in json payload!') # Log data service_long_name = service_data['service_long_name'] repos = service_data['repos'] repo_url = service_data['repo_url'] branch = service_data['branch'] LOGGER.info('received %s notification on repository %s, branch %s', service_long_name, repo_url, branch) subprojects = SubProject.objects.filter(repo__in=repos) if branch is not None: subprojects = subprojects.filter(branch=branch) # Trigger updates updates = 0 for obj in subprojects: if not obj.project.enable_hooks: continue updates += 1 LOGGER.info('%s notification will update %s', service_long_name, obj) perform_update(obj) if updates == 0: return hook_response('No matching repositories found!', 'failure') return hook_response()
def get_plural_type(code, pluralequation): """Get correct plural type for language.""" # Remove not needed parenthesis if pluralequation[-1] == ';': pluralequation = pluralequation[:-1] # Get base language code base_code = code.replace('_', '-').split('-')[0] # No plural if pluralequation == '0': return data.PLURAL_NONE # Standard plural equations for mapping in data.PLURAL_MAPPINGS: if pluralequation in mapping[0]: return mapping[1] # Arabic special case if base_code in ('ar',): return data.PLURAL_ARABIC # Log error in case of uknown mapping LOGGER.error( 'Can not guess type of plural for %s: %s', code, pluralequation ) return data.PLURAL_UNKNOWN
def report_error(self, exc, request, message): """Wrapper for handling error situations""" extra = {'mt_url': self.request_url, 'mt_params': self.request_params} report_error(exc, request, extra) LOGGER.error(message, self.name) LOGGER.info('Last URL: %s, params: %s', extra['mt_url'], extra['mt_params'])
def get_plural_type(base_code, pluralequation): """Get correct plural type for language.""" # Remove not needed parenthesis if pluralequation[-1] == ';': pluralequation = pluralequation[:-1] # No plural if pluralequation == '0': return data.PLURAL_NONE # Standard plural equations for mapping in data.PLURAL_MAPPINGS: if pluralequation in mapping[0]: return mapping[1] # Arabic special case if base_code in ('ar',): return data.PLURAL_ARABIC # Log error in case of uknown mapping LOGGER.error( 'Can not guess type of plural for %s: %s', base_code, pluralequation ) return data.PLURAL_UNKNOWN
def get_plural_type(code, pluralequation): ''' Gets correct plural type for language. ''' # Remove not needed parenthesis if pluralequation[-1] == ';': pluralequation = pluralequation[:-1] if pluralequation[0] == '(' and pluralequation[-1] == ')': pluralequation = pluralequation[1:-1] # Get base language code base_code = code.replace('_', '-').split('-')[0] # No plural if pluralequation == '0': return data.PLURAL_NONE # Standard plural equations for mapping in data.PLURAL_MAPPINGS: if pluralequation in mapping[0]: return mapping[1] # Arabic special case if base_code in ('ar',): return data.PLURAL_ARABIC # Log error in case of uknown mapping LOGGER.error( 'Can not guess type of plural for %s: %s', code, pluralequation ) return data.PLURAL_UNKNOWN
def get_plural_type(base_code, plural_formula): """Get correct plural type for language.""" # Remove not needed parenthesis if plural_formula[-1] == ";": plural_formula = plural_formula[:-1] # No plural if plural_formula == "0": return data.PLURAL_NONE # Remove whitespace formula = plural_formula.replace(" ", "") # Standard plural formulas for mapping in data.PLURAL_MAPPINGS: if formula in mapping[0]: return mapping[1] # Arabic special case if base_code in ("ar", ): return data.PLURAL_ARABIC # Log error in case of uknown mapping LOGGER.error("Can not guess type of plural for %s: %s", base_code, plural_formula) return data.PLURAL_UNKNOWN
def get_avatar_image(request, user, size): """Return avatar image from cache (if available) or download it.""" cache_key = '-'.join(( 'avatar-img', user.username, str(size) )) # Try using avatar specific cache if available try: cache = caches['avatar'] except InvalidCacheBackendError: cache = caches['default'] image = cache.get(cache_key) if image is None: try: image = download_avatar_image(user, size) cache.set(cache_key, image) except (IOError, CertificateError) as error: report_error( error, sys.exc_info(), request, extra_data={'avatar': user.username}, level='debug', ) LOGGER.error( 'Failed to fetch avatar for %s: %s', user.username, str(error) ) return get_fallback_avatar(size) return image
def bitbucket_hook_helper(data): ''' API to handle service hooks from Bitbucket. ''' if 'push' in data: return bitbucket_webhook_helper(data) # Parse owner, branch and repository name owner = data['repository']['owner'] slug = data['repository']['slug'] if data['commits']: branch = data['commits'][-1]['branch'] else: branch = None params = {'owner': owner, 'slug': slug} # Construct possible repository URLs if data['repository']['scm'] == 'git': repos = [repo % params for repo in BITBUCKET_GIT_REPOS] elif data['repository']['scm'] == 'hg': repos = [repo % params for repo in BITBUCKET_HG_REPOS] else: LOGGER.error('unsupported repository: %s', repr(data['repository'])) raise ValueError('unsupported repository') return { 'service_long_name': 'Bitbucket', 'repo_url': ''.join([data['canon_url'], data['repository']['absolute_url']]), 'repos': repos, 'branch': branch, }
def mail_admins_contact(request, subject, message, context, sender): """Send a message to the admins, as defined by the ADMINS setting.""" LOGGER.info( 'contact form from %s', sender, ) if not settings.ADMINS: messages.error( request, _('Message could not be sent to administrator!') ) LOGGER.error( 'ADMINS not configured, can not send message!' ) return mail = EmailMultiAlternatives( '{0}{1}'.format(settings.EMAIL_SUBJECT_PREFIX, subject % context), message % context, to=[a[1] for a in settings.ADMINS], headers={'Reply-To': sender}, ) mail.send(fail_silently=False) messages.success( request, _('Message has been sent to administrator.') )
def report_error( error, request=None, extra_data=None, level='warning', prefix='Handled exception', skip_sentry=False, print_tb=False, ): """Wrapper for error reporting This can be used for store exceptions in error reporting solutions as rollbar while handling error gracefully and giving user cleaner message. """ if HAS_ROLLBAR and hasattr(settings, 'ROLLBAR'): rollbar.report_exc_info(request=request, extra_data=extra_data, level=level) if not skip_sentry and settings.SENTRY_DSN: sentry_sdk.capture_exception() LOGGER.error('%s: %s: %s', prefix, error.__class__.__name__, force_text(error)) if print_tb: LOGGER.exception(prefix)
def get_avatar_image(user, size): """ Returns avatar image from cache (if available) or downloads it. """ cache_key = 'avatar-img-{0}-{1}'.format( user.username, size ) # Try using avatar specific cache if available try: cache = caches['avatar'] except InvalidCacheBackendError: cache = caches['default'] image = cache.get(cache_key) if image is None: try: image = download_avatar_image(user, size) cache.set(cache_key, image) except IOError as error: report_error( error, sys.exc_info(), extra_data={'avatar': user.username} ) LOGGER.error( 'Failed to fetch avatar for %s: %s', user.username, str(error) ) return get_fallback_avatar(size) return image
def mail_admins_contact(request, subject, message, context, sender, to): """Send a message to the admins, as defined by the ADMINS setting.""" LOGGER.info("contact form from %s", sender) if not to and settings.ADMINS: to = [a[1] for a in settings.ADMINS] elif not settings.ADMINS: messages.error(request, _("Could not send message to administrator.")) LOGGER.error("ADMINS not configured, cannot send message") return mail = EmailMultiAlternatives( "{}{}".format(settings.EMAIL_SUBJECT_PREFIX, subject % context), "{}\n{}".format( message % context, TEMPLATE_FOOTER.format( address=get_ip_address(request), agent=get_user_agent(request), username=request.user.username, ), ), to=to, headers={"Reply-To": sender}, ) mail.send(fail_silently=False) messages.success( request, _("Your request has been sent, you will shortly hear from us."))
def bitbucket_hook_helper(data): """API to handle service hooks from Bitbucket.""" if 'push' in data: return bitbucket_webhook_helper(data) # Parse owner, branch and repository name owner = data['repository']['owner'] slug = data['repository']['slug'] if data['commits']: branch = data['commits'][-1]['branch'] else: branch = None params = {'owner': owner, 'slug': slug} # Construct possible repository URLs if data['repository']['scm'] == 'git': repos = [repo % params for repo in BITBUCKET_GIT_REPOS] elif data['repository']['scm'] == 'hg': repos = [repo % params for repo in BITBUCKET_HG_REPOS] else: LOGGER.error( 'unsupported repository: %s', repr(data['repository']) ) raise ValueError('unsupported repository') return { 'service_long_name': 'Bitbucket', 'repo_url': ''.join([ data['canon_url'], data['repository']['absolute_url'] ]), 'repos': repos, 'branch': branch, }
def send_mails(mails): """Sends multiple mails in single connection.""" try: connection = get_connection() connection.send_messages([mail for mail in mails if mail is not None]) except SMTPException as error: LOGGER.error('Failed to send email: %s', error) report_error(error, sys.exc_info())
def send_mails(mails): """Sends multiple mails in single connection.""" try: connection = get_connection() connection.send_messages(mails) except SMTPException as error: LOGGER.error('Failed to send email: %s', error) report_error(error, sys.exc_info())
def report_error(self, exc, request, message): """Wrapper for handling error situations""" extra = {'mt_url': self.request_url, 'mt_params': self.request_params} report_error(exc, request, extra, prefix='Machine translation error') LOGGER.error(message, self.name) LOGGER.info( 'Last URL: %s, params: %s', extra['mt_url'], extra['mt_params'] )
def report_error(error, exc_info, request=None, extra_data=None): """Wrapper for error reporting This can be used for store exceptions in error reporting solutions as rollbar while handling error gracefully and giving user cleaner message. """ if HAS_ROLLBAR and hasattr(settings, 'ROLLBAR'): rollbar.report_exc_info(exc_info, request, extra_data=extra_data) LOGGER.error( 'Handled exception %s: %s', error.__class__.__name__, str(error) )
def bitbucket_hook_helper(data, request): """API to handle service hooks from Bitbucket.""" # Bitbucket ping event if request and request.META.get("HTTP_X_EVENT_KEY") not in ( "repo:push", "repo:refs_changed", "pullrequest:fulfilled", "pr:merged", ): return None if "pullRequest" in data: # The pr:merged event repository = data["pullRequest"]["fromRef"]["repository"] else: repository = data["repository"] full_name = bitbucket_extract_full_name(repository) repo_url = bitbucket_extract_repo_url(data, repository) # Extract repository links if "links" in repository and "clone" in repository["links"]: repos = [val["href"] for val in repository["links"]["clone"]] else: repo_servers = {"bitbucket.org", urlparse(repo_url).hostname} repos = [] if "scm" not in data["repository"]: templates = BITBUCKET_GIT_REPOS + BITBUCKET_HG_REPOS elif data["repository"]["scm"] == "hg": templates = BITBUCKET_HG_REPOS else: templates = BITBUCKET_GIT_REPOS # Construct possible repository URLs for repo in templates: repos.extend( repo.format(full_name=full_name, server=server) for server in repo_servers ) if not repos: LOGGER.error("unsupported repository: %s", repr(data["repository"])) raise ValueError("unsupported repository") return { "service_long_name": "Bitbucket", "repo_url": repo_url, "repos": repos, "branch": bitbucket_extract_branch(data), "full_name": f"{full_name}.git", }
def report_error(self, exc, message): """Wrapper for handling error situations""" report_error( exc, sys.exc_info(), {'mt_url': self.request_url, 'mt_params': self.request_params} ) LOGGER.error( message, self.name, ) LOGGER.error( 'Last fetched URL: %s, params: %s', self.request_url, self.request_params, )
def clean_checksum(self): """Validate whether checksum is valid and fetches unit for it.""" if 'checksum' not in self.cleaned_data: return unit_set = self.translation.unit_set try: self.cleaned_data['unit'] = unit_set.filter( id_hash=self.cleaned_data['checksum'])[0] except (Unit.DoesNotExist, IndexError): LOGGER.error('message %s disappeared!', self.cleaned_data['checksum']) raise ValidationError( _('Message you wanted to translate is no longer available!'))
def report_error(error, exc_info, request=None, extra_data=None): """Wrapper for error reporting This can be used for store exceptions in error reporting solutions as rollbar while handling error gracefully and giving user cleaner message. """ if HAS_ROLLBAR and hasattr(settings, 'ROLLBAR'): rollbar.report_exc_info(exc_info, request, extra_data=extra_data) LOGGER.error('Handled exception %s: %s', error.__class__.__name__, force_text(error).encode('utf-8')) # Print error when running testsuite if sys.argv[1:2] == ['test']: traceback.print_exc()
def bitbucket_hook_helper(data, request): """API to handle service hooks from Bitbucket.""" # Bitbucket ping event if request and request.META.get('HTTP_X_EVENT_KEY') not in ( 'repo:push', 'repo:refs_changed', ): return None repository = data['repository'] full_name = bitbucket_extract_full_name(repository) repo_url = bitbucket_extract_repo_url(data, repository) # Extract repository links if 'links' in repository and 'clone' in repository['links']: repos = [val['href'] for val in repository['links']['clone']] else: repo_servers = {'bitbucket.org', urlparse(repo_url).hostname} repos = [] if 'scm' not in data['repository']: templates = BITBUCKET_GIT_REPOS + BITBUCKET_HG_REPOS elif data['repository']['scm'] == 'hg': templates = BITBUCKET_HG_REPOS else: templates = BITBUCKET_GIT_REPOS # Construct possible repository URLs for repo in templates: repos.extend( ( repo.format(full_name=full_name, server=server) for server in repo_servers ) ) if not repos: LOGGER.error('unsupported repository: %s', repr(data['repository'])) raise ValueError('unsupported repository') return { 'service_long_name': 'Bitbucket', 'repo_url': repo_url, 'repos': repos, 'branch': bitbucket_extract_branch(data), 'full_name': '{}.git'.format(full_name), }
def report_error(error, exc_info, request=None, extra_data=None): """Wrapper for error reporting This can be used for store exceptions in error reporting solutions as rollbar while handling error gracefully and giving user cleaner message. """ if HAS_ROLLBAR and hasattr(settings, 'ROLLBAR'): rollbar.report_exc_info(exc_info, request, extra_data=extra_data) LOGGER.error( 'Handled exception %s: %s', error.__class__.__name__, force_text(error).encode('utf-8') ) # Print error when running testsuite if sys.argv[1:2] == ['test']: traceback.print_exc()
def report_error(error, request=None, extra_data=None): """Wrapper for error reporting This can be used for store exceptions in error reporting solutions as rollbar while handling error gracefully and giving user cleaner message. """ if HAS_ROLLBAR and hasattr(settings, 'ROLLBAR'): rollbar.report_exc_info(request=request, extra_data=extra_data, level='warning') if HAS_RAVEN and hasattr(settings, 'RAVEN_CONFIG'): raven_client.captureException(request=request, extra=extra_data, level='warning') LOGGER.error('Handled exception %s: %s', error.__class__.__name__, force_text(error))
def clean_checksum(self): """Validate whether checksum is valid and fetches unit for it.""" if 'checksum' not in self.cleaned_data: return unit_set = self.translation.unit_set try: self.cleaned_data['unit'] = unit_set.filter( id_hash=self.cleaned_data['checksum'] )[0] except (Unit.DoesNotExist, IndexError): LOGGER.error( 'string %s disappeared!', self.cleaned_data['checksum'] ) raise ValidationError(_( 'The string you wanted to translate is no longer available!' ))
def translate(self, language, text, unit, user): ''' Returns list of machine translations. ''' language = self.convert_language(language) if text == '': return [] if not self.is_supported(language): return [] try: translations = self.download_translations( language, text, unit, user ) return [ { 'text': trans[0], 'quality': trans[1], 'service': trans[2], 'source': trans[3] } for trans in translations ] except Exception as exc: report_error( exc, sys.exc_info(), {'mt_url': self.request_url, 'mt_params': self.request_params} ) LOGGER.error( 'Failed to fetch translations from %s', self.name, ) LOGGER.error( 'Last fetched URL: %s, params: %s', self.request_url, self.request_params, ) raise MachineTranslationError('{}: {}'.format( exc.__class__.__name__, str(exc) ))
def mail_admins_contact(request, subject, message, context, sender): """ Sends a message to the admins, as defined by the ADMINS setting. """ LOGGER.info("contact form from %s", sender) if not settings.ADMINS: messages.error(request, _("Message could not be sent to administrator!")) LOGGER.error("ADMINS not configured, can not send message!") return mail = EmailMultiAlternatives( "%s%s" % (settings.EMAIL_SUBJECT_PREFIX, subject % context), message % context, to=[a[1] for a in settings.ADMINS], headers={"Reply-To": sender}, ) mail.send(fail_silently=False) messages.success(request, _("Message has been sent to administrator."))
def report_error(error, request=None, extra_data=None): """Wrapper for error reporting This can be used for store exceptions in error reporting solutions as rollbar while handling error gracefully and giving user cleaner message. """ if HAS_ROLLBAR and hasattr(settings, 'ROLLBAR'): rollbar.report_exc_info( request=request, extra_data=extra_data, level='warning' ) if HAS_RAVEN and hasattr(settings, 'RAVEN_CONFIG'): raven_client.captureException( request=request, extra=extra_data, level='warning' ) LOGGER.error( 'Handled exception %s: %s', error.__class__.__name__, force_text(error) )
def clean_checksum(self): ''' Validates whether checksum is valid and fetches unit for it. ''' if 'checksum' not in self.cleaned_data: return unit_set = self.translation.unit_set try: self.cleaned_data['unit'] = unit_set.filter( checksum=self.cleaned_data['checksum'], )[0] except (Unit.DoesNotExist, IndexError): LOGGER.error( 'message %s disappeared!', self.cleaned_data['checksum'] ) raise ValidationError( _('Message you wanted to translate is no longer available!') )
def supported_languages(self): ''' Returns list of supported languages. ''' cache_key = '%s-languages' % self.mtid # Try using list from cache languages = cache.get(cache_key) if languages is not None: return languages # Download try: languages = self.download_languages() except Exception as exc: report_error( exc, sys.exc_info(), {'mt_url': self.request_url, 'mt_params': self.request_params} ) LOGGER.error( 'Failed to fetch languages from %s, using defaults', self.name, ) LOGGER.error( 'Last fetched URL: %s, params: %s', self.request_url, self.request_params, ) if settings.DEBUG: raise return self.default_languages # Update cache cache.set(cache_key, languages, 3600 * 48) return languages
def log_error(self, msg, *args): self.log_hook('ERROR', msg, *args) return LOGGER.error(': '.join((self.full_slug, msg)), *args)
def log_error(self, msg, *args): return LOGGER.error(': '.join((self.full_slug, msg)), *args)
def vcs_service_hook(request, service): """Shared code between VCS service hooks. Currently used for bitbucket_hook, github_hook and gitlab_hook, but should be usable for other VCS services (Google Code, custom coded sites, etc.) too. """ # We support only post methods if not settings.ENABLE_HOOKS: return HttpResponseNotAllowed(()) # Check if we got payload try: data = parse_hook_payload(request) except (ValueError, KeyError, UnicodeError): return HttpResponseBadRequest('Could not parse JSON payload!') # Get service helper hook_helper = HOOK_HANDLERS[service] # Send the request data to the service handler. try: service_data = hook_helper(data) except Exception as error: LOGGER.error('failed to parse service %s data', service) report_error(error, sys.exc_info()) return HttpResponseBadRequest('Invalid data in json payload!') # Log data service_long_name = service_data['service_long_name'] repos = service_data['repos'] repo_url = service_data['repo_url'] branch = service_data['branch'] # Generate filter spfilter = Q(repo__in=repos) # We need to match also URLs which include username and password for repo in repos: if not repo.startswith('https://'): continue spfilter = spfilter | (Q(repo__startswith='https://') & Q(repo__endswith='@{0}'.format(repo[8:]))) all_subprojects = SubProject.objects.filter(spfilter) if branch is not None: all_subprojects = all_subprojects.filter(branch=branch) subprojects = all_subprojects.filter(project__enable_hooks=True) LOGGER.info( 'received %s notification on repository %s, branch %s,' '%d matching components, %d to process', service_long_name, repo_url, branch, all_subprojects.count(), subprojects.count(), ) # Trigger updates updates = 0 for obj in subprojects: updates += 1 LOGGER.info('%s notification will update %s', service_long_name, obj) perform_update(obj) if updates == 0: return hook_response('No matching repositories found!', 'failure') return hook_response()
def log_error(self, msg, *args): return LOGGER.error(self.log_prefix + msg, *args)
def log_error(self, msg, *args): self.log_hook('ERROR', msg, *args) return LOGGER.error( ': '.join((self.full_slug, msg)), *args )
def log_error(self, msg, *args): return LOGGER.error( self.log_prefix + msg, *args )
def report_error(self, message): """Wrapper for handling error situations.""" report_error(cause="Machinery error") LOGGER.error(message, self.name)
def vcs_service_hook(request, service): """Shared code between VCS service hooks. Currently used for bitbucket_hook, github_hook and gitlab_hook, but should be usable for other VCS services (Google Code, custom coded sites, etc.) too. """ # We support only post methods if not settings.ENABLE_HOOKS: return HttpResponseNotAllowed(()) # Check if we got payload try: data = parse_hook_payload(request) except (ValueError, KeyError, UnicodeError): return HttpResponseBadRequest('Could not parse JSON payload!') # Get service helper hook_helper = HOOK_HANDLERS[service] # Send the request data to the service handler. try: service_data = hook_helper(data) except Exception as error: LOGGER.error('failed to parse service %s data', service) report_error(error, sys.exc_info()) return HttpResponseBadRequest('Invalid data in json payload!') # This happens on ping request upon installation if service_data is None: return hook_response('Hook working') # Log data service_long_name = service_data['service_long_name'] repos = service_data['repos'] repo_url = service_data['repo_url'] branch = service_data['branch'] full_name = service_data['full_name'] # Generate filter spfilter = Q(repo__in=repos) | Q(repo__iendswith=full_name) # We need to match also URLs which include username and password for repo in repos: if not repo.startswith('https://'): continue spfilter = spfilter | ( Q(repo__startswith='https://') & Q(repo__endswith='@{0}'.format(repo[8:])) ) all_components = Component.objects.filter(spfilter) if branch is not None: all_components = all_components.filter(branch=branch) components = all_components.filter(project__enable_hooks=True) LOGGER.info( 'received %s notification on repository %s, branch %s, ' '%d matching components, %d to process', service_long_name, repo_url, branch, all_components.count(), components.count(), ) # Trigger updates updates = 0 for obj in components: updates += 1 LOGGER.info( '%s notification will update %s', service_long_name, obj ) perform_update(obj) if updates == 0: return hook_response('No matching repositories found!', 'failure') return hook_response('Update triggered: {}'.format( ', '.join([obj.log_prefix for obj in components]) ))
def log_error(self, msg, *args): return LOGGER.error( ': '.join((self.full_slug, msg)), *args )
def log_error(self, msg, *args): self.log_hook("ERROR", msg, *args) return LOGGER.error(": ".join((self.full_slug, msg)), *args)
def log_error(self, msg, *args): return LOGGER.error( ': '.join((self.log_prefix, msg)), *args )
def report_error(self, exc, message): """Wrapper for handling error situations.""" report_error(exc, prefix='Machinery error') LOGGER.error(message, self.name)
def report_error(self, exc, message): """Wrapper for handling error situations""" report_error(exc, prefix='Machinery error') LOGGER.error(message, self.name) LOGGER.info('Last URL: %s, params: %s', self.request_url, self.request_params)
def vcs_service_hook(request, service): """Shared code between VCS service hooks. Currently used for bitbucket_hook, github_hook and gitlab_hook, but should be usable for other VCS services (Google Code, custom coded sites, etc.) too. """ # We support only post methods if not settings.ENABLE_HOOKS: return HttpResponseNotAllowed(()) # Check if we got payload try: data = parse_hook_payload(request) except (ValueError, KeyError): return HttpResponseBadRequest("Could not parse JSON payload!") if not data: return HttpResponseBadRequest("Invalid data in json payload!") # Get service helper hook_helper = HOOK_HANDLERS[service] # Send the request data to the service handler. try: service_data = hook_helper(data, request) except Exception as error: LOGGER.error("failed to parse service %s data", service) report_error(error, request) return HttpResponseBadRequest("Invalid data in json payload!") # This happens on ping request upon installation if service_data is None: return hook_response("Hook working", status=201) # Log data service_long_name = service_data["service_long_name"] repos = service_data["repos"] repo_url = service_data["repo_url"] branch = service_data["branch"] full_name = service_data["full_name"] # Generate filter spfilter = Q(repo__in=repos) | Q(repo__iendswith=full_name) # We need to match also URLs which include username and password for repo in repos: if repo.startswith("http://"): spfilter = spfilter | ( Q(repo__startswith="http://") & Q(repo__endswith="@{0}".format(repo[7:])) ) elif repo.startswith("https://"): spfilter = spfilter | ( Q(repo__startswith="https://") & Q(repo__endswith="@{0}".format(repo[8:])) ) all_components = Component.objects.filter(spfilter) if branch is not None: all_components = all_components.filter(branch=branch) components = all_components.filter(project__enable_hooks=True) LOGGER.info( "received %s notification on repository %s, branch %s, " "%d matching components, %d to process", service_long_name, repo_url, branch, all_components.count(), components.count(), ) # Trigger updates updates = 0 for obj in components: updates += 1 LOGGER.info("%s notification will update %s", service_long_name, obj) Change.objects.create( component=obj, action=Change.ACTION_HOOK, details=service_data ) perform_update.delay("Component", obj.pk) if updates == 0: return hook_response("No matching repositories found!", "failure", status=202) return hook_response( "Update triggered: {}".format(", ".join(obj.full_slug for obj in components)) )
def log_error(self, msg, *args): return LOGGER.error(": ".join((self.log_prefix, msg)), *args)
def vcs_service_hook(request, service): ''' Shared code between VCS service hooks. Currently used for bitbucket_hook, github_hook and gitlab_hook, but should be usable for other VCS services (Google Code, custom coded sites, etc.) too. ''' # Check for enabled hooks if appsettings.ENABLE_HOOKS: allowed_methods = ('POST',) else: allowed_methods = () # We support only post methods if not appsettings.ENABLE_HOOKS or request.method not in allowed_methods: return HttpResponseNotAllowed(allowed_methods) # Check if we got payload try: # GitLab sends json as application/json if request.META['CONTENT_TYPE'] == 'application/json': data = json.loads(request.body) # Bitbucket and GitHub sends json as x-www-form-data else: data = json.loads(request.POST['payload']) except (ValueError, KeyError): return HttpResponseBadRequest('Could not parse JSON payload!') # Get service helper if service not in HOOK_HANDLERS: LOGGER.error('service %s is not supported', service) return HttpResponseBadRequest('invalid service') hook_helper = HOOK_HANDLERS[service] # Send the request data to the service handler. try: service_data = hook_helper(data) except KeyError: LOGGER.error('failed to parse service %s data', service) return HttpResponseBadRequest('Invalid data in json payload!') # Log data service_long_name = service_data['service_long_name'] repos = service_data['repos'] repo_url = service_data['repo_url'] branch = service_data['branch'] LOGGER.info( 'received %s notification on repository %s, branch %s', service_long_name, repo_url, branch ) subprojects = SubProject.objects.filter(repo__in=repos) if branch is not None: subprojects = subprojects.filter(branch=branch) # Trigger updates for obj in subprojects: if not obj.project.enable_hooks: continue LOGGER.info( '%s notification will update %s', service_long_name, obj ) perform_update(obj) return hook_response()