from import_paypal import PAYPAL_ORGS, import_org_period
celery_logger = get_task_logger(__name__)
celery_logger.setLevel(LOG_LEVEL)
if SENTRY_ENABLE:
celery_logger.info("Enabling Sentry")
import sentry_sdk
from sentry_sdk.integrations.celery import CeleryIntegration
from sentry_sdk.integrations.logging import LoggingIntegration
sentry_logging = LoggingIntegration(level=logging.DEBUG, event_level=logging.WARNING) # Capture debug and above as breadcrumbs
sentry_sdk.init(
dsn=SENTRY_DSN,
environment=SENTRY_ENVIRONMENT,
integrations=[sentry_logging, CeleryIntegration()],
)
WORKER_LOG_FORMAT = "%(levelname)s %(name)s/%(module)s:%(lineno)d - %(message)s"
WORKER_TASK_LOG_FORMAT = "%(levelname)s [%(task_id).8s] %(name)s/%(module)s:%(lineno)d - %(message)s"
celery_app = Celery("worker", broker=CELERY_BROKER_URL)
@celery_app.task()
def import_last_x_days_paypal(days=3):
celery_logger.info("---> starting import_last_x_days...")
today = datetime.now(tz=ZONE)
# Change this value if your application is running behind a proxy,
# e.g. HTTP_CF_Connecting_IP for Cloudflare or X_FORWARDED_FOR
REAL_IP_ENVIRON = os.environ.get("REAL_IP_ENVIRON", "REMOTE_ADDR")
# The maximum length of a graphql query to log in tracings
OPENTRACING_MAX_QUERY_LENGTH_LOG = 2000
# Slugs for menus precreated in Django migrations
DEFAULT_MENUS = {"top_menu_name": "navbar", "bottom_menu_name": "footer"}
# Sentry
SENTRY_DSN = os.environ.get("SENTRY_DSN")
if SENTRY_DSN:
sentry_sdk.init(
dsn=SENTRY_DSN, integrations=[CeleryIntegration(), DjangoIntegration()]
)
GRAPHENE = {
"RELAY_CONNECTION_ENFORCE_FIRST_OR_LAST": True,
"RELAY_CONNECTION_MAX_LIMIT": 100,
"MIDDLEWARE": [
"saleor.graphql.middleware.OpentracingGrapheneMiddleware",
"saleor.graphql.middleware.JWTMiddleware",
"saleor.graphql.middleware.app_middleware",
],
}
PLUGINS_MANAGER = "saleor.plugins.manager.PluginsManager"
PLUGINS = [
# unless that ip is part of ALLOWED_HOSTS. The ip changes so we need to grab it
# dynamically.
# See: https://stackoverflow.com/questions/34428877/django-allowed-host-setting-for-elastic-beanstalk-instance-behind-elastic-load-b?noredirect=1&lq=1
import requests
import sentry_sdk
from sentry_sdk.integrations.celery import CeleryIntegration
from sentry_sdk.integrations.django import DjangoIntegration
from sentry_sdk.integrations.logging import ignore_logger
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = False
PRODUCTION = True
sentry_sdk.init(
dsn=os.environ["SENTRY_KEY"],
integrations=[DjangoIntegration(), CeleryIntegration()],
)
ignore_logger("django.security.DisallowedHost") # Avoid constant log spam
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/2.1/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = os.environ["SECRET_KEY"]
CLOUDWATCH_EVENT_SECRET = os.environ["CLOUDWATCH_EVENT_SECRET"]
MAILGUN_API_KEY = os.environ["MAILGUN_API_KEY"]
from sentry_sdk.integrations.redis import RedisIntegration
from mergify_engine import config
from mergify_engine import exceptions
LOG = daiquiri.getLogger(__name__)
def fixup_sentry_reporting(event, hint): # pragma: no cover
# NOTE(sileht): Block exceptions that celery will retry until
# the retries handler manually send the event.
is_celery_task = "celery-job" in event.get("extra", {})
is_exception = "exc_info" in hint
if is_exception and is_celery_task:
exc_type, exc_value, tb = hint["exc_info"]
backoff = exceptions.need_retry(exc_value)
if backoff or isinstance(exc_value, celery.exceptions.Retry):
return None
return event
if config.SENTRY_URL: # pragma: no cover
sentry_sdk.init(
config.SENTRY_URL,
environment=config.SENTRY_ENVIRONMENT,
before_send=fixup_sentry_reporting,
integrations=[CeleryIntegration(), FlaskIntegration(), RedisIntegration()],
)
# Change this value if your application is running behind a proxy,
# e.g. HTTP_CF_Connecting_IP for Cloudflare or X_FORWARDED_FOR
REAL_IP_ENVIRON = os.environ.get("REAL_IP_ENVIRON", "REMOTE_ADDR")
# The maximum length of a graphql query to log in tracings
OPENTRACING_MAX_QUERY_LENGTH_LOG = 2000
# Slugs for menus precreated in Django migrations
DEFAULT_MENUS = {"top_menu_name": "navbar", "bottom_menu_name": "footer"}
# Sentry
SENTRY_DSN = os.environ.get("SENTRY_DSN")
if SENTRY_DSN:
sentry_sdk.init(dsn=SENTRY_DSN,
integrations=[CeleryIntegration(),
DjangoIntegration()])
GRAPHENE = {
"RELAY_CONNECTION_ENFORCE_FIRST_OR_LAST":
True,
"RELAY_CONNECTION_MAX_LIMIT":
100,
"MIDDLEWARE": [
"saleor.graphql.middleware.OpentracingGrapheneMiddleware",
"saleor.graphql.middleware.JWTMiddleware",
"saleor.graphql.middleware.app_middleware",
],
}
PLUGINS_MANAGER = "saleor.plugins.manager.PluginsManager"
def create_app():
global app_created
if not app_created:
BlueprintsManager.register(app)
graphql_views.init_app(app)
Migrate(app, db)
app.config.from_object(env('APP_CONFIG',
default='config.ProductionConfig'))
if not app.config['SECRET_KEY']:
if app.config['PRODUCTION']:
app.logger.error(
'SECRET_KEY must be set in .env or environment variables in production'
)
exit(1)
else:
random_secret = secrets.token_hex()
app.logger.warning(
f'Using random secret "{ random_secret }" for development server. '
'This is NOT recommended. Set proper SECRET_KEY in .env or environment variables'
)
app.config['SECRET_KEY'] = random_secret
db.init_app(app)
if app.config['CACHING']:
cache.init_app(app, config={'CACHE_TYPE': 'simple'})
else:
cache.init_app(app, config={'CACHE_TYPE': 'null'})
stripe.api_key = 'SomeStripeKey'
app.config['JSONIFY_PRETTYPRINT_REGULAR'] = False
app.config['FILE_SYSTEM_STORAGE_FILE_VIEW'] = 'static'
app.logger.addHandler(logging.StreamHandler(sys.stdout))
app.logger.setLevel(logging.ERROR)
# set up jwt
app.config['JWT_HEADER_TYPE'] = 'JWT'
app.config['JWT_ACCESS_TOKEN_EXPIRES'] = timedelta(days=1)
app.config['JWT_REFRESH_TOKEN_EXPIRES'] = timedelta(days=365)
app.config['JWT_ERROR_MESSAGE_KEY'] = 'error'
app.config['JWT_TOKEN_LOCATION'] = ['cookies', 'headers']
app.config['JWT_REFRESH_COOKIE_PATH'] = '/v1/auth/token/refresh'
app.config['JWT_SESSION_COOKIE'] = False
app.config['JWT_BLACKLIST_ENABLED'] = True
app.config['JWT_BLACKLIST_TOKEN_CHECKS'] = ['refresh']
_jwt = JWTManager(app)
_jwt.user_loader_callback_loader(jwt_user_loader)
_jwt.token_in_blacklist_loader(is_token_blacklisted)
# setup celery
app.config['CELERY_BROKER_URL'] = app.config['REDIS_URL']
app.config['CELERY_RESULT_BACKEND'] = app.config['CELERY_BROKER_URL']
app.config['CELERY_ACCEPT_CONTENT'] = ['json', 'application/text']
CORS(app, resources={r"/*": {"origins": "*"}})
AuthManager.init_login(app)
if app.config['TESTING'] and app.config['PROFILE']:
# Profiling
app.wsgi_app = ProfilerMiddleware(app.wsgi_app, restrictions=[30])
# development api
with app.app_context():
from app.api.admin_statistics_api.events import event_statistics
from app.api.auth import auth_routes
from app.api.custom.attendees import attendee_blueprint
from app.api.bootstrap import api_v1
from app.api.celery_tasks import celery_routes
from app.api.event_copy import event_copy
from app.api.exports import export_routes
from app.api.imports import import_routes
from app.api.uploads import upload_routes
from app.api.users import user_misc_routes
from app.api.orders import order_misc_routes
from app.api.role_invites import role_invites_misc_routes
from app.api.auth import authorised_blueprint
from app.api.admin_translations import admin_blueprint
from app.api.orders import alipay_blueprint
from app.api.sessions import sessions_blueprint
from app.api.settings import admin_misc_routes
from app.api.server_version import info_route
from app.api.custom.orders import ticket_blueprint
from app.api.custom.orders import order_blueprint
from app.api.custom.invoices import event_blueprint
app.register_blueprint(api_v1)
app.register_blueprint(event_copy)
app.register_blueprint(upload_routes)
app.register_blueprint(export_routes)
app.register_blueprint(import_routes)
app.register_blueprint(celery_routes)
app.register_blueprint(auth_routes)
app.register_blueprint(event_statistics)
app.register_blueprint(user_misc_routes)
app.register_blueprint(attendee_blueprint)
app.register_blueprint(order_misc_routes)
app.register_blueprint(role_invites_misc_routes)
app.register_blueprint(authorised_blueprint)
app.register_blueprint(admin_blueprint)
app.register_blueprint(alipay_blueprint)
app.register_blueprint(admin_misc_routes)
app.register_blueprint(info_route)
app.register_blueprint(ticket_blueprint)
app.register_blueprint(order_blueprint)
app.register_blueprint(event_blueprint)
app.register_blueprint(sessions_blueprint)
add_engine_pidguard(db.engine)
if app.config['SQLALCHEMY_DATABASE_URI' # pytype: disable=attribute-error
].startswith("sqlite://"):
sqlite_datetime_fix()
sa.orm.configure_mappers()
if app.config['SERVE_STATIC']:
app.add_url_rule('/static/<path:filename>',
endpoint='static',
view_func=app.send_static_file)
# sentry
if not app_created and 'SENTRY_DSN' in app.config:
sentry_sdk.init(
app.config['SENTRY_DSN'],
integrations=[
FlaskIntegration(),
RedisIntegration(),
CeleryIntegration(),
SqlalchemyIntegration(),
],
release=app.config['SENTRY_RELEASE_NAME'],
traces_sample_rate=app.config['SENTRY_TRACES_SAMPLE_RATE'],
)
# redis
redis_store.init_app(app)
# Initialize Extensions
shell.init_app(app)
limiter.init_app(app)
app_created = True
return app
def configure_sdk():
from sentry_sdk.integrations.logging import LoggingIntegration
from sentry_sdk.integrations.django import DjangoIntegration
from sentry_sdk.integrations.celery import CeleryIntegration
from sentry_sdk.integrations.redis import RedisIntegration
assert sentry_sdk.Hub.main.client is None
sdk_options = dict(settings.SENTRY_SDK_CONFIG)
relay_dsn = sdk_options.pop("relay_dsn", None)
internal_project_key = get_project_key()
upstream_dsn = sdk_options.pop("dsn", None)
sdk_options["traces_sampler"] = traces_sampler
if upstream_dsn:
upstream_transport = make_transport(
get_options(dsn=upstream_dsn, **sdk_options))
else:
upstream_transport = None
if relay_dsn:
relay_transport = make_transport(
get_options(dsn=relay_dsn, **sdk_options))
elif internal_project_key and internal_project_key.dsn_private:
relay_transport = make_transport(
get_options(dsn=internal_project_key.dsn_private, **sdk_options))
else:
relay_transport = None
_override_on_full_queue(relay_transport,
"internal.uncaptured.events.relay")
_override_on_full_queue(upstream_transport,
"internal.uncaptured.events.upstream")
class MultiplexingTransport(sentry_sdk.transport.Transport):
def capture_envelope(self, envelope):
# Temporarily capture envelope counts to compare to ingested
# transactions.
metrics.incr("internal.captured.events.envelopes")
transaction = envelope.get_transaction_event()
# Temporarily also capture counts for one specific transaction to check ingested amount
if (transaction and transaction.get("transaction")
== "/api/0/organizations/{organization_slug}/issues/"):
metrics.incr("internal.captured.events.envelopes.issues")
# Assume only transactions get sent via envelopes
if options.get(
"transaction-events.force-disable-internal-project"):
return
self._capture_anything("capture_envelope", envelope)
def capture_event(self, event):
if event.get("type") == "transaction" and options.get(
"transaction-events.force-disable-internal-project"):
return
self._capture_anything("capture_event", event)
def _capture_anything(self, method_name, *args, **kwargs):
# Upstream should get the event first because it is most isolated from
# the this sentry installation.
if upstream_transport:
metrics.incr("internal.captured.events.upstream")
# TODO(mattrobenolt): Bring this back safely.
# from sentry import options
# install_id = options.get('sentry:install-id')
# if install_id:
# event.setdefault('tags', {})['install-id'] = install_id
getattr(upstream_transport, method_name)(*args, **kwargs)
if relay_transport and options.get(
"store.use-relay-dsn-sample-rate") == 1:
if is_current_event_safe():
metrics.incr("internal.captured.events.relay")
getattr(relay_transport, method_name)(*args, **kwargs)
else:
metrics.incr(
"internal.uncaptured.events.relay",
skip_internal=False,
tags={"reason": "unsafe"},
)
sentry_sdk.init(
transport=MultiplexingTransport(),
integrations=[
DjangoIntegration(),
CeleryIntegration(),
LoggingIntegration(event_level=None),
RustInfoIntegration(),
RedisIntegration(),
],
**sdk_options,
)
https://docs.djangoproject.com/en/2.1/ref/settings/
"""
import logging
import os
from datetime import timedelta
import dj_database_url
import sentry_sdk
from sentry_sdk.integrations.celery import CeleryIntegration
from sentry_sdk.integrations.django import DjangoIntegration
from sentry_sdk.integrations.redis import RedisIntegration
sentry_sdk.init(
dsn=os.environ.get("SENTRY_DSN"),
integrations=[CeleryIntegration(), DjangoIntegration(), RedisIntegration()],
release=os.environ.get("BUILD_REVISION") or None,
environment=(
os.environ.get("SENTRY_ENVIRONMENT")
or os.environ.get("NODE_ENV")
or "development"
),
traces_sample_rate=1.0,
traceparent_v2=True,
_experiments={"fast_serialize": True},
)
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
# Quick-start development settings - unsuitable for production
def configure_sdk():
from sentry_sdk.integrations.logging import LoggingIntegration
from sentry_sdk.integrations.django import DjangoIntegration
from sentry_sdk.integrations.celery import CeleryIntegration
from sentry_sdk.integrations.redis import RedisIntegration
assert sentry_sdk.Hub.main.client is None
sdk_options = dict(settings.SENTRY_SDK_CONFIG)
relay_dsn = sdk_options.pop("relay_dsn", None)
internal_project_key = get_project_key()
upstream_dsn = sdk_options.pop("dsn", None)
if upstream_dsn:
upstream_transport = make_transport(
get_options(dsn=upstream_dsn, **sdk_options))
else:
upstream_transport = None
if relay_dsn:
relay_transport = make_transport(
get_options(dsn=relay_dsn, **sdk_options))
elif internal_project_key and internal_project_key.dsn_private:
relay_transport = make_transport(
get_options(dsn=internal_project_key.dsn_private, **sdk_options))
else:
relay_transport = None
class MultiplexingTransport(sentry_sdk.transport.Transport):
def capture_envelope(self, envelope):
# Assume only transactions get sent via envelopes
if options.get(
"transaction-events.force-disable-internal-project"):
return
self._capture_anything("capture_envelope", envelope)
def capture_event(self, event):
if event.get("type") == "transaction" and options.get(
"transaction-events.force-disable-internal-project"):
return
self._capture_anything("capture_event", event)
def _capture_anything(self, method_name, *args, **kwargs):
# Upstream should get the event first because it is most isolated from
# the this sentry installation.
if upstream_transport:
metrics.incr("internal.captured.events.upstream")
# TODO(mattrobenolt): Bring this back safely.
# from sentry import options
# install_id = options.get('sentry:install-id')
# if install_id:
# event.setdefault('tags', {})['install-id'] = install_id
getattr(upstream_transport, method_name)(*args, **kwargs)
if relay_transport and options.get(
"store.use-relay-dsn-sample-rate") == 1:
if is_current_event_safe():
metrics.incr("internal.captured.events.relay")
getattr(relay_transport, method_name)(*args, **kwargs)
else:
metrics.incr("internal.uncaptured.events.relay",
skip_internal=False)
sentry_sdk.init(transport=MultiplexingTransport(),
integrations=[
DjangoIntegration(),
CeleryIntegration(),
LoggingIntegration(event_level=None),
RustInfoIntegration(),
RedisIntegration(),
],
traceparent_v2=True,
**sdk_options)
import sentry_sdk
from sentry_sdk.integrations.celery import CeleryIntegration
from sentry_sdk.integrations.django import DjangoIntegration
from sentry_sdk.integrations.logging import LoggingIntegration, ignore_logger
from django.utils.translation import ugettext_lazy as _
from celery.schedules import crontab
SENTRY_DSN = os.environ.get("SENTRY_DSN", "")
if SENTRY_DSN: # pragma: no cover
sentry_sdk.init(
dsn=SENTRY_DSN,
integrations=[DjangoIntegration(), CeleryIntegration(), LoggingIntegration()],
send_default_pii=True,
traces_sample_rate=0,
)
ignore_logger("django.security.DisallowedHost")
# -----------------------------------------------------------------------------------
# Default to debugging
# -----------------------------------------------------------------------------------
DEBUG = True
# -----------------------------------------------------------------------------------
# Sets TESTING to True if this configuration is read during a unit test
# -----------------------------------------------------------------------------------
TESTING = sys.argv[1:2] == ["test"]
def configure(app, settings):
if settings.SENTRY_DSN and settings.ENV != Environment.DEVELOPMENT:
sentry_sdk.init(
settings.SENTRY_DSN.get_secret_value(),
integrations=[CeleryIntegration()],
)
def create_app():
"""Initialize the App"""
app = Flask(__name__,
static_url_path="/static",
static_folder=path.join(path.abspath(path.dirname(__file__)),
"static"))
"""Flask Config"""
app_settings = getenv("APP_SETTINGS", "src.config.ProductionConfig")
app.config.from_object(app_settings)
if (app_settings == "src.config.ProductionConfig"
and not app.config.get("SEND_MAIL")):
app.logger.warning("Sending Emails disabled on production!")
"""Set FLASK_ENV and FLASK_DEBUG cause that doesn't happen auto anymore"""
if app.config.get("DEBUG"):
environ["FLASK_ENV"] = "development" # pragma: nocover
environ["FLASK_DEBUG"] = "1" # pragma: nocover
if app.config.get("SENTRY_DSN"):
"""Initialize Sentry if we're in production"""
sentry_sdk.init(dsn=app.config.get("SENTRY_DSN"),
environment=app.config.get("SENTRY_ENV"),
release=f"backend@{__version__}",
integrations=[FlaskIntegration(),
CeleryIntegration()],
traces_sample_rate=1.0)
"""Setup Extensions"""
CORS(app)
db.init_app(app)
swagger.init_app(app)
mail.init_app(app)
bcrypt.init_app(app)
socketio.init_app(app,
cors_allowed_origins="*",
json=json,
message_queue=app.config.get("SOCKETIO_MESSAGE_QUEUE"))
from src.common.json import JSONEncoderBase
app.json_encoder = JSONEncoderBase
"""Register Blueprints"""
from src.api.auth import auth_blueprint
from src.api.hackers import hackers_blueprint
from src.api.sponsors import sponsors_blueprint
from src.api.stats import stats_blueprint
from src.api.events import events_blueprint
from src.api.club_events import club_events_blueprint
from src.api.email_verification import email_verify_blueprint
app.register_blueprint(auth_blueprint, url_prefix="/api")
app.register_blueprint(hackers_blueprint, url_prefix="/api")
app.register_blueprint(sponsors_blueprint, url_prefix="/api")
app.register_blueprint(stats_blueprint, url_prefix="/api")
app.register_blueprint(events_blueprint, url_prefix="/api")
app.register_blueprint(club_events_blueprint, url_prefix="/api")
app.register_blueprint(email_verify_blueprint, url_prefix="/api")
"""Register Error Handlers"""
from src.common import error_handlers
app.register_error_handler(HTTPException, error_handlers.handle_exception)
"""Initialize Celery"""
celery = make_celery(app)
@before_render_template.connect_via(app)
def _sentry_pre_render_template(sender, template, context, **extra):
parent = sentry_sdk.Hub.current.scope.span
if parent is not None:
span = parent.start_child(op="flask.render_template")
span.set_data("flask.render_template.sender", sender)
span.set_data("flask.render_template.template", template)
span.set_data("flask.render_template.context", context)
span.set_data("flask.render_template.extra", extra)
g._sentry_span_render_template = span
@template_rendered.connect_via(app)
def _sentry_template_rendered(sender, template, context, **extra):
span = g.pop("_sentry_span_render_template", None)
if span is not None:
span.finish()
return app, celery
STATIC_URL = '/static/'
STATIC_ROOT = os.environ.get('STATIC_ROOT',
os.path.expanduser('~/.cache/promgen'))
SITE_ID = 1
TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'
if "SENTRY_DSN" in os.environ:
import sentry_sdk
from sentry_sdk.integrations.django import DjangoIntegration
from sentry_sdk.integrations.celery import CeleryIntegration
os.environ.setdefault("SENTRY_RELEASE", __version__)
sentry_sdk.init(integrations=[DjangoIntegration(), CeleryIntegration()])
REST_FRAMEWORK = {
'DEFAULT_AUTHENTICATION_CLASSES': (
'rest_framework.authentication.TokenAuthentication',
'rest_framework.authentication.SessionAuthentication',
),
'DEFAULT_PERMISSION_CLASSES':
('rest_framework.permissions.DjangoModelPermissionsOrAnonReadOnly', ),
'DEFAULT_FILTER_BACKENDS':
('django_filters.rest_framework.DjangoFilterBackend', )
}
# If CELERY_BROKER_URL is set in our environment, then we configure celery as
# expected. If it is not configured, then we set CELERY_TASK_ALWAYS_EAGER to
# force celery to run all tasks in the same process (effectively runs each task
from .helpers.default_apps import DEFAULT_APPS
from .helpers.i18n_settings import (DEFAULT_LOCALE_PATHS, DEFAULT_LANGUAGES)
from .helpers.middlewares import DEFAULT_MIDDLEWARES
from .helpers.rest_framework_settings import (REST_FRAMEWORK_SETTINGS)
from .helpers.storages import STORAGES, DEFAULT_STORAGE
from .helpers.templates import DEFAULT_TEMPLATES
from .helpers.validators import DEFAULT_VALIDATORS
BASE_DIR: str = dirname(dirname(abspath(__file__)))
# Environment variables
env = Env()
Env.read_env()
# Sentry
SENTRY_DSN: str = env.str(var='SENTRY_DSN')
integrations: Tuple = (DjangoIntegration(), CeleryIntegration())
init(dsn=SENTRY_DSN, integrations=integrations)
# Django
DEBUG: bool = env.bool(var='DEBUG')
SECRET_KEY: str = env.str(var='SECRET_KEY')
APPEND_SLASH: bool = True
ALLOWED_HOSTS: Tuple = ('*', )
INSTALLED_APPS: Tuple = DEFAULT_APPS
MIDDLEWARE: Tuple = DEFAULT_MIDDLEWARES
ROOT_URLCONF: str = 'core.urls'
TEMPLATES: Tuple = DEFAULT_TEMPLATES
WSGI_APPLICATION: str = 'core.wsgi.application'
DATABASES: MappingProxyType = MappingProxyType({'default': env.db()})
AUTH_PASSWORD_VALIDATORS: Tuple = DEFAULT_VALIDATORS
# Security
SECURE_BROWSER_XSS_FILTER: bool = True
import asyncio
from typing import Callable
from loguru import logger
from app.core import config
from app.core.celery_app import celery
from app.video_website_spider import Dispatcher
if config.DSN: # pragma: no cover
import sentry_sdk
from sentry_sdk.integrations.celery import CeleryIntegration
logger.debug("setup sentry for celery")
sentry_sdk.init(
dsn=config.DSN, release=config.COMMIT_SHA, integrations=[CeleryIntegration()],
)
dispatcher = Dispatcher()
@celery.task
def submit_bangumi(subject_id: int, url: str):
dispatcher.subject(subject_id, url)
@celery.task
def submit_ep(ep_id: int, url: str):
dispatcher.ep(ep_id, url)
logger = logging.getLogger(__name__)
app = Celery()
app.config_from_object(tasks_settings)
def register_signal():
"""Adapted from `raven.contrib.celery.register_signal`. Remove args and
kwargs from logs so that keys aren't leaked to Sentry.
"""
def process_failure_signal(sender, task_id, *args, **kwargs):
with sentry_sdk.configure_scope() as scope:
scope.set_tag('task_id', task_id)
scope.set_tag('task', sender)
sentry_sdk.capture_exception()
task_failure.connect(process_failure_signal, weak=False)
sentry_dsn = config.get_nullable('SENTRY_DSN', None)
if sentry_dsn:
sentry_logging = LoggingIntegration(
level=logging.INFO, # Capture INFO level and above as breadcrumbs
event_level=None, # Do not send logs of any level as events
)
sentry_sdk.init(sentry_dsn,
release=__version__,
integrations=[CeleryIntegration(), sentry_logging])
register_signal()
def __init__(self, worker_beat, dsn, **options):
# initialize sentry for celery
if dsn:
sentry_sdk.init(dsn[0], integrations=[CeleryIntegration()])
import logging
import sentry_sdk
from envparse import env
from sentry_sdk.integrations.celery import CeleryIntegration
from sentry_sdk.integrations.falcon import FalconIntegration
from sentry_sdk.integrations.redis import RedisIntegration
# загружаем конфиг
env.read_envfile()
# включаем логи
logging.basicConfig(format='[%(asctime)s][%(levelname)s] %(message)s',
level=logging.INFO)
# formatter = json_log_formatter.JSONFormatter()
# json_handler = logging.FileHandler(filename='/var/log/wildsearch_app_log.json')
# json_handler.setFormatter(formatter)
logger = logging.getLogger(__name__)
# logger.addHandler(json_handler)
# включаем Sentry
if env('SENTRY_DSN', default=None) is not None:
sentry_sdk.init(env('SENTRY_DSN'), integrations=[FalconIntegration(), CeleryIntegration(), RedisIntegration()])
import logging
import os
import sentry_sdk
from celery import Celery
from sentry_sdk.integrations.celery import CeleryIntegration
from sentry_sdk.integrations.django import DjangoIntegration
from sentry_sdk.integrations.logging import LoggingIntegration
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings')
from django.conf import settings
# Sentry
if settings.IS_PROD:
sentry_logging = LoggingIntegration(level=settings.SENTRY_LOG_LEVEL, event_level=logging.ERROR)
sentry_sdk.init(dsn=settings.SENTRY_DSN, integrations=[sentry_logging, DjangoIntegration(), CeleryIntegration()])
app = Celery('{{ cookiecutter.project_slug }}')
app.config_from_object('django.conf:settings', namespace='CELERY')
app.autodiscover_tasks()
def ENV_SETTING(key, default):
return env.get_value(key, default=default)
if ENV_SETTING("ENABLE_SENTRY") == "True":
integrations = [DjangoIntegration()]
if ENV_SETTING("SENTRY_REDIS") == "True":
from sentry_sdk.integrations.redis import RedisIntegration
integrations.append(RedisIntegration())
if ENV_SETTING("SENTRY_CELERY") == "True":
from sentry_sdk.integrations.celery import CeleryIntegration
integrations.append(CeleryIntegration())
sentry_sdk.init(
dsn=ENV_SETTING("SENTRY_DSN"),
integrations=integrations,
send_default_pii=True,
)
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = Path(__file__).resolve().parents[2]
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/3.0/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = "y#swzfnq4br$0vmcjs@yej^&qmv_tualw#(awwi=he@=!@#&8u"
parent_dir = os.path.abspath(os.path.join(os.getcwd(), ".."))
sys.path.append(parent_dir)
sys.path.append(os.getcwd())
import config
from sql_persistence import session
from sql_persistence.interface import SQLPersistenceInterface
from eth_manager.ABIs import (dai_abi, erc20_abi, bancor_converter_abi,
bancor_network_abi)
from eth_manager.processor import TransactionProcessor
from eth_manager.contract_registry import ContractRegistry
from eth_manager import utils
sentry_sdk.init(config.SENTRY_SERVER_DSN, integrations=[CeleryIntegration()])
eth_config = dict()
eth_config['ethereum_chain_id'] = config.ETH_CHAIN_ID
eth_config['gas_price_gwei'] = config.ETH_GAS_PRICE
eth_config['gas_limit'] = config.ETH_GAS_LIMIT
ETH_CHECK_TRANSACTION_RETRIES = config.ETH_CHECK_TRANSACTION_RETRIES
ETH_CHECK_TRANSACTION_RETRIES_TIME_LIMIT = config.ETH_CHECK_TRANSACTION_RETRIES_TIME_LIMIT
ETH_CHECK_TRANSACTION_BASE_TIME = config.ETH_CHECK_TRANSACTION_BASE_TIME
celery_app = Celery('tasks',
broker=config.REDIS_URL,
backend=config.REDIS_URL,
task_serializer='json')
"version": 1,
"disable_existing_loggers": False,
"filters": {"require_debug_false": {"()": "django.utils.log.RequireDebugFalse"}},
"formatters": {
"verbose": {
"format": "%(levelname)s %(asctime)s %(module)s "
"%(process)d %(thread)d %(message)s"
}
},
"handlers": {
"console": {
"level": "DEBUG",
"class": "logging.StreamHandler",
"formatter": "verbose",
}
},
"root": {"level": "INFO", "handlers": ["console"]},
"loggers": {
"django.security.DisallowedHost": {
"level": "ERROR",
"handlers": ["console"],
"propagate": True,
}
},
}
# Sentry
# ------------------------------------------------------------------------------
SENTRY_DSN = env("SENTRY_DSN", default=None)
sentry_sdk.init(SENTRY_DSN, integrations=[DjangoIntegration(), CeleryIntegration()])
def create_app():
global app_created
if not app_created:
BlueprintsManager.register(app)
Migrate(app, db)
app.config.from_object(env('APP_CONFIG',
default='config.ProductionConfig'))
db.init_app(app)
_manager = Manager(app)
_manager.add_command('db', MigrateCommand)
if app.config['CACHING']:
cache.init_app(app, config={'CACHE_TYPE': 'simple'})
else:
cache.init_app(app, config={'CACHE_TYPE': 'null'})
stripe.api_key = 'SomeStripeKey'
app.secret_key = 'super secret key'
app.config['JSONIFY_PRETTYPRINT_REGULAR'] = False
app.config['FILE_SYSTEM_STORAGE_FILE_VIEW'] = 'static'
app.logger.addHandler(logging.StreamHandler(sys.stdout))
app.logger.setLevel(logging.ERROR)
# set up jwt
app.config['JWT_HEADER_TYPE'] = 'JWT'
app.config['JWT_ACCESS_TOKEN_EXPIRES'] = timedelta(days=1)
app.config['JWT_REFRESH_TOKEN_EXPIRES'] = timedelta(days=365)
app.config['JWT_ERROR_MESSAGE_KEY'] = 'error'
app.config['JWT_TOKEN_LOCATION'] = ['cookies', 'headers']
app.config['JWT_REFRESH_COOKIE_PATH'] = '/v1/auth/token/refresh'
app.config['JWT_SESSION_COOKIE'] = False
app.config['JWT_BLACKLIST_ENABLED'] = True
app.config['JWT_BLACKLIST_TOKEN_CHECKS'] = ['refresh']
_jwt = JWTManager(app)
_jwt.user_loader_callback_loader(jwt_user_loader)
_jwt.token_in_blacklist_loader(is_token_blacklisted)
# setup celery
app.config['CELERY_BROKER_URL'] = app.config['REDIS_URL']
app.config['CELERY_RESULT_BACKEND'] = app.config['CELERY_BROKER_URL']
app.config['CELERY_ACCEPT_CONTENT'] = ['json', 'application/text']
CORS(app, resources={r"/*": {"origins": "*"}})
AuthManager.init_login(app)
if app.config['TESTING'] and app.config['PROFILE']:
# Profiling
app.wsgi_app = ProfilerMiddleware(app.wsgi_app, restrictions=[30])
# development api
with app.app_context():
from app.api.admin_statistics_api.events import event_statistics
from app.api.auth import auth_routes
from app.api.attendees import attendee_misc_routes
from app.api.bootstrap import api_v1
from app.api.celery_tasks import celery_routes
from app.api.event_copy import event_copy
from app.api.exports import export_routes
from app.api.imports import import_routes
from app.api.uploads import upload_routes
from app.api.users import user_misc_routes
from app.api.orders import order_misc_routes
from app.api.role_invites import role_invites_misc_routes
from app.api.auth import authorised_blueprint
from app.api.admin_translations import admin_blueprint
from app.api.orders import alipay_blueprint
from app.api.settings import admin_misc_routes
from app.api.server_version import info_route
from app.api.custom.orders import ticket_blueprint
app.register_blueprint(api_v1)
app.register_blueprint(event_copy)
app.register_blueprint(upload_routes)
app.register_blueprint(export_routes)
app.register_blueprint(import_routes)
app.register_blueprint(celery_routes)
app.register_blueprint(auth_routes)
app.register_blueprint(event_statistics)
app.register_blueprint(user_misc_routes)
app.register_blueprint(attendee_misc_routes)
app.register_blueprint(order_misc_routes)
app.register_blueprint(role_invites_misc_routes)
app.register_blueprint(ticket_blueprint)
app.register_blueprint(authorised_blueprint)
app.register_blueprint(admin_blueprint)
app.register_blueprint(alipay_blueprint)
app.register_blueprint(admin_misc_routes)
app.register_blueprint(info_route)
add_engine_pidguard(db.engine)
if app.config['SQLALCHEMY_DATABASE_URI'].startswith("sqlite://"):
sqlite_datetime_fix()
sa.orm.configure_mappers()
if app.config['SERVE_STATIC']:
app.add_url_rule('/static/<path:filename>',
endpoint='static',
view_func=app.send_static_file)
# sentry
if not app_created and 'SENTRY_DSN' in app.config:
sentry_sdk.init(app.config['SENTRY_DSN'],
integrations=[
FlaskIntegration(),
RedisIntegration(),
CeleryIntegration(),
SqlalchemyIntegration()
])
# redis
redis_store.init_app(app)
# elasticsearch
if app.config['ENABLE_ELASTICSEARCH']:
client.init_app(app)
connections.add_connection('default', client.elasticsearch)
with app.app_context():
try:
cron_rebuild_events_elasticsearch.delay()
except Exception:
pass
app_created = True
return app, _manager, db, _jwt
def configure_sdk():
from sentry_sdk.integrations.logging import LoggingIntegration
from sentry_sdk.integrations.django import DjangoIntegration
from sentry_sdk.integrations.celery import CeleryIntegration
assert sentry_sdk.Hub.main.client is None
sdk_options = dict(settings.SENTRY_SDK_CONFIG)
relay_dsn = sdk_options.pop("relay_dsn", None)
internal_project_key = get_project_key()
upstream_dsn = sdk_options.pop("dsn", None)
if upstream_dsn:
upstream_transport = make_transport(get_options(dsn=upstream_dsn, **sdk_options))
else:
upstream_transport = None
if relay_dsn:
relay_transport = make_transport(get_options(dsn=relay_dsn, **sdk_options))
elif internal_project_key and internal_project_key.dsn_private:
relay_transport = make_transport(
get_options(dsn=internal_project_key.dsn_private, **sdk_options)
)
else:
relay_transport = None
def capture_event(event):
if event.get("type") == "transaction" and options.get(
"transaction-events.force-disable-internal-project"
):
return
# Upstream should get the event first because it is most isolated from
# the this sentry installation.
if upstream_transport:
metrics.incr("internal.captured.events.upstream")
# TODO(mattrobenolt): Bring this back safely.
# from sentry import options
# install_id = options.get('sentry:install-id')
# if install_id:
# event.setdefault('tags', {})['install-id'] = install_id
upstream_transport.capture_event(event)
if relay_transport and options.get("store.use-relay-dsn-sample-rate") == 1:
if is_current_event_safe():
metrics.incr("internal.captured.events.relay")
relay_transport.capture_event(event)
else:
metrics.incr("internal.uncaptured.events.relay", skip_internal=False)
if event.get("type") != "transaction":
sdk_logger.warn("internal-error.unsafe-stacktrace.relay")
sentry_sdk.init(
transport=capture_event,
integrations=[
DjangoIntegration(),
CeleryIntegration(),
LoggingIntegration(event_level=None),
RustInfoIntegration(),
],
traceparent_v2=True,
**sdk_options
)
encoded = o.name
elif isinstance(o, dt.datetime):
encoded = o.isoformat() + 'Z'
else:
try:
encoded = o.to_dict()
except AttributeError:
encoded = super().default(o)
return encoded
# Configura sentry
sentry_sdk.init(
dsn=os.environ['SENTRY_DSN'],
integrations=[FlaskIntegration(), CeleryIntegration()],
)
# Obtiene las variables de ambiente
STP_PRIVATE_LOCATION = os.environ['STP_PRIVATE_LOCATION']
STP_BUCKET_S3 = os.environ['STP_BUCKET_S3']
STP_PRIVATE_KEY = os.environ['STP_PRIVATE_KEY']
STP_WSDL = os.environ['STP_WSDL']
STP_EMPRESA = os.environ['STP_EMPRESA']
STP_KEY_PASSPHRASE = os.environ['STP_KEY_PASSPHRASE']
STP_PREFIJO = os.environ['STP_PREFIJO']
EDIT_HOSTS = os.environ['EDIT_HOSTS']
DATABASE_URI = os.environ['DATABASE_URI']
SPEID_ENV = os.getenv('SPEID_ENV', '')
import logging
import os
import sentry_sdk
from sentry_sdk.integrations.celery import CeleryIntegration
from sentry_sdk.integrations.django import DjangoIntegration
from sentry_sdk.integrations.logging import LoggingIntegration
from sentry_sdk.integrations.redis import RedisIntegration
from posthog.settings.base_variables import TEST
if not TEST:
if os.getenv("SENTRY_DSN"):
sentry_sdk.utils.MAX_STRING_LENGTH = 10_000_000
# https://docs.sentry.io/platforms/python/
sentry_logging = sentry_logging = LoggingIntegration(level=logging.INFO, event_level=None)
sentry_sdk.init(
dsn=os.environ["SENTRY_DSN"],
integrations=[DjangoIntegration(), CeleryIntegration(), RedisIntegration(), sentry_logging],
request_bodies="always",
send_default_pii=True,
environment=os.getenv("SENTRY_ENVIRONMENT", "production"),
)
"handlers": ["console"],
"propagate": False
},
"django.security.DisallowedHost": {
"level": "ERROR",
"handlers": ["console"],
"propagate": False,
},
},
}
# Sentry
# ------------------------------------------------------------------------------
SENTRY_DSN = env("SENTRY_DSN")
SENTRY_LOG_LEVEL = env.int("DJANGO_SENTRY_LOG_LEVEL", logging.INFO)
sentry_logging = LoggingIntegration(
level=SENTRY_LOG_LEVEL, # Capture info and above as breadcrumbs
event_level=logging.ERROR, # Send errors as events
)
integrations = [sentry_logging, DjangoIntegration(), CeleryIntegration()]
sentry_sdk.init(
dsn=SENTRY_DSN,
integrations=integrations,
environment=env("SENTRY_ENVIRONMENT", default="production"),
traces_sample_rate=env.float("SENTRY_TRACES_SAMPLE_RATE", default=0.0),
)
# Your stuff...
# ------------------------------------------------------------------------------
# Change this value if your application is running behind a proxy,
# e.g. HTTP_CF_Connecting_IP for Cloudflare or X_FORWARDED_FOR
REAL_IP_ENVIRON = os.environ.get("REAL_IP_ENVIRON", "REMOTE_ADDR")
# Slugs for menus precreated in Django migrations
DEFAULT_MENUS = {"top_menu_name": "navbar", "bottom_menu_name": "footer"}
# Slug for channel precreated in Django migrations
DEFAULT_CHANNEL_SLUG = os.environ.get("DEFAULT_CHANNEL_SLUG",
"default-channel")
# Sentry
sentry_sdk.utils.MAX_STRING_LENGTH = 4096
SENTRY_DSN = os.environ.get("SENTRY_DSN")
SENTRY_OPTS = {"integrations": [CeleryIntegration(), DjangoIntegration()]}
def SENTRY_INIT(dsn: str, sentry_opts: dict):
"""Init function for sentry.
Will only be called if SENTRY_DSN is not None, during core start, can be
overriden in separate settings file.
"""
sentry_sdk.init(dsn, **sentry_opts)
ignore_logger("graphql.execution.utils")
GRAPHENE = {
"RELAY_CONNECTION_ENFORCE_FIRST_OR_LAST": True,
"RELAY_CONNECTION_MAX_LIMIT": 100,
'stream': sys.stdout,
},
},
'root': {
'level': 'INFO',
'handlers': ['ecs'],
},
'loggers': {
'django': {
'level': 'INFO',
'handlers': ['ecs'],
'propagate': False,
},
'django.db.backends': {
'level': 'ERROR',
'handlers': ['ecs'],
'propagate': False,
},
},
}
sentry_sdk.init(
dsn=env('DJANGO_SENTRY_DSN'),
environment=env('SENTRY_ENVIRONMENT'),
integrations=[
CeleryIntegration(),
DjangoIntegration(),
],
in_app_include=['datahub'],
)
from .plone import fetch_content
from .plone import fetch_schema
from .removal import remove
from celery import Celery
import os
# sentry integration
sentry_dsn = os.environ.get("SENTRY_DSN", None)
sentry_project = os.environ.get("SENTRY_PROJECT", None)
if sentry_dsn is not None:
try:
import sentry_sdk
from sentry_sdk.integrations.celery import CeleryIntegration
sentry_sdk.init(sentry_dsn, integrations=[CeleryIntegration()])
logger.debug("Enable sentry logging.")
if sentry_project is not None:
with sentry_sdk.configure_scope() as scope:
scope.set_tag("project", sentry_project)
except ImportError:
logger.exception(
"sentry-logging configured, but package sentry_dsn not installed.\n"
"try: pip install sentry_dsn")
raise
# configure tasks
app = Celery("collective.elastic.ingest",
broker=os.environ.get("CELERY_BROKER"))
