def _get_rules(): """Return the current rules to be used with categorize.categorize. NB: Reloads the rules file at each call. """ rules_file = current_app.config.get('CATEGORIZE_RULES_FILE') if not rules_file: return None return categorize.load_rules(rules_file)
def _get_rules(): """Return the current rules to be used with categorize.categorize. NB: Reloads the rules file at each call. """ rules_file = current_app.config.get('CATEGORIZE_RULES_FILE') if not rules_file: return None return categorize.load_rules(rules_file)
def create_app(_read_config=True, **config): app = flask.Flask(__name__, static_folder=None, template_folder=os.path.join(PROJECT_ROOT, 'templates')) app.wsgi_app = ProxyFix(app.wsgi_app) # app.wsgi_app = TracerMiddleware(app.wsgi_app, app) # This key is insecure and you should override it on the server app.config['SECRET_KEY'] = 't\xad\xe7\xff%\xd2.\xfe\x03\x02=\xec\xaf\\2+\xb8=\xf7\x8a\x9aLD\xb1' app.config['SQLALCHEMY_COMMIT_ON_TEARDOWN'] = True app.config['SQLALCHEMY_DATABASE_URI'] = 'postgresql:///changes' app.config['SQLALCHEMY_POOL_SIZE'] = 60 app.config['SQLALCHEMY_MAX_OVERFLOW'] = 20 # required for flask-debugtoolbar and the db perf metrics we record app.config['SQLALCHEMY_RECORD_QUERIES'] = True app.config['REDIS_URL'] = 'redis://localhost/0' app.config['DEBUG'] = True app.config['HTTP_PORT'] = 5000 app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0 app.config['API_TRACEBACKS'] = True # Expiration delay between when a snapshot image becomes superceded and when # it becomes truly expired (and thus no longer included in the sync information # for any cluster that runs that particular image's plan) app.config['CACHED_SNAPSHOT_EXPIRATION_DELTA'] = timedelta(hours=1) # default snapshot ID to use when no project-specific active image available app.config['DEFAULT_SNAPSHOT'] = None app.config['SNAPSHOT_S3_BUCKET'] = None app.config['LXC_PRE_LAUNCH'] = None app.config['LXC_POST_LAUNCH'] = None # Location of artifacts server that is passed to changes-client # (include http:// or https://) app.config['ARTIFACTS_SERVER'] = None app.config['CHANGES_CLIENT_DEFAULT_BUILD_TYPE'] = 'legacy' # This is a hash from each build type (string identifiers used in # build step configuration) to a "build spec", a definition of # how to use changes-client to build. To use changes-client, the key # 'uses_client' must be set to True. # # Required build spec keys for client: # adapter -> basic or lxc # jenkins-command -> command to run from jenkins directly ($JENKINS_COMMAND) # commands -> array of hash from script -> string that represents a script # # Optional keys (lxc-only) # pre-launch -> lxc pre-launch script # post-launch -> lxc post-launch script # release -> lxc release app.config['CHANGES_CLIENT_BUILD_TYPES'] = { 'legacy': {'uses_client': False}, } app.config['CELERY_ACCEPT_CONTENT'] = ['changes_json'] app.config['CELERY_ACKS_LATE'] = True app.config['CELERY_BROKER_URL'] = 'redis://localhost/0' app.config['CELERY_DEFAULT_QUEUE'] = "default" app.config['CELERY_DEFAULT_EXCHANGE'] = "default" app.config['CELERY_DEFAULT_EXCHANGE_TYPE'] = "direct" app.config['CELERY_DEFAULT_ROUTING_KEY'] = "default" app.config['CELERY_DISABLE_RATE_LIMITS'] = True app.config['CELERY_IGNORE_RESULT'] = True app.config['CELERY_RESULT_BACKEND'] = None app.config['CELERY_RESULT_SERIALIZER'] = 'changes_json' app.config['CELERY_SEND_EVENTS'] = False app.config['CELERY_TASK_RESULT_EXPIRES'] = 1 app.config['CELERY_TASK_SERIALIZER'] = 'changes_json' app.config['CELERYD_PREFETCH_MULTIPLIER'] = 1 app.config['CELERYD_MAX_TASKS_PER_CHILD'] = 10000 # By default, Celery logs writes to stdout/stderr as WARNING, which # is a bit harsh considering that some of the code is code we don't # own calling 'print'. This flips the default back to INFO, which seems # more appropriate. Can be overridden by the Changes config. app.config['CELERY_REDIRECT_STDOUTS_LEVEL'] = 'INFO' app.config['CELERY_QUEUES'] = ( Queue('job.sync', routing_key='job.sync'), Queue('job.create', routing_key='job.create'), Queue('celery', routing_key='celery'), Queue('events', routing_key='events'), Queue('default', routing_key='default'), Queue('repo.sync', Exchange('fanout', 'fanout'), routing_key='repo.sync'), ) app.config['CELERY_ROUTES'] = { 'create_job': { 'queue': 'job.create', 'routing_key': 'job.create', }, 'sync_job': { 'queue': 'job.sync', 'routing_key': 'job.sync', }, 'sync_job_step': { 'queue': 'job.sync', 'routing_key': 'job.sync', }, 'sync_build': { 'queue': 'job.sync', 'routing_key': 'job.sync', }, 'check_repos': { 'queue': 'repo.sync', 'routing_key': 'repo.sync', }, 'sync_repo': { 'queue': 'repo.sync', 'routing_key': 'repo.sync', }, 'run_event_listener': { 'queue': 'events', 'routing_key': 'events', }, 'fire_signal': { 'queue': 'events', 'routing_key': 'events', }, } app.config['EVENT_LISTENERS'] = ( ('changes.listeners.mail.build_finished_handler', 'build.finished'), ('changes.listeners.green_build.build_finished_handler', 'build.finished'), ('changes.listeners.build_revision.revision_created_handler', 'revision.created'), ('changes.listeners.build_finished_notifier.build_finished_handler', 'build.finished'), ('changes.listeners.phabricator_listener.build_finished_handler', 'build.finished'), ('changes.listeners.analytics_notifier.build_finished_handler', 'build.finished'), ('changes.listeners.analytics_notifier.job_finished_handler', 'job.finished'), ('changes.listeners.snapshot_build.build_finished_handler', 'build.finished'), ) # restrict outbound notifications to the given domains app.config['MAIL_DOMAIN_WHITELIST'] = () app.config['DEBUG_TB_ENABLED'] = True # celerybeat must be running for our cleanup tasks to execute # e.g. celery worker -B app.config['CELERYBEAT_SCHEDULE'] = { 'cleanup-tasks': { 'task': 'cleanup_tasks', 'schedule': timedelta(minutes=1), }, 'check-repos': { 'task': 'check_repos', 'schedule': timedelta(minutes=2), }, 'aggregate-flaky-tests': { 'task': 'aggregate_flaky_tests', # Hour 7 GMT is midnight PST, hopefully a time of low load 'schedule': crontab(hour=7, minute=0), }, } app.config['CELERY_TIMEZONE'] = 'UTC' app.config['SENTRY_DSN'] = None app.config['SENTRY_INCLUDE_PATHS'] = [ 'changes', ] app.config['JENKINS_URL'] = None app.config['JENKINS_CLUSTERS'] = {} app.config['KOALITY_URL'] = None app.config['KOALITY_API_KEY'] = None app.config['GOOGLE_CLIENT_ID'] = None app.config['GOOGLE_CLIENT_SECRET'] = None app.config['GOOGLE_DOMAIN'] = None # must be a URL-safe base64-encoded 32-byte key app.config['COOKIE_ENCRYPTION_KEY'] = 'theDefaultKeyIs32BytesLongAndTotallyURLSafe=' app.config['REPO_ROOT'] = None app.config['DEFAULT_FILE_STORAGE'] = 'changes.storage.s3.S3FileStorage' app.config['S3_ACCESS_KEY'] = None app.config['S3_SECRET_KEY'] = None app.config['S3_BUCKET'] = None app.config['PHABRICATOR_HOST'] = None app.config['PHABRICATOR_USERNAME'] = None app.config['PHABRICATOR_CERT'] = None app.config['MAIL_DEFAULT_SENDER'] = 'changes@localhost' app.config['BASE_URI'] = 'http://localhost:5000' # if set to a string, most (all?) of the frontend js will make API calls # to the host this string is set to (e.g. http://changes.bigcompany.com) # THIS IS JUST FOR EASIER TESTING IN DEVELOPMENT. Although it won't even # work in prod: you'll have to start chrome with --disable-web-security to # make this work. Override this this in your changes.conf.py file app.config['WEBAPP_USE_ANOTHER_HOST'] = None # Custom changes content unique to your deployment. This is intended to # customize the look and feel, provide contextual help and add custom links # to other internal tools. You should put your files in webapp/custom and # link them here. # # e.g. /acmecorp-changes/changes.js # # Some of the custom_content hooks can show images. Assume that the webserver # is willing to serve any file within the directory of the js file app.config['WEBAPP_CUSTOM_JS'] = None # This can be a .less file. We import it after the variables.less, # so you can override them in your file # Note: if you change this and nothing seems to happen, try deleting # webapp/.webassets-cache and bundled.css. This probably won't happen, though app.config['WEBAPP_CUSTOM_CSS'] = None # In minutes, the timeout applied to jobs without a timeout specified at build time. # A timeout should nearly always be specified; this is just a safeguard so that # unspecified timeout doesn't mean "is allowed to run indefinitely". app.config['DEFAULT_JOB_TIMEOUT_MIN'] = 60 # Number of milliseconds a transaction can run before triggering a warning. app.config['TRANSACTION_MS_WARNING_THRESHOLD'] = 2500 # Maximum number of jobsteps to retry for a given job app.config['JOBSTEP_RETRY_MAX'] = 1 # we opt these users into the new ui...redirecting them if they # hit the homepage app.config['NEW_UI_OPTIN_USERS'] = set([]) # the PHID of the user creating quarantine tasks. We can use this to show # the list of open quarantine tasks inline app.config['QUARANTINE_PHID'] = None app.config.update(config) if _read_config: if os.environ.get('CHANGES_CONF'): # CHANGES_CONF=/etc/changes.conf.py app.config.from_envvar('CHANGES_CONF') else: # Look for ~/.changes/changes.conf.py path = os.path.normpath(os.path.expanduser('~/.changes/changes.conf.py')) app.config.from_pyfile(path, silent=True) # default the DSN for changes-client to the server's DSN app.config.setdefault('CLIENT_SENTRY_DSN', app.config['SENTRY_DSN']) if not app.config['BASE_URI']: raise ValueError('You must set ``BASE_URI`` in your configuration.') parsed_url = urlparse(app.config['BASE_URI']) app.config.setdefault('SERVER_NAME', parsed_url.netloc) app.config.setdefault('PREFERRED_URL_SCHEME', parsed_url.scheme) if app.debug: app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0 else: app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 30 app.url_map.converters['uuid'] = UUIDConverter # now that config is set up, let's ensure the CUSTOM_JS / CUSTOM_CSS # variables are safe (within the changes directory) and convert them to # absolute paths if app.config['WEBAPP_CUSTOM_CSS']: app.config['WEBAPP_CUSTOM_CSS'] = os.path.join( PROJECT_ROOT, 'webapp/custom/', app.config['WEBAPP_CUSTOM_CSS']) enforce_is_subdir( app.config['WEBAPP_CUSTOM_CSS'], os.path.join(PROJECT_ROOT, 'webapp/custom')) if app.config['WEBAPP_CUSTOM_JS']: app.config['WEBAPP_CUSTOM_JS'] = os.path.join( PROJECT_ROOT, 'webapp/custom/', app.config['WEBAPP_CUSTOM_JS']) enforce_is_subdir( app.config['WEBAPP_CUSTOM_JS'], os.path.join(PROJECT_ROOT, 'webapp/custom')) # init sentry first sentry.init_app(app) @app.before_request def capture_user(*args, **kwargs): if 'uid' in session: sentry.client.user_context({ 'id': session['uid'], 'email': session['email'], }) api.init_app(app) db.init_app(app) mail.init_app(app) queue.init_app(app) redis.init_app(app) statsreporter.init_app(app) configure_debug_toolbar(app) from raven.contrib.celery import register_signal, register_logger_signal register_signal(sentry.client) register_logger_signal(sentry.client, loglevel=logging.WARNING) # configure debug routes first if app.debug: configure_debug_routes(app) configure_templates(app) # TODO: these can be moved to wsgi app entrypoints configure_api_routes(app) app_static_root = configure_web_routes(app) # blueprint for our new v2 webapp blueprint = create_v2_blueprint(app, app_static_root) app.register_blueprint(blueprint, url_prefix='/v2') configure_jobs(app) configure_transaction_logging(app) rules_file = app.config.get('CATEGORIZE_RULES_FILE') if rules_file: # Fail at startup if we have a bad rules file. categorize.load_rules(rules_file) return app
def create_app(_read_config=True, **config): app = flask.Flask(__name__, static_folder=None, template_folder=os.path.join(PROJECT_ROOT, 'templates')) app.wsgi_app = ProxyFix(app.wsgi_app) # app.wsgi_app = TracerMiddleware(app.wsgi_app, app) # This key is insecure and you should override it on the server app.config['SECRET_KEY'] = 't\xad\xe7\xff%\xd2.\xfe\x03\x02=\xec\xaf\\2+\xb8=\xf7\x8a\x9aLD\xb1' app.config['SQLALCHEMY_COMMIT_ON_TEARDOWN'] = True app.config['SQLALCHEMY_DATABASE_URI'] = 'postgresql:///changes' app.config['SQLALCHEMY_POOL_SIZE'] = 60 app.config['SQLALCHEMY_MAX_OVERFLOW'] = 20 # required for flask-debugtoolbar app.config['SQLALCHEMY_RECORD_QUERIES'] = True app.config['REDIS_URL'] = 'redis://localhost/0' app.config['DEBUG'] = True app.config['HTTP_PORT'] = 5000 app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0 app.config['API_TRACEBACKS'] = True # default snapshot ID to use when no project-specific active image available app.config['DEFAULT_SNAPSHOT'] = None app.config['SNAPSHOT_S3_BUCKET'] = None app.config['LXC_PRE_LAUNCH'] = None app.config['LXC_POST_LAUNCH'] = None app.config['CELERY_ACCEPT_CONTENT'] = ['changes_json'] app.config['CELERY_ACKS_LATE'] = True app.config['CELERY_BROKER_URL'] = 'redis://localhost/0' app.config['CELERY_DEFAULT_QUEUE'] = "default" app.config['CELERY_DEFAULT_EXCHANGE'] = "default" app.config['CELERY_DEFAULT_EXCHANGE_TYPE'] = "direct" app.config['CELERY_DEFAULT_ROUTING_KEY'] = "default" app.config['CELERY_DISABLE_RATE_LIMITS'] = True app.config['CELERY_IGNORE_RESULT'] = True app.config['CELERY_RESULT_BACKEND'] = None app.config['CELERY_RESULT_SERIALIZER'] = 'changes_json' app.config['CELERY_SEND_EVENTS'] = False app.config['CELERY_TASK_RESULT_EXPIRES'] = 1 app.config['CELERY_TASK_SERIALIZER'] = 'changes_json' app.config['CELERYD_PREFETCH_MULTIPLIER'] = 1 app.config['CELERYD_MAX_TASKS_PER_CHILD'] = 10000 # By default, Celery logs writes to stdout/stderr as WARNING, which # is a bit harsh considering that some of the code is code we don't # own calling 'print'. This flips the default back to INFO, which seems # more appropriate. Can be overridden by the Changes config. app.config['CELERY_REDIRECT_STDOUTS_LEVEL'] = 'INFO' app.config['CELERY_QUEUES'] = ( Queue('job.sync', routing_key='job.sync'), Queue('job.create', routing_key='job.create'), Queue('celery', routing_key='celery'), Queue('events', routing_key='events'), Queue('default', routing_key='default'), Queue('repo.sync', Exchange('fanout', 'fanout'), routing_key='repo.sync'), ) app.config['CELERY_ROUTES'] = { 'create_job': { 'queue': 'job.create', 'routing_key': 'job.create', }, 'sync_job': { 'queue': 'job.sync', 'routing_key': 'job.sync', }, 'sync_job_step': { 'queue': 'job.sync', 'routing_key': 'job.sync', }, 'sync_build': { 'queue': 'job.sync', 'routing_key': 'job.sync', }, 'check_repos': { 'queue': 'repo.sync', 'routing_key': 'repo.sync', }, 'sync_repo': { 'queue': 'repo.sync', 'routing_key': 'repo.sync', }, 'run_event_listener': { 'queue': 'events', 'routing_key': 'events', }, 'fire_signal': { 'queue': 'events', 'routing_key': 'events', }, } app.config['EVENT_LISTENERS'] = ( ('changes.listeners.mail.build_finished_handler', 'build.finished'), ('changes.listeners.green_build.build_finished_handler', 'build.finished'), ('changes.listeners.build_revision.revision_created_handler', 'revision.created'), ('changes.listeners.phabricator_listener.build_finished_handler', 'build.finished'), ('changes.listeners.analytics_notifier.build_finished_handler', 'build.finished'), ('changes.listeners.log_processing.job_finished_handler', 'job.finished'), ) # restrict outbound notifications to the given domains app.config['MAIL_DOMAIN_WHITELIST'] = () app.config['DEBUG_TB_ENABLED'] = True # celerybeat must be running for our cleanup tasks to execute # e.g. celery worker -B app.config['CELERYBEAT_SCHEDULE'] = { 'cleanup-tasks': { 'task': 'cleanup_tasks', 'schedule': timedelta(minutes=1), }, 'check-repos': { 'task': 'check_repos', 'schedule': timedelta(minutes=2), }, } app.config['CELERY_TIMEZONE'] = 'UTC' app.config['SENTRY_DSN'] = None app.config['SENTRY_INCLUDE_PATHS'] = [ 'changes', ] app.config['JENKINS_AUTH'] = None app.config['JENKINS_URL'] = None app.config['JENKINS_TOKEN'] = None app.config['JENKINS_CLUSTERS'] = {} app.config['KOALITY_URL'] = None app.config['KOALITY_API_KEY'] = None app.config['GOOGLE_CLIENT_ID'] = None app.config['GOOGLE_CLIENT_SECRET'] = None app.config['GOOGLE_DOMAIN'] = None app.config['REPO_ROOT'] = None app.config['DEFAULT_FILE_STORAGE'] = 'changes.storage.s3.S3FileStorage' app.config['S3_ACCESS_KEY'] = None app.config['S3_SECRET_KEY'] = None app.config['S3_BUCKET'] = None app.config['PHABRICATOR_HOST'] = None app.config['PHABRICATOR_USERNAME'] = None app.config['PHABRICATOR_CERT'] = None app.config['MAIL_DEFAULT_SENDER'] = 'changes@localhost' app.config['BASE_URI'] = 'http://localhost:5000' # In minutes, the timeout applied to jobs without a timeout specified at build time. # A timeout should nearly always be specified; this is just a safeguard so that # unspecified timeout doesn't mean "is allowed to run indefinitely". app.config['DEFAULT_JOB_TIMEOUT_MIN'] = 60 app.config.update(config) if _read_config: if os.environ.get('CHANGES_CONF'): # CHANGES_CONF=/etc/changes.conf.py app.config.from_envvar('CHANGES_CONF') else: # Look for ~/.changes/changes.conf.py path = os.path.normpath(os.path.expanduser('~/.changes/changes.conf.py')) app.config.from_pyfile(path, silent=True) # default the DSN for changes-client to the server's DSN app.config.setdefault('CLIENT_SENTRY_DSN', app.config['SENTRY_DSN']) if not app.config['BASE_URI']: raise ValueError('You must set ``BASE_URI`` in your configuration.') parsed_url = urlparse(app.config['BASE_URI']) app.config.setdefault('SERVER_NAME', parsed_url.netloc) app.config.setdefault('PREFERRED_URL_SCHEME', parsed_url.scheme) if app.debug: app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0 else: app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 30 app.url_map.converters['uuid'] = UUIDConverter # init sentry first sentry.init_app(app) @app.before_request def capture_user(*args, **kwargs): if 'uid' in session: sentry.client.user_context({ 'id': session['uid'], 'email': session['email'], }) api.init_app(app) db.init_app(app) mail.init_app(app) queue.init_app(app) redis.init_app(app) statsreporter.init_app(app) configure_debug_toolbar(app) from raven.contrib.celery import register_signal, register_logger_signal register_signal(sentry.client) register_logger_signal(sentry.client, loglevel=logging.WARNING) # configure debug routes first if app.debug: configure_debug_routes(app) configure_templates(app) # TODO: these can be moved to wsgi app entrypoints configure_api_routes(app) configure_web_routes(app) configure_jobs(app) rules_file = app.config.get('CATEGORIZE_RULES_FILE') if rules_file: # Fail at startup if we have a bad rules file. categorize.load_rules(rules_file) return app
def create_app(_read_config=True, **config): app = flask.Flask(__name__, static_folder=None, template_folder=os.path.join(PROJECT_ROOT, 'templates')) # app.wsgi_app = TracerMiddleware(app.wsgi_app, app) # This key is insecure and you should override it on the server app.config['SECRET_KEY'] = 't\xad\xe7\xff%\xd2.\xfe\x03\x02=\xec\xaf\\2+\xb8=\xf7\x8a\x9aLD\xb1' app.config['SQLALCHEMY_COMMIT_ON_TEARDOWN'] = True app.config['SQLALCHEMY_DATABASE_URI'] = 'postgresql:///changes' app.config['SQLALCHEMY_POOL_SIZE'] = 60 app.config['SQLALCHEMY_MAX_OVERFLOW'] = 20 # required for flask-debugtoolbar and the db perf metrics we record app.config['SQLALCHEMY_RECORD_QUERIES'] = True app.config['REDIS_URL'] = 'redis://localhost/0' app.config['DEBUG'] = True app.config['HTTP_PORT'] = 5000 app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0 app.config['API_TRACEBACKS'] = True # Expiration delay between when a snapshot image becomes superceded and when # it becomes truly expired (and thus no longer included in the sync information # for any cluster that runs that particular image's plan) app.config['CACHED_SNAPSHOT_EXPIRATION_DELTA'] = timedelta(hours=1) # default snapshot ID to use when no project-specific active image available app.config['DEFAULT_SNAPSHOT'] = None app.config['SNAPSHOT_S3_BUCKET'] = None app.config['LXC_PRE_LAUNCH'] = None app.config['LXC_POST_LAUNCH'] = None # APT mirror URLs to use for new LXC containers created by changes-client. # NB: these aren't currently supported in the public changes-client repo. app.config['LXC_APT_MIRROR'] = None app.config['LXC_APT_SECURITY_MIRROR'] = None # name of the template to use for LXC (usually the name of a particular # Linux distro). Defaults to ubuntu. app.config['LXC_TEMPLATE'] = 'ubuntu' # Location of artifacts server that is passed to changes-client # (include http:// or https://) # # The default artifact server url uses a random uri which is expected to fail # without being overridden. This value is referenced in test code. app.config['ARTIFACTS_SERVER'] = 'http://localhost:1234' # the binary to use for running changes-client. Default is just # "changes-client", but can also be specified as e.g. a full path. app.config['CHANGES_CLIENT_BINARY'] = 'changes-client' app.config['CHANGES_CLIENT_DEFAULT_BUILD_TYPE'] = 'legacy' # Base URI to use for git repos that we want to clone (currently only used # for the "other_repos" buildstep config). The repo name is appended # directly to this, so it should already contain necessary colons and # slashes, etc. For example, if GIT_DEFAULT_BASE_URI is `[email protected]:` # and a repo is specified as `changes.git`, the clone url will be # `[email protected]:changes.git` app.config['GIT_DEFAULT_BASE_URI'] = None # Same as GIT_DEFAULT_BASE_URI but used for mercurial repos. app.config['MERCURIAL_DEFAULT_BASE_URI'] = None # This is a hash from each build type (string identifiers used in # build step configuration) to a "build spec", a definition of # how to use changes-client to build. To use changes-client, the key # 'uses_client' must be set to True. # # Required build spec keys for client: # adapter -> basic or lxc # jenkins-command -> command to run from jenkins directly ($JENKINS_COMMAND) # commands -> array of hash from script -> string that represents a script # # Optional keys (lxc-only) # pre-launch -> lxc pre-launch script # post-launch -> lxc post-launch script # release -> lxc release app.config['CHANGES_CLIENT_BUILD_TYPES'] = { 'legacy': {'uses_client': False}, } app.config['CELERY_ACCEPT_CONTENT'] = ['changes_json'] app.config['CELERY_ACKS_LATE'] = True app.config['CELERY_BROKER_URL'] = 'redis://localhost/0' app.config['CELERY_DEFAULT_QUEUE'] = "default" app.config['CELERY_DEFAULT_EXCHANGE'] = "default" app.config['CELERY_DEFAULT_EXCHANGE_TYPE'] = "direct" app.config['CELERY_DEFAULT_ROUTING_KEY'] = "default" app.config['CELERY_DISABLE_RATE_LIMITS'] = True app.config['CELERY_IGNORE_RESULT'] = True app.config['CELERY_RESULT_BACKEND'] = None app.config['CELERY_RESULT_SERIALIZER'] = 'changes_json' app.config['CELERY_SEND_EVENTS'] = False app.config['CELERY_TASK_RESULT_EXPIRES'] = 1 app.config['CELERY_TASK_SERIALIZER'] = 'changes_json' app.config['CELERYD_PREFETCH_MULTIPLIER'] = 1 app.config['CELERYD_MAX_TASKS_PER_CHILD'] = 10000 # By default, Celery logs writes to stdout/stderr as WARNING, which # is a bit harsh considering that some of the code is code we don't # own calling 'print'. This flips the default back to INFO, which seems # more appropriate. Can be overridden by the Changes config. app.config['CELERY_REDIRECT_STDOUTS_LEVEL'] = 'INFO' app.config['CELERY_QUEUES'] = ( Queue('job.sync', routing_key='job.sync'), Queue('job.create', routing_key='job.create'), Queue('celery', routing_key='celery'), Queue('events', routing_key='events'), Queue('default', routing_key='default'), Queue('repo.sync', Exchange('fanout', 'fanout'), routing_key='repo.sync'), Broadcast('repo.update'), ) app.config['CELERY_ROUTES'] = { 'create_job': { 'queue': 'job.create', 'routing_key': 'job.create', }, 'sync_job': { 'queue': 'job.sync', 'routing_key': 'job.sync', }, 'sync_job_step': { 'queue': 'job.sync', 'routing_key': 'job.sync', }, 'sync_build': { 'queue': 'job.sync', 'routing_key': 'job.sync', }, 'check_repos': { 'queue': 'repo.sync', 'routing_key': 'repo.sync', }, 'sync_repo': { 'queue': 'repo.sync', 'routing_key': 'repo.sync', }, 'run_event_listener': { 'queue': 'events', 'routing_key': 'events', }, 'fire_signal': { 'queue': 'events', 'routing_key': 'events', }, 'update_local_repos': { 'queue': 'repo.update', } } app.config['EVENT_LISTENERS'] = ( ('changes.listeners.mail.build_finished_handler', 'build.finished'), ('changes.listeners.green_build.build_finished_handler', 'build.finished'), ('changes.listeners.build_revision.revision_created_handler', 'revision.created'), ('changes.listeners.build_finished_notifier.build_finished_handler', 'build.finished'), ('changes.listeners.phabricator_listener.build_finished_handler', 'build.finished'), ('changes.listeners.analytics_notifier.build_finished_handler', 'build.finished'), ('changes.listeners.analytics_notifier.job_finished_handler', 'job.finished'), ('changes.listeners.stats_notifier.build_finished_handler', 'build.finished'), ('changes.listeners.snapshot_build.build_finished_handler', 'build.finished'), ) # restrict outbound notifications to the given domains app.config['MAIL_DOMAIN_WHITELIST'] = () app.config['DEBUG_TB_ENABLED'] = True # celerybeat must be running for our cleanup tasks to execute # e.g. celery worker -B app.config['CELERYBEAT_SCHEDULE'] = { 'cleanup-tasks': { 'task': 'cleanup_tasks', 'schedule': timedelta(minutes=1), }, 'check-repos': { 'task': 'check_repos', 'schedule': timedelta(minutes=2), }, 'aggregate-flaky-tests': { 'task': 'aggregate_flaky_tests', # Hour 7 GMT is midnight PST, hopefully a time of low load 'schedule': crontab(hour=7, minute=0), }, 'delete-old-data': { 'task': 'delete_old_data', 'schedule': timedelta(minutes=60), }, 'update-local-repos': { 'task': 'update_local_repos', 'schedule': timedelta(minutes=1), } } app.config['CELERY_TIMEZONE'] = 'UTC' app.config['SENTRY_DSN'] = None app.config['SENTRY_INCLUDE_PATHS'] = [ 'changes', ] app.config['KOALITY_URL'] = None app.config['KOALITY_API_KEY'] = None app.config['GOOGLE_CLIENT_ID'] = None app.config['GOOGLE_CLIENT_SECRET'] = None app.config['GOOGLE_DOMAIN'] = None # must be a URL-safe base64-encoded 32-byte key app.config['COOKIE_ENCRYPTION_KEY'] = 'theDefaultKeyIs32BytesLongAndTotallyURLSafe=' app.config['REPO_ROOT'] = None app.config['DEFAULT_FILE_STORAGE'] = 'changes.storage.s3.S3FileStorage' app.config['S3_ACCESS_KEY'] = None app.config['S3_SECRET_KEY'] = None app.config['S3_BUCKET'] = None app.config['PHABRICATOR_LINK_HOST'] = None app.config['PHABRICATOR_API_HOST'] = None app.config['PHABRICATOR_USERNAME'] = None app.config['PHABRICATOR_CERT'] = None # Configuration to access Zookeeper - currently used to discover mesos master leader instance # E.g., if mesos master is configured to talk to zk://zk1:2181,zk2:2181/mesos, # set ZOOKEEPER_HOSTS = 'zk1:2181,zk2:2181' # ZOOKEEPER_MESOS_MASTER_PATH = '/mesos' # # This is only used to control mesos slave offline/online status from within Changes # Comma-separated list of host:port (or ip:port) to Zookeeper instances. app.config['ZOOKEEPER_HOSTS'] = 'zk:2181' # Namespace within zookeeper where mesos master election is performed. app.config['ZOOKEEPER_MESOS_MASTER_PATH'] = '/mesos' # List of valid tables to be written to when reporting project analytics. # Analytics artifacts targeting tables not listed here will be considered invalid. app.config['ANALYTICS_PROJECT_TABLES'] = [] # URL any project analytics JSON entries will be posted to. # Entries will be posted as JSON, with the intended table specified as 'source' in the URL params. app.config['ANALYTICS_PROJECT_POST_URL'] = None app.config['SUPPORT_CONTACT'] = 'support' app.config['MAIL_DEFAULT_SENDER'] = 'changes@localhost' app.config['BASE_URI'] = 'http://localhost:5000' # if set to a string, most (all?) of the frontend js will make API calls # to the host this string is set to (e.g. http://changes.bigcompany.com) # THIS IS JUST FOR EASIER TESTING IN DEVELOPMENT. Although it won't even # work in prod: you'll have to start chrome with --disable-web-security to # make this work. Override this in your changes.conf.py file app.config['WEBAPP_USE_ANOTHER_HOST'] = None # Custom changes content unique to your deployment. This is intended to # customize the look and feel, provide contextual help and add custom links # to other internal tools. You should put your files in webapp/custom and # link them here. # # e.g. /acmecorp-changes/changes.js # # Some of the custom_content hooks can show images. Assume that the webserver # is willing to serve any file within the directory of the js file app.config['WEBAPP_CUSTOM_JS'] = None # This can be a .less file. We import it after the variables.less, # so you can override them in your file # Note: if you change this and nothing seems to happen, try deleting # webapp/.webassets-cache and bundled.css. This probably won't happen, though # If not specified, we will search for CUSTOM_CSS_FILE in the custom dir. app.config['WEBAPP_CUSTOM_CSS'] = None # In minutes, the timeout applied to jobs without a timeout specified at build time. # A timeout should nearly always be specified; this is just a safeguard so that # unspecified timeout doesn't mean "is allowed to run indefinitely". app.config['DEFAULT_JOB_TIMEOUT_MIN'] = 60 # Number of milliseconds a transaction can run before triggering a warning. app.config['TRANSACTION_MS_WARNING_THRESHOLD'] = 2500 # Hard maximum number of jobsteps to retry for a given job app.config['JOBSTEP_RETRY_MAX'] = 6 # Maximum number of machines that we'll retry jobsteps for. This allows us # to retry more jobsteps if it's always the same machine failing. app.config['JOBSTEP_MACHINE_RETRY_MAX'] = 2 # the PHID of the user creating quarantine tasks. We can use this to show # the list of open quarantine tasks inline app.config['QUARANTINE_PHID'] = None # The max length a test's output to be stored. If it is longer, the it will # be truncated. app.config['TEST_MESSAGE_MAX_LEN'] = 64 * 1024 # sources.list entry, format is: # deb uri distribution [component1] [component2] [...] # Used for bazel setup, so source should have bazel package. app.config['APT_SPEC'] = None # rsync source for encap # Example: rsync://example.com/encap/ app.config['ENCAP_RSYNC_URL'] = None # In some configurations, build slaves might not have access to the Changes API via the # normal address; if PATCH_BASE_URI is specified, it'll be used as the base URI for # PATCH_URI variables provided to build slaves. app.config['PATCH_BASE_URI'] = None # name of default cluster to use for autogenerated jobs app.config['DEFAULT_CLUSTER'] = None app.config.update(config) if _read_config: if os.environ.get('CHANGES_CONF'): # CHANGES_CONF=/etc/changes.conf.py app.config.from_envvar('CHANGES_CONF') else: # Look for ~/.changes/changes.conf.py path = os.path.normpath(os.path.expanduser('~/.changes/changes.conf.py')) app.config.from_pyfile(path, silent=True) # default the DSN for changes-client to the server's DSN app.config.setdefault('CLIENT_SENTRY_DSN', app.config['SENTRY_DSN']) # Backwards compatibility with old configs containing BASE_URI if 'WEB_BASE_URI' not in app.config and 'BASE_URI' in app.config: app.config['WEB_BASE_URI'] = app.config['BASE_URI'] if 'INTERNAL_BASE_URI' not in app.config and 'BASE_URI' in app.config: app.config['INTERNAL_BASE_URI'] = app.config['BASE_URI'] parsed_url = urlparse(app.config['WEB_BASE_URI']) app.config.setdefault('PREFERRED_URL_SCHEME', 'https') if app.debug: app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0 else: app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 30 app.url_map.converters['uuid'] = UUIDConverter # now that config is set up, let's ensure the CUSTOM_JS / CUSTOM_CSS # variables are safe (within the changes directory) and convert them to # absolute paths if app.config['WEBAPP_CUSTOM_CSS']: app.config['WEBAPP_CUSTOM_CSS'] = os.path.join( PROJECT_ROOT, 'webapp/custom/', app.config['WEBAPP_CUSTOM_CSS']) enforce_is_subdir( app.config['WEBAPP_CUSTOM_CSS'], os.path.join(PROJECT_ROOT, 'webapp/custom')) else: app.config['WEBAPP_CUSTOM_CSS'] = _find_custom_css() if app.config['WEBAPP_CUSTOM_JS']: app.config['WEBAPP_CUSTOM_JS'] = os.path.join( PROJECT_ROOT, 'webapp/custom/', app.config['WEBAPP_CUSTOM_JS']) enforce_is_subdir( app.config['WEBAPP_CUSTOM_JS'], os.path.join(PROJECT_ROOT, 'webapp/custom')) # init sentry first sentry.init_app(app) @app.before_request def capture_user(*args, **kwargs): from changes.api.auth import get_current_user user = get_current_user() if user is not None: sentry.client.user_context({ 'id': user.id, 'email': user.email, }) api.init_app(app) db.init_app(app) mail.init_app(app) queue.init_app(app) redis.init_app(app) statsreporter.init_app(app) configure_debug_toolbar(app) from raven.contrib.celery import register_signal, register_logger_signal register_signal(sentry.client) register_logger_signal(sentry.client, loglevel=logging.WARNING) # configure debug routes first if app.debug: configure_debug_routes(app) configure_templates(app) # TODO: these can be moved to wsgi app entrypoints configure_api_routes(app) configure_web_routes(app) configure_jobs(app) configure_transaction_logging(app) rules_file = app.config.get('CATEGORIZE_RULES_FILE') if rules_file: # Fail at startup if we have a bad rules file. categorize.load_rules(rules_file) import jinja2 webapp_template_folder = os.path.join(PROJECT_ROOT, 'webapp/html') template_folder = os.path.join(PROJECT_ROOT, 'templates') template_loader = jinja2.ChoiceLoader([ app.jinja_loader, jinja2.FileSystemLoader([webapp_template_folder, template_folder]) ]) app.jinja_loader = template_loader return app
def create_app(_read_config=True, **config): app = flask.Flask(__name__, static_folder=None, template_folder=os.path.join(PROJECT_ROOT, 'templates')) app.wsgi_app = ProxyFix(app.wsgi_app) # app.wsgi_app = TracerMiddleware(app.wsgi_app, app) # This key is insecure and you should override it on the server app.config['SECRET_KEY'] = 't\xad\xe7\xff%\xd2.\xfe\x03\x02=\xec\xaf\\2+\xb8=\xf7\x8a\x9aLD\xb1' app.config['SQLALCHEMY_COMMIT_ON_TEARDOWN'] = True app.config['SQLALCHEMY_DATABASE_URI'] = 'postgresql:///changes' app.config['SQLALCHEMY_POOL_SIZE'] = 60 app.config['SQLALCHEMY_MAX_OVERFLOW'] = 20 # required for flask-debugtoolbar and the db perf metrics we record app.config['SQLALCHEMY_RECORD_QUERIES'] = True app.config['REDIS_URL'] = 'redis://localhost/0' app.config['DEBUG'] = True app.config['HTTP_PORT'] = 5000 app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0 app.config['API_TRACEBACKS'] = True # Expiration delay between when a snapshot image becomes superceded and when # it becomes truly expired (and thus no longer included in the sync information # for any cluster that runs that particular image's plan) app.config['CACHED_SNAPSHOT_EXPIRATION_DELTA'] = timedelta(hours=1) # default snapshot ID to use when no project-specific active image available app.config['DEFAULT_SNAPSHOT'] = None app.config['SNAPSHOT_S3_BUCKET'] = None app.config['LXC_PRE_LAUNCH'] = None app.config['LXC_POST_LAUNCH'] = None # Location of artifacts server that is passed to changes-client # (include http:// or https://) # # The default artifact server url uses a random uri which is expected to fail # without being overridden. This value is referenced in test code. app.config['ARTIFACTS_SERVER'] = 'http://*****:*****@localhost' app.config['BASE_URI'] = 'http://localhost:5000' # if set to a string, most (all?) of the frontend js will make API calls # to the host this string is set to (e.g. http://changes.bigcompany.com) # THIS IS JUST FOR EASIER TESTING IN DEVELOPMENT. Although it won't even # work in prod: you'll have to start chrome with --disable-web-security to # make this work. Override this this in your changes.conf.py file app.config['WEBAPP_USE_ANOTHER_HOST'] = None # Custom changes content unique to your deployment. This is intended to # customize the look and feel, provide contextual help and add custom links # to other internal tools. You should put your files in webapp/custom and # link them here. # # e.g. /acmecorp-changes/changes.js # # Some of the custom_content hooks can show images. Assume that the webserver # is willing to serve any file within the directory of the js file app.config['WEBAPP_CUSTOM_JS'] = None # This can be a .less file. We import it after the variables.less, # so you can override them in your file # Note: if you change this and nothing seems to happen, try deleting # webapp/.webassets-cache and bundled.css. This probably won't happen, though app.config['WEBAPP_CUSTOM_CSS'] = None # In minutes, the timeout applied to jobs without a timeout specified at build time. # A timeout should nearly always be specified; this is just a safeguard so that # unspecified timeout doesn't mean "is allowed to run indefinitely". app.config['DEFAULT_JOB_TIMEOUT_MIN'] = 60 # Number of milliseconds a transaction can run before triggering a warning. app.config['TRANSACTION_MS_WARNING_THRESHOLD'] = 2500 # Maximum number of jobsteps to retry for a given job app.config['JOBSTEP_RETRY_MAX'] = 2 # we opt these users into the new ui...redirecting them if they # hit the homepage app.config['NEW_UI_OPTIN_USERS'] = set([]) # the PHID of the user creating quarantine tasks. We can use this to show # the list of open quarantine tasks inline app.config['QUARANTINE_PHID'] = None # The max length a test's output to be stored. If it is longer, the it will # be truncated. app.config['TEST_MESSAGE_MAX_LEN'] = 64 * 1024 app.config['USE_OLD_UI'] = False app.config.update(config) if _read_config: if os.environ.get('CHANGES_CONF'): # CHANGES_CONF=/etc/changes.conf.py app.config.from_envvar('CHANGES_CONF') else: # Look for ~/.changes/changes.conf.py path = os.path.normpath(os.path.expanduser('~/.changes/changes.conf.py')) app.config.from_pyfile(path, silent=True) # default the DSN for changes-client to the server's DSN app.config.setdefault('CLIENT_SENTRY_DSN', app.config['SENTRY_DSN']) if not app.config['BASE_URI']: raise ValueError('You must set ``BASE_URI`` in your configuration.') parsed_url = urlparse(app.config['BASE_URI']) app.config.setdefault('SERVER_NAME', parsed_url.netloc) app.config.setdefault('PREFERRED_URL_SCHEME', parsed_url.scheme) if app.debug: app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0 else: app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 30 app.url_map.converters['uuid'] = UUIDConverter # now that config is set up, let's ensure the CUSTOM_JS / CUSTOM_CSS # variables are safe (within the changes directory) and convert them to # absolute paths if app.config['WEBAPP_CUSTOM_CSS']: app.config['WEBAPP_CUSTOM_CSS'] = os.path.join( PROJECT_ROOT, 'webapp/custom/', app.config['WEBAPP_CUSTOM_CSS']) enforce_is_subdir( app.config['WEBAPP_CUSTOM_CSS'], os.path.join(PROJECT_ROOT, 'webapp/custom')) if app.config['WEBAPP_CUSTOM_JS']: app.config['WEBAPP_CUSTOM_JS'] = os.path.join( PROJECT_ROOT, 'webapp/custom/', app.config['WEBAPP_CUSTOM_JS']) enforce_is_subdir( app.config['WEBAPP_CUSTOM_JS'], os.path.join(PROJECT_ROOT, 'webapp/custom')) # init sentry first sentry.init_app(app) @app.before_request def capture_user(*args, **kwargs): if 'uid' in session: sentry.client.user_context({ 'id': session['uid'], 'email': session['email'], }) api.init_app(app) db.init_app(app) mail.init_app(app) queue.init_app(app) redis.init_app(app) statsreporter.init_app(app) configure_debug_toolbar(app) from raven.contrib.celery import register_signal, register_logger_signal register_signal(sentry.client) register_logger_signal(sentry.client, loglevel=logging.WARNING) # configure debug routes first if app.debug: configure_debug_routes(app) configure_templates(app) # TODO: these can be moved to wsgi app entrypoints configure_api_routes(app) configure_web_routes(app) configure_jobs(app) configure_transaction_logging(app) rules_file = app.config.get('CATEGORIZE_RULES_FILE') if rules_file: # Fail at startup if we have a bad rules file. categorize.load_rules(rules_file) import jinja2 webapp_template_folder = os.path.join(PROJECT_ROOT, 'webapp/html') template_folder = os.path.join(PROJECT_ROOT, 'templates') template_loader = jinja2.ChoiceLoader([ app.jinja_loader, jinja2.FileSystemLoader([webapp_template_folder, template_folder]) ]) app.jinja_loader = template_loader return app
def create_app(_read_config=True, **config): app = flask.Flask(__name__, static_folder=None, template_folder=os.path.join(PROJECT_ROOT, 'templates')) # app.wsgi_app = TracerMiddleware(app.wsgi_app, app) # This key is insecure and you should override it on the server app.config[ 'SECRET_KEY'] = 't\xad\xe7\xff%\xd2.\xfe\x03\x02=\xec\xaf\\2+\xb8=\xf7\x8a\x9aLD\xb1' app.config['SQLALCHEMY_COMMIT_ON_TEARDOWN'] = True app.config['SQLALCHEMY_DATABASE_URI'] = 'postgresql:///changes' app.config['SQLALCHEMY_POOL_SIZE'] = 60 app.config['SQLALCHEMY_MAX_OVERFLOW'] = 20 # required for flask-debugtoolbar and the db perf metrics we record app.config['SQLALCHEMY_RECORD_QUERIES'] = True app.config['REDIS_URL'] = 'redis://localhost/0' app.config['GROUPER_API_URL'] = 'https://localhost/' app.config['GROUPER_PERMISSIONS_ADMIN'] = 'changes.prod.admin' app.config[ 'GROUPER_PERMISSIONS_PROJECT_ADMIN'] = 'changes.prod.project.admin' app.config['GROUPER_EXCLUDED_ROLES'] = ['np-owner'] app.config['DEBUG'] = True app.config['HTTP_PORT'] = 5000 app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0 app.config['BAZEL_ARTIFACT_SUFFIX'] = '.bazel' app.config['BAZEL_TEST_OUTPUT_RELATIVE_PATH'] = 'bazel-testlogs/' app.config['API_TRACEBACKS'] = True # Expiration delay between when a snapshot image becomes superceded and when # it becomes truly expired (and thus no longer included in the sync information # for any cluster that runs that particular image's plan) app.config['CACHED_SNAPSHOT_EXPIRATION_DELTA'] = timedelta(hours=1) # default snapshot ID to use when no project-specific active image available app.config['DEFAULT_SNAPSHOT'] = None app.config['SNAPSHOT_S3_BUCKET'] = None app.config['LXC_PRE_LAUNCH'] = None app.config['LXC_POST_LAUNCH'] = None # APT mirror URLs to use for new LXC containers created by changes-client. # NB: these aren't currently supported in the public changes-client repo. app.config['LXC_APT_MIRROR'] = None app.config['LXC_APT_SECURITY_MIRROR'] = None # name of the template to use for LXC (usually the name of a particular # Linux distro). Defaults to ubuntu. app.config['LXC_TEMPLATE'] = 'ubuntu' # Location of artifacts server that is passed to changes-client # (include http:// or https://) # # The default artifact server url uses a random uri which is expected to fail # without being overridden. This value is referenced in test code. app.config['ARTIFACTS_SERVER'] = 'http://localhost:1234' # The default max artifact size handlers should be capable of processing. app.config['MAX_ARTIFACT_BYTES'] = 200 * 1024 * 1024 # The max artifact size the analytics json handler should be capable of processing. app.config['MAX_ARTIFACT_BYTES_ANALYTICS_JSON'] = 70 * 1024 * 1024 # the binary to use for running changes-client. Default is just # "changes-client", but can also be specified as e.g. a full path. app.config['CHANGES_CLIENT_BINARY'] = 'changes-client' app.config['CHANGES_CLIENT_DEFAULT_BUILD_TYPE'] = 'legacy' # Base URI to use for git repos that we want to clone (currently only used # for the "other_repos" buildstep config). The repo name is appended # directly to this, so it should already contain necessary colons and # slashes, etc. For example, if GIT_DEFAULT_BASE_URI is `[email protected]:` # and a repo is specified as `changes.git`, the clone url will be # `[email protected]:changes.git` app.config['GIT_DEFAULT_BASE_URI'] = None # Same as GIT_DEFAULT_BASE_URI but used for mercurial repos. app.config['MERCURIAL_DEFAULT_BASE_URI'] = None # This is a hash from each build type (string identifiers used in # build step configuration) to a "build spec", a definition of # how to use changes-client to build. To use changes-client, the key # 'uses_client' must be set to True. # # Required build spec keys for client: # adapter -> basic or lxc # jenkins-command -> command to run from jenkins directly ($JENKINS_COMMAND) # commands -> array of hash from script -> string that represents a script # # Optional keys (lxc-only) # pre-launch -> lxc pre-launch script # post-launch -> lxc post-launch script # release -> lxc release app.config['CHANGES_CLIENT_BUILD_TYPES'] = { 'legacy': { 'uses_client': False }, } app.config['CELERY_ACCEPT_CONTENT'] = ['changes_json'] app.config['CELERY_ACKS_LATE'] = True app.config['CELERY_BROKER_URL'] = 'redis://localhost/0' app.config['CELERY_DEFAULT_QUEUE'] = "default" app.config['CELERY_DEFAULT_EXCHANGE'] = "default" app.config['CELERY_DEFAULT_EXCHANGE_TYPE'] = "direct" app.config['CELERY_DEFAULT_ROUTING_KEY'] = "default" app.config['CELERY_DISABLE_RATE_LIMITS'] = True app.config['CELERY_IGNORE_RESULT'] = True app.config['CELERY_RESULT_BACKEND'] = None app.config['CELERY_RESULT_SERIALIZER'] = 'changes_json' app.config['CELERY_SEND_EVENTS'] = False app.config['CELERY_TASK_RESULT_EXPIRES'] = 1 app.config['CELERY_TASK_SERIALIZER'] = 'changes_json' app.config['CELERYD_PREFETCH_MULTIPLIER'] = 1 app.config['CELERYD_MAX_TASKS_PER_CHILD'] = 10000 # By default, Celery logs writes to stdout/stderr as WARNING, which # is a bit harsh considering that some of the code is code we don't # own calling 'print'. This flips the default back to INFO, which seems # more appropriate. Can be overridden by the Changes config. app.config['CELERY_REDIRECT_STDOUTS_LEVEL'] = 'INFO' app.config['CELERY_QUEUES'] = ( Queue('job.sync', routing_key='job.sync'), Queue('job.create', routing_key='job.create'), Queue('celery', routing_key='celery'), Queue('events', routing_key='events'), Queue('default', routing_key='default'), Queue('delete', routing_key='delete'), Queue('repo.sync', Exchange('fanout', 'fanout'), routing_key='repo.sync'), Queue('grouper.sync', routing_key='grouper.sync'), Broadcast('repo.update'), ) app.config['CELERY_ROUTES'] = { 'create_job': { 'queue': 'job.create', 'routing_key': 'job.create', }, 'sync_job': { 'queue': 'job.sync', 'routing_key': 'job.sync', }, 'sync_job_step': { 'queue': 'job.sync', 'routing_key': 'job.sync', }, 'sync_build': { 'queue': 'job.sync', 'routing_key': 'job.sync', }, 'check_repos': { 'queue': 'repo.sync', 'routing_key': 'repo.sync', }, 'sync_grouper': { 'queue': 'grouper.sync', 'routing_key': 'grouper.sync', }, 'sync_repo': { 'queue': 'repo.sync', 'routing_key': 'repo.sync', }, 'run_event_listener': { 'queue': 'events', 'routing_key': 'events', }, 'fire_signal': { 'queue': 'events', 'routing_key': 'events', }, 'update_local_repos': { 'queue': 'repo.update', }, 'delete_old_data': { 'queue': 'delete', 'routing_key': 'delete', }, 'delete_old_data_10m': { 'queue': 'delete', 'routing_key': 'delete', }, 'delete_old_data_5h_delayed': { 'queue': 'delete', 'routing_key': 'delete', }, } app.config['EVENT_LISTENERS'] = ( ('changes.listeners.mail.build_finished_handler', 'build.finished'), ('changes.listeners.green_build.revision_result_updated_handler', 'revision_result.updated'), ('changes.listeners.build_revision.revision_created_handler', 'revision.created'), ('changes.listeners.build_finished_notifier.build_finished_handler', 'build.finished'), ('changes.listeners.phabricator_listener.build_finished_handler', 'build.finished'), ('changes.listeners.analytics_notifier.build_finished_handler', 'build.finished'), ('changes.listeners.analytics_notifier.job_finished_handler', 'job.finished'), ('changes.listeners.revision_result.revision_result_build_finished_handler', 'build.finished'), ('changes.listeners.stats_notifier.build_finished_handler', 'build.finished'), ('changes.listeners.snapshot_build.build_finished_handler', 'build.finished'), ) # restrict outbound notifications to the given domains app.config['MAIL_DOMAIN_WHITELIST'] = () app.config['DEBUG_TB_ENABLED'] = True app.config['DEBUG_TB_PANELS'] = ( 'flask_debugtoolbar.panels.versions.VersionDebugPanel', 'flask_debugtoolbar.panels.timer.TimerDebugPanel', 'flask_debugtoolbar.panels.headers.HeaderDebugPanel', 'flask_debugtoolbar.panels.request_vars.RequestVarsDebugPanel', # Disable the config vars panel by default; it can contain sensitive information. # 'flask_debugtoolbar.panels.config_vars.ConfigVarsDebugPanel', 'flask_debugtoolbar.panels.template.TemplateDebugPanel', 'flask_debugtoolbar.panels.sqlalchemy.SQLAlchemyDebugPanel', 'flask_debugtoolbar.panels.logger.LoggingPanel', 'flask_debugtoolbar.panels.profiler.ProfilerDebugPanel') # celerybeat must be running for our cleanup tasks to execute # e.g. celery worker -B app.config['CELERYBEAT_SCHEDULE'] = { 'cleanup-tasks': { 'task': 'cleanup_tasks', 'schedule': timedelta(minutes=1), }, 'check-repos': { 'task': 'check_repos', 'schedule': timedelta(minutes=2), }, 'sync-grouper': { 'task': 'sync_grouper', 'schedule': timedelta(minutes=1), }, 'aggregate-flaky-tests': { 'task': 'aggregate_flaky_tests', # Hour 7 GMT is midnight PST, hopefully a time of low load 'schedule': crontab(hour=7, minute=0), }, 'delete-old-data-10m': { 'task': 'delete_old_data_10m', 'schedule': timedelta(minutes=10), }, 'delete-old-data-5h-delayed': { 'task': 'delete_old_data_5h_delayed', # This task runs every 4 hours but looks at 5 hours worth of tests # so consecutive runs will look at sets of tests that will overlap. # This is to make it unlikely to miss tests in between. # # While this is looking at 5 hours worth of tests, this should not be long running # as the shorter delete tasks will catch most cases and this checks # a time frame that should've been cleaned by them already. 'schedule': crontab(hour='*/4'), }, 'update-local-repos': { 'task': 'update_local_repos', 'schedule': timedelta(minutes=1), } } app.config['CELERY_TIMEZONE'] = 'UTC' app.config['SENTRY_DSN'] = None app.config['SENTRY_INCLUDE_PATHS'] = [ 'changes', ] app.config['KOALITY_URL'] = None app.config['KOALITY_API_KEY'] = None app.config['GOOGLE_CLIENT_ID'] = None app.config['GOOGLE_CLIENT_SECRET'] = None app.config['GOOGLE_DOMAIN'] = None # must be a URL-safe base64-encoded 32-byte key app.config[ 'COOKIE_ENCRYPTION_KEY'] = 'theDefaultKeyIs32BytesLongAndTotallyURLSafe=' app.config['REPO_ROOT'] = None app.config['DEFAULT_FILE_STORAGE'] = 'changes.storage.s3.S3FileStorage' app.config['S3_ACCESS_KEY'] = None app.config['S3_SECRET_KEY'] = None app.config['S3_BUCKET'] = None app.config['PHABRICATOR_LINK_HOST'] = None app.config['PHABRICATOR_API_HOST'] = None app.config['PHABRICATOR_USERNAME'] = None app.config['PHABRICATOR_CERT'] = None # Configuration to access Zookeeper - currently used to discover mesos master leader instance # E.g., if mesos master is configured to talk to zk://zk1:2181,zk2:2181/mesos, # set ZOOKEEPER_HOSTS = 'zk1:2181,zk2:2181' # ZOOKEEPER_MESOS_MASTER_PATH = '/mesos' # # This is only used to control mesos slave offline/online status from within Changes # Comma-separated list of host:port (or ip:port) to Zookeeper instances. app.config['ZOOKEEPER_HOSTS'] = 'zk:2181' # Namespace within zookeeper where mesos master election is performed. app.config['ZOOKEEPER_MESOS_MASTER_PATH'] = '/mesos' # List of valid tables to be written to when reporting project analytics. # Analytics artifacts targeting tables not listed here will be considered invalid. app.config['ANALYTICS_PROJECT_TABLES'] = [] # URL any project analytics JSON entries will be posted to. # Entries will be posted as JSON, with the intended table specified as 'source' in the URL params. app.config['ANALYTICS_PROJECT_POST_URL'] = None app.config['SUPPORT_CONTACT'] = 'support' app.config['MAIL_DEFAULT_SENDER'] = 'changes@localhost' app.config['BASE_URI'] = 'http://localhost:5000' # if set to a string, most (all?) of the frontend js will make API calls # to the host this string is set to (e.g. http://changes.bigcompany.com) # THIS IS JUST FOR EASIER TESTING IN DEVELOPMENT. Although it won't even # work in prod: you'll have to start chrome with --disable-web-security to # make this work. Override this in your changes.conf.py file app.config['WEBAPP_USE_ANOTHER_HOST'] = None # Custom changes content unique to your deployment. This is intended to # customize the look and feel, provide contextual help and add custom links # to other internal tools. You should put your files in webapp/custom and # link them here. # # e.g. /acmecorp-changes/changes.js # # Some of the custom_content hooks can show images. Assume that the webserver # is willing to serve any file within the directory of the js file app.config['WEBAPP_CUSTOM_JS'] = None # This can be a .less file. We import it after the variables.less, # so you can override them in your file # Note: if you change this and nothing seems to happen, try deleting # webapp/.webassets-cache and bundled.css. This probably won't happen, though # If not specified, we will search for CUSTOM_CSS_FILE in the custom dir. app.config['WEBAPP_CUSTOM_CSS'] = None # In minutes, the timeout applied to jobs without a timeout specified at build time. # A timeout should nearly always be specified; this is just a safeguard so that # unspecified timeout doesn't mean "is allowed to run indefinitely". app.config['DEFAULT_JOB_TIMEOUT_MIN'] = 60 # Number of milliseconds a transaction can run before triggering a warning. app.config['TRANSACTION_MS_WARNING_THRESHOLD'] = 2500 # Hard maximum number of jobsteps to retry for a given job app.config['JOBSTEP_RETRY_MAX'] = 6 # Maximum number of machines that we'll retry jobsteps for. This allows us # to retry more jobsteps if it's always the same machine failing. app.config['JOBSTEP_MACHINE_RETRY_MAX'] = 2 # the PHID of the user creating quarantine tasks. We can use this to show # the list of open quarantine tasks inline app.config['QUARANTINE_PHID'] = None # The max length a test's output to be stored. If it is longer, the it will # be truncated. app.config['TEST_MESSAGE_MAX_LEN'] = 64 * 1024 # List of packages needed to install bazel and any environment. app.config['BAZEL_APT_PKGS'] = ['bazel'] # rsync source for encap # Example: rsync://example.com/encap/ app.config['ENCAP_RSYNC_URL'] = None # In some configurations, build slaves might not have access to the Changes API via the # normal address; if PATCH_BASE_URI is specified, it'll be used as the base URI for # PATCH_URI variables provided to build slaves. app.config['PATCH_BASE_URI'] = None # name of default cluster to use for autogenerated jobs app.config['DEFAULT_CLUSTER'] = None # Maximum number of cpus allowed for a bazel executor. Since we expose `bazel.cpus` to # the user, this number needs to be bounded to avoid runaway resource allocation (by always # allocating large chunks of resources, like 12-16 cores), and to avoid invalid configuration # (like, requesting more cpus than available on a single slave, typically 32) app.config['MAX_CPUS_PER_EXECUTOR'] = 16 # Minimum memory allowed per executor (in MB) app.config['MIN_MEM_MB_PER_EXECUTOR'] = 1024 # Maximum memory allowed per executor (in MB) app.config['MAX_MEM_MB_PER_EXECUTOR'] = 16384 # Maximum number of bazel executors allowed. app.config['MAX_EXECUTORS'] = 10 # Absolute path to Bazel root (passed via --output_root to Bazel) # Storing bazel cache in tmpfs could be a bad idea because: # - tmpfs means any files stored here will be stored purely in RAM and will eat into container limits # - these containers are not persisted from the snapshot # # Bazel will create parent directories (if the user has appropriate permissions), if missing. app.config['BAZEL_ROOT_PATH'] = '/tmp/bazel_changes' # List of mandatory flags to be passed to `bazel test` app.config['BAZEL_MANDATORY_TEST_FLAGS'] = [ '--spawn_strategy=sandboxed', '--genrule_strategy=sandboxed', '--keep_going', ] app.config['BAZEL_ADDITIONAL_TEST_FLAGS_WHITELIST_REGEX'] = [ r'^--test_env=[A-Za-z0-9=]+', r'^--test_arg=[A-Za-z0-9=]+', r'^--define=[A-Za-z0-9=]+', ] app.config['SELECTIVE_TESTING_PROPAGATION_LIMIT'] = 30 app.config['SELECTIVE_TESTING_ENABLED'] = False # Debug config entries passed to every autobazel jobstep app.config['BAZEL_DEBUG_CONFIG'] = {} # Extra test setup commands to be executed before collect-targets or `bazel test` invocations. app.config['BAZEL_EXTRA_SETUP_CMD'] = ['exit 0'] # Jobsteps go from 'pending_allocation' to 'allocated' once an external scheduler claims them, and # once they begin running they're updated to 'in_progress'. If the scheduler somehow fails or drops # the task, this value is used to time out the 'allocated' status and revert back to 'pending_allocation'. # For current and expected schedulers, we don't allocate unless we think we can execute immediately, so # a 3 minute timeout is conservative and should be safe. app.config['JOBSTEP_ALLOCATION_TIMEOUT_SECONDS'] = 3 * 60 app.config.update(config) if _read_config: if os.environ.get('CHANGES_CONF'): # CHANGES_CONF=/etc/changes.conf.py app.config.from_envvar('CHANGES_CONF') else: # Look for ~/.changes/changes.conf.py path = os.path.normpath( os.path.expanduser('~/.changes/changes.conf.py')) app.config.from_pyfile(path, silent=True) # default the DSN for changes-client to the server's DSN app.config.setdefault('CLIENT_SENTRY_DSN', app.config['SENTRY_DSN']) # Backwards compatibility with old configs containing BASE_URI if 'WEB_BASE_URI' not in app.config and 'BASE_URI' in app.config: app.config['WEB_BASE_URI'] = app.config['BASE_URI'] if 'INTERNAL_BASE_URI' not in app.config and 'BASE_URI' in app.config: app.config['INTERNAL_BASE_URI'] = app.config['BASE_URI'] parsed_url = urlparse(app.config['WEB_BASE_URI']) app.config.setdefault('PREFERRED_URL_SCHEME', 'https') if app.debug: app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0 else: app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 30 app.url_map.converters['uuid'] = UUIDConverter # now that config is set up, let's ensure the CUSTOM_JS / CUSTOM_CSS # variables are safe (within the changes directory) and convert them to # absolute paths if app.config['WEBAPP_CUSTOM_CSS']: app.config['WEBAPP_CUSTOM_CSS'] = os.path.join( PROJECT_ROOT, 'webapp/custom/', app.config['WEBAPP_CUSTOM_CSS']) enforce_is_subdir(app.config['WEBAPP_CUSTOM_CSS'], os.path.join(PROJECT_ROOT, 'webapp/custom')) else: app.config['WEBAPP_CUSTOM_CSS'] = _find_custom_css() if app.config['WEBAPP_CUSTOM_JS']: app.config['WEBAPP_CUSTOM_JS'] = os.path.join( PROJECT_ROOT, 'webapp/custom/', app.config['WEBAPP_CUSTOM_JS']) enforce_is_subdir(app.config['WEBAPP_CUSTOM_JS'], os.path.join(PROJECT_ROOT, 'webapp/custom')) # init sentry first sentry.init_app(app) @app.before_request def capture_user(*args, **kwargs): from changes.api.auth import get_current_user user = get_current_user() if user is not None: sentry.client.user_context({ 'id': user.id, 'email': user.email, }) api.init_app(app) db.init_app(app) mail.init_app(app) queue.init_app(app) redis.init_app(app) statsreporter.init_app(app) configure_debug_toolbar(app) from raven.contrib.celery import register_signal, register_logger_signal register_signal(sentry.client) register_logger_signal(sentry.client, loglevel=logging.WARNING) # configure debug routes first if app.debug: configure_debug_routes(app) configure_templates(app) # TODO: these can be moved to wsgi app entrypoints configure_api_routes(app) configure_web_routes(app) configure_jobs(app) configure_transaction_logging(app) rules_file = app.config.get('CATEGORIZE_RULES_FILE') if rules_file: # Fail at startup if we have a bad rules file. categorize.load_rules(rules_file) import jinja2 webapp_template_folder = os.path.join(PROJECT_ROOT, 'webapp/html') template_folder = os.path.join(PROJECT_ROOT, 'templates') template_loader = jinja2.ChoiceLoader([ app.jinja_loader, jinja2.FileSystemLoader([webapp_template_folder, template_folder]) ]) app.jinja_loader = template_loader return app