def webserver(args): print(settings.HEADER) from airflow.www.app import cached_app app = cached_app(conf) workers = args.workers or conf.get('webserver', 'workers') worker_timeout = (args.worker_timeout or conf.get('webserver', 'webserver_worker_timeout')) if args.debug: print("Starting the web server on port {0} and host {1}.".format( args.port, args.hostname)) app.run(debug=True, port=args.port, host=args.hostname) else: pid, stdout, stderr, log_file = setup_locations("webserver", pid=args.pid) print('Running the Gunicorn server with {workers} {args.workerclass}' 'workers on host {args.hostname} and port ' '{args.port} with a timeout of {worker_timeout}...'.format( **locals())) sp = subprocess.Popen([ 'gunicorn', '-w', str(args.workers), '-k', str(args.workerclass), '-t', str(args.worker_timeout), '-b', args.hostname + ':' + str(args.port), '-n', 'airflow-webserver', '--pid', pid, 'airflow.www.app:cached_app()' ]) if args.foreground: sp.wait()
def _import_users(users_list): # pylint: disable=redefined-outer-name appbuilder = cached_app().appbuilder # pylint: disable=no-member users_created = [] users_updated = [] for user in users_list: roles = [] for rolename in user['roles']: role = appbuilder.sm.find_role(rolename) if not role: valid_roles = appbuilder.sm.get_all_roles() print( f"Error: '{rolename}' is not a valid role. Valid roles are: {valid_roles}" ) sys.exit(1) else: roles.append(role) required_fields = [ 'username', 'firstname', 'lastname', 'email', 'roles' ] for field in required_fields: if not user.get(field): print("Error: '{}' is a required field, but was not " "specified".format(field)) sys.exit(1) existing_user = appbuilder.sm.find_user(email=user['email']) if existing_user: print("Found existing user with email '{}'".format(user['email'])) existing_user.roles = roles existing_user.first_name = user['firstname'] existing_user.last_name = user['lastname'] if existing_user.username != user['username']: print("Error: Changing the username is not allowed - " "please delete and recreate the user with " "email '{}'".format(user['email'])) sys.exit(1) appbuilder.sm.update_user(existing_user) users_updated.append(user['email']) else: print("Creating new user with email '{}'".format(user['email'])) appbuilder.sm.add_user( username=user['username'], first_name=user['firstname'], last_name=user['lastname'], email=user['email'], role=roles[0], # add_user() requires exactly 1 role ) if len(roles) > 1: new_user = appbuilder.sm.find_user(email=user['email']) new_user.roles = roles appbuilder.sm.update_user(new_user) users_created.append(user['email']) return users_created, users_updated
def test_should_respect_base_url_when_proxy_fix_and_base_url_is_set_up_but_headers_missing( self): app = application.cached_app(testing=True) app.url_map.add(Rule("/debug", endpoint="debug")) def debug_view(): from flask import request # Should use original REMOTE_ADDR assert request.remote_addr == '192.168.0.1' # Should respect base_url assert request.url == "http://invalid:9000/internal-client/debug" return Response("success") app.view_functions['debug'] = debug_view new_environ = { "PATH_INFO": "/internal-client/debug", "REMOTE_ADDR": "192.168.0.1", "HTTP_HOST": "invalid:9000", } environ = create_environ(environ_overrides=new_environ) response = Response.from_app(app, environ) assert b"success" == response.get_data() assert response.status_code == 200
def users_create(args): """Creates new user in the DB""" appbuilder = cached_app().appbuilder # pylint: disable=no-member role = appbuilder.sm.find_role(args.role) if not role: valid_roles = appbuilder.sm.get_all_roles() raise SystemExit('{} is not a valid role. Valid roles are: {}'.format( args.role, valid_roles)) if args.use_random_password: password = ''.join(random.choice(string.printable) for _ in range(16)) elif args.password: password = args.password else: password = getpass.getpass('Password:'******'Repeat for confirmation:') if password != password_confirmation: raise SystemExit('Passwords did not match!') if appbuilder.sm.find_user(args.username): print('{} already exist in the db'.format(args.username)) return user = appbuilder.sm.add_user(args.username, args.firstname, args.lastname, args.email, role, password) if user: print('{} user {} created.'.format(args.role, args.username)) else: raise SystemExit('Failed to create user.')
def webserver(args): print(settings.HEADER) from airflow.www.app import cached_app app = cached_app(conf) workers = args.workers or conf.get('webserver', 'workers') worker_timeout = (args.worker_timeout or conf.get('webserver', 'webserver_worker_timeout')) if args.debug: print("Starting the web server on port {0} and host {1}.".format( args.port, args.hostname)) app.run(debug=True, port=args.port, host=args.hostname) else: pid, stdout, stderr, log_file = setup_locations("webserver", pid=args.pid) print('Running the Gunicorn server with {workers} {args.workerclass}' 'workers on host {args.hostname} and port ' '{args.port} with a timeout of {worker_timeout}...'.format( **locals())) run_args = [ 'gunicorn', '-w ' + str(args.workers), '-k ' + str(args.workerclass), '-t ' + str(args.worker_timeout), '-b ' + args.hostname + ':' + str(args.port), '-n ' + 'airflow-webserver', '-p ' + str(pid) ] if args.daemon: run_args.append("-D") module = "airflow.www.app:cached_app()".encode() run_args.append(module) os.execvp('gunicorn', run_args)
def test_should_respect_base_url_ignore_proxy_headers(self): app = application.cached_app(testing=True) app.url_map.add(Rule("/debug", endpoint="debug")) def debug_view(): from flask import request # Should ignore HTTP_X_FORWARDED_FOR assert request.remote_addr == '192.168.0.2' # Should ignore HTTP_X_FORWARDED_PROTO, HTTP_X_FORWARDED_HOST, HTTP_X_FORWARDED_PORT, # HTTP_X_FORWARDED_PREFIX assert request.url == 'http://invalid:9000/internal-client/debug' return Response("success") app.view_functions['debug'] = debug_view new_environ = { "PATH_INFO": "/internal-client/debug", "REMOTE_ADDR": "192.168.0.2", "HTTP_HOST": "invalid:9000", "HTTP_X_FORWARDED_FOR": "192.168.0.1", "HTTP_X_FORWARDED_PROTO": "https", "HTTP_X_FORWARDED_HOST": "valid", "HTTP_X_FORWARDED_PORT": "445", "HTTP_X_FORWARDED_PREFIX": "/proxy-prefix", } environ = create_environ(environ_overrides=new_environ) response = Response.from_app(app, environ) assert b"success" == response.get_data() assert response.status_code == 200
def webserver(args): print(settings.HEADER) from airflow.www.app import cached_app app = cached_app(conf) workers = args.workers or conf.get('webserver', 'workers') worker_timeout = (args.worker_timeout or conf.get('webserver', 'webserver_worker_timeout')) if args.debug: print( "Starting the web server on port {0} and host {1}.".format( args.port, args.hostname)) app.run(debug=True, port=args.port, host=args.hostname) else: pid, stdout, stderr, log_file = setup_locations("webserver", pid=args.pid) print( 'Running the Gunicorn server with {workers} {args.workerclass}' 'workers on host {args.hostname} and port ' '{args.port} with a timeout of {worker_timeout}...'.format(**locals())) sp = subprocess.Popen([ 'gunicorn', '-w', str(args.workers), '-k', str(args.workerclass), '-t', str(args.worker_timeout), '-b', args.hostname + ':' + str(args.port), '-n', 'airflow-webserver', '--pid', pid, 'airflow.www.app:cached_app()'] ) if args.foreground: sp.wait()
def webserver(args): print(settings.HEADER) log_to_stdout() from airflow.www.app import cached_app app = cached_app(configuration) threads = args.threads or configuration.get("webserver", "threads") if args.debug: print("Starting the web server on port {0} and host {1}.".format(args.port, args.hostname)) app.run(debug=True, port=args.port, host=args.hostname) else: print( "Running the Gunicorn server with {threads}" "on host {args.hostname} and port " "{args.port}...".format(**locals()) ) sp = subprocess.Popen( [ "gunicorn", "-w", str(args.threads), "-t", "120", "-b", args.hostname + ":" + str(args.port), "airflow.www.app:cached_app()", ] ) sp.wait()
def users_create(args): """Creates new user in the DB""" appbuilder = cached_app().appbuilder role = appbuilder.sm.find_role(args.role) if not role: valid_roles = appbuilder.sm.get_all_roles() raise SystemExit( f'{args.role} is not a valid role. Valid roles are: {valid_roles}') if args.use_random_password: password = ''.join(random.choice(string.printable) for _ in range(16)) elif args.password: password = args.password else: password = getpass.getpass('Password:'******'Repeat for confirmation:') if password != password_confirmation: raise SystemExit('Passwords did not match') if appbuilder.sm.find_user(args.username): print(f'{args.username} already exist in the db') return user = appbuilder.sm.add_user(args.username, args.firstname, args.lastname, args.email, role, password) if user: print(f'User "{args.username}" created with role "{args.role}"') else: raise SystemExit('Failed to create user')
def users_manage_role(args, remove=False): """Deletes or appends user roles""" user = _find_user(args) appbuilder = cached_app().appbuilder role = appbuilder.sm.find_role(args.role) if not role: valid_roles = appbuilder.sm.get_all_roles() raise SystemExit( f'"{args.role}" is not a valid role. Valid roles are: {valid_roles}' ) if remove: if role not in user.roles: raise SystemExit( f'User "{user.username}" is not a member of role "{args.role}"' ) user.roles = [r for r in user.roles if r != role] appbuilder.sm.update_user(user) print(f'User "{user.username}" removed from role "{args.role}"') else: if role in user.roles: raise SystemExit( f'User "{user.username}" is already a member of role "{args.role}"' ) user.roles.append(role) appbuilder.sm.update_user(user) print(f'User "{user.username}" added to role "{args.role}"')
def roles_list(args): """Lists all existing roles""" appbuilder = cached_app().appbuilder # pylint: disable=no-member roles = appbuilder.sm.get_all_roles() AirflowConsole().print_as(data=sorted([r.name for r in roles]), output=args.output, mapper=lambda x: {"name": x})
def test_should_respect_proxy_fix(self): app = application.cached_app(testing=True) app.url_map.add(Rule("/debug", endpoint="debug")) def debug_view(): from flask import request # Should respect HTTP_X_FORWARDED_FOR self.assertEqual(request.remote_addr, '192.168.0.1') # Should respect HTTP_X_FORWARDED_PROTO, HTTP_X_FORWARDED_HOST, HTTP_X_FORWARDED_PORT, # HTTP_X_FORWARDED_PREFIX self.assertEqual(request.url, 'https://valid:445/proxy-prefix/debug') return Response("success") app.view_functions['debug'] = debug_view new_environ = { "PATH_INFO": "/debug", "REMOTE_ADDR": "192.168.0.2", "HTTP_HOST": "invalid:9000", "HTTP_X_FORWARDED_FOR": "192.168.0.1", "HTTP_X_FORWARDED_PROTO": "https", "HTTP_X_FORWARDED_HOST": "valid", "HTTP_X_FORWARDED_PORT": "445", "HTTP_X_FORWARDED_PREFIX": "/proxy-prefix", } environ = create_environ(environ_overrides=new_environ) response = Response.from_app(app, environ) self.assertEqual(b"success", response.get_data()) self.assertEqual(response.status_code, 200)
def test_admin_views(self): app = cached_app() [admin] = app.extensions['admin'] category = admin._menu_categories['Test Plugin'] [admin_view ] = [v for v in category.get_children() if isinstance(v, MenuView)] self.assertEqual('Test View', admin_view.name)
def test_admin_views(self): app = cached_app() [admin] = app.extensions['admin'] category = admin._menu_categories['Test Plugin'] [admin_view] = [v for v in category.get_children() if isinstance(v, MenuView)] self.assertEqual('Test View', admin_view.name)
def convert_permissions(permissions, view_menus, convert_action, convert_dag_id): """Creates new empty role in DB""" appbuilder = cached_app().appbuilder # pylint: disable=no-member roles = appbuilder.sm.get_all_roles() views_to_remove = set() for permission_name in permissions: # pylint: disable=too-many-nested-blocks for view_menu in view_menus: view_name = view_menu.name old_pvm = appbuilder.sm.find_permission_view_menu(permission_name, view_name) if not old_pvm: continue views_to_remove.add(view_name) new_permission_name = convert_action(permission_name) new_pvm = appbuilder.sm.add_permission_view_menu(new_permission_name, convert_dag_id(view_name)) for role in roles: if appbuilder.sm.exist_permission_on_roles(view_name, permission_name, [role.id]): appbuilder.sm.add_permission_role(role, new_pvm) appbuilder.sm.del_permission_role(role, old_pvm) print(f"DELETING: {role.name} ----> {view_name}.{permission_name}") appbuilder.sm.del_permission_view_menu(permission_name, view_name) print(f"DELETING: perm_view ----> {view_name}.{permission_name}") for view_name in views_to_remove: if appbuilder.sm.find_view_menu(view_name): appbuilder.sm.del_view_menu(view_name) print(f"DELETING: view_menu ----> {view_name}") if 'can_dag_read' in permissions: for permission_name in permissions: if appbuilder.sm.find_permission(permission_name): appbuilder.sm.del_permission(permission_name) print(f"DELETING: permission ----> {permission_name}")
def test_menu_links(self): app = cached_app() [admin] = app.extensions['admin'] category = admin._menu_categories['Test Plugin'] [menu_link] = [ml for ml in category.get_children() if isinstance(ml, MenuLink)] self.assertEqual('Test Menu Link', menu_link.name)
def test_dag_link(self): from airflow.www.app import cached_app with cached_app(testing=True).test_request_context(): html = str(utils.dag_link({'dag_id': '<a&1>', 'execution_date': datetime.now()})) self.assertIn('%3Ca%261%3E', html) self.assertNotIn('<a&1>', html)
def roles_list(args): """Lists all existing roles""" appbuilder = cached_app().appbuilder # pylint: disable=no-member roles = appbuilder.sm.get_all_roles() print("Existing roles:\n") role_names = sorted([[r.name] for r in roles]) msg = tabulate(role_names, headers=['Role'], tablefmt=args.output) print(msg)
def setUp(self): super(TestMountPoint, self).setUp() configuration.load_test_config() configuration.conf.set("webserver", "base_url", "http://localhost:8080/test") config = dict() config['WTF_CSRF_METHODS'] = [] app = application.cached_app(config=config, testing=True) self.client = Client(app)
def downgrade(): # noqa: D103 permissions = ['can_read', 'can_edit'] vms = cached_app().appbuilder.sm.get_all_view_menu() view_menus = [ vm for vm in vms if (vm.name == permissions.RESOURCE_DAG or vm.name.startswith('DAG:')) ] convert_permissions(permissions, view_menus, downgrade_action, downgrade_dag_id)
def test_should_set_sqlalchemy_engine_options(self): app = application.cached_app(testing=True) engine_params = { 'pool_size': 3, 'pool_recycle': 120, 'pool_pre_ping': True, 'max_overflow': 5 } assert app.config['SQLALCHEMY_ENGINE_OPTIONS'] == engine_params
def setUpClass(cls): configuration.load_test_config() configuration.conf.set("webserver", "base_url", "http://localhost:8080/test") # Clear cached app to remount base_url forcefully application.app = None app = application.cached_app(config={'WTF_CSRF_ENABLED': False}, testing=True) cls.client = Client(app, BaseResponse)
def sync_perm(args): """Updates permissions for existing roles and DAGs""" appbuilder = cached_app().appbuilder # pylint: disable=no-member print('Updating permission, view-menu for all existing roles') appbuilder.sm.sync_roles() print('Updating permission on all DAG views') dags = DagBag().dags.values() for dag in dags: appbuilder.sm.sync_perm_for_dag(dag.dag_id, dag.access_control)
def test_dag_link_when_dag_is_none(self): """Test that when there is no dag_id, dag_link does not contain hyperlink""" from airflow.www.app import cached_app with cached_app(testing=True).test_request_context(): html = str(utils.dag_link({})) assert 'None' in html assert '<a href=' not in html
def setUpClass(cls): # Clear cached app to remount base_url forcefully application.app = None with conf_vars({ ("webserver", "base_url"): "http://localhost:8080/test" }): app = application.cached_app(config={'WTF_CSRF_ENABLED': False}, testing=True) cls.client = Client(app, BaseResponse)
def _import_users(users_list: List[Dict[str, Any]]): appbuilder = cached_app().appbuilder users_created = [] users_updated = [] try: UserSchema(many=True).load(users_list) except ValidationError as e: msg = [] for row_num, failure in e.normalized_messages().items(): msg.append(f'[Item {row_num}]') for key, value in failure.items(): msg.append(f'\t{key}: {value}') raise SystemExit( "Error: Input file didn't pass validation. See below:\n{}".format( '\n'.join(msg))) for user in users_list: roles = [] for rolename in user['roles']: role = appbuilder.sm.find_role(rolename) if not role: valid_roles = appbuilder.sm.get_all_roles() raise SystemExit( f'Error: "{rolename}" is not a valid role. Valid roles are: {valid_roles}' ) roles.append(role) existing_user = appbuilder.sm.find_user(email=user['email']) if existing_user: print(f"Found existing user with email '{user['email']}'") if existing_user.username != user['username']: raise SystemExit( f"Error: Changing the username is not allowed - please delete and recreate the user with" f" email {user['email']!r}") existing_user.roles = roles existing_user.first_name = user['firstname'] existing_user.last_name = user['lastname'] appbuilder.sm.update_user(existing_user) users_updated.append(user['email']) else: print(f"Creating new user with email '{user['email']}'") appbuilder.sm.add_user( username=user['username'], first_name=user['firstname'], last_name=user['lastname'], email=user['email'], role=roles, ) users_created.append(user['email']) return users_created, users_updated
def users_delete(args): """Deletes user from DB""" user = _find_user(args) appbuilder = cached_app().appbuilder # pylint: disable=no-member if appbuilder.sm.del_register_user(user): print(f'User {args.username} deleted') else: raise SystemExit('Failed to delete user')
def sync_perm(args): """Updates permissions for existing roles and DAGs""" appbuilder = cached_app().appbuilder # pylint: disable=no-member print('Updating actions and resources for all existing roles') # Add missing permissions for all the Base Views _before_ syncing/creating roles appbuilder.add_permissions(update_perms=True) appbuilder.sm.sync_roles() if args.include_dags: print('Updating permission on all DAG views') appbuilder.sm.create_dag_specific_permissions()
def users_delete(args): """Deletes user from DB""" user = _find_user(args) appbuilder = cached_app().appbuilder if appbuilder.sm.del_register_user(user): print(f'User "{user.username}" deleted') else: raise SystemExit('Failed to delete user')
def users_list(args): """Lists users at the command line""" appbuilder = cached_app().appbuilder users = appbuilder.sm.get_all_users() fields = ['id', 'username', 'email', 'first_name', 'last_name', 'roles'] AirflowConsole().print_as( data=users, output=args.output, mapper=lambda x: {f: x.__getattribute__(f) for f in fields})
def users_list(args): """Lists users at the command line""" appbuilder = cached_app().appbuilder # pylint: disable=no-member users = appbuilder.sm.get_all_users() fields = ['id', 'username', 'email', 'first_name', 'last_name', 'roles'] users = [[user.__getattribute__(field) for field in fields] for user in users] msg = tabulate(users, [field.capitalize().replace('_', ' ') for field in fields], tablefmt=args.output) print(msg)
def sync_perm(args): """Updates permissions for existing roles and DAGs""" appbuilder = cached_app().appbuilder # pylint: disable=no-member print('Updating permission, view-menu for all existing roles') appbuilder.sm.sync_roles() # Add missing permissions for all the Base Views appbuilder.add_permissions(update_perms=True) print('Updating permission on all DAG views') dags = DagBag(read_dags_from_db=True).dags.values() for dag in dags: appbuilder.sm.sync_perm_for_dag(dag.dag_id, dag.access_control)
def test_should_set_sqlalchemy_engine_options(self): app = application.cached_app(testing=True) engine_params = { 'pool_size': 3, 'pool_recycle': 120, 'pool_pre_ping': True, 'max_overflow': 5 } if app.config['SQLALCHEMY_DATABASE_URI'].startswith('mysql'): engine_params['isolation_level'] = 'READ COMMITTED' assert app.config['SQLALCHEMY_ENGINE_OPTIONS'] == engine_params
def webserver(args): print(settings.HEADER) from airflow.www.app import cached_app app = cached_app(conf) access_logfile = args.access_logfile or conf.get("webserver", "access_logfile") error_logfile = args.error_logfile or conf.get("webserver", "error_logfile") workers = args.workers or conf.get("webserver", "workers") worker_timeout = args.worker_timeout or conf.get("webserver", "webserver_worker_timeout") if args.debug: print("Starting the web server on port {0} and host {1}.".format(args.port, args.hostname)) app.run(debug=True, port=args.port, host=args.hostname) else: pid, stdout, stderr, log_file = setup_locations("webserver", pid=args.pid) print( textwrap.dedent( """\ Running the Gunicorn Server with: Workers: {workers} {args.workerclass} Host: {args.hostname}:{args.port} Timeout: {worker_timeout} Logfiles: {access_logfile} {error_logfile} =================================================================\ """.format( **locals() ) ) ) run_args = [ "gunicorn", "-w " + str(args.workers), "-k " + str(args.workerclass), "-t " + str(args.worker_timeout), "-b " + args.hostname + ":" + str(args.port), "-n " + "airflow-webserver", "-p " + str(pid), ] if args.access_logfile: run_args += ["--access-logfile", str(args.access_logfile)] if args.error_logfile: run_args += ["--error-logfile", str(args.error_logfile)] if args.daemon: run_args += ["-D"] module = "airflow.www.app:cached_app()".encode() run_args += [module] os.execvp("gunicorn", run_args)
def webserver(args): print(settings.HEADER) from airflow.www.app import cached_app app = cached_app(conf) access_logfile = args.access_logfile or conf.get('webserver', 'access_logfile') error_logfile = args.error_logfile or conf.get('webserver', 'error_logfile') workers = args.workers or conf.get('webserver', 'workers') worker_timeout = (args.worker_timeout or conf.get('webserver', 'webserver_worker_timeout')) if args.debug: print( "Starting the web server on port {0} and host {1}.".format( args.port, args.hostname)) app.run(debug=True, port=args.port, host=args.hostname) else: pid, stdout, stderr, log_file = setup_locations("webserver", pid=args.pid) print( textwrap.dedent('''\ Running the Gunicorn Server with: Workers: {workers} {args.workerclass} Host: {args.hostname}:{args.port} Timeout: {worker_timeout} Logfiles: {access_logfile} {error_logfile} =================================================================\ '''.format(**locals()))) run_args = [ 'gunicorn', '-w ' + str(args.workers), '-k ' + str(args.workerclass), '-t ' + str(args.worker_timeout), '-b ' + args.hostname + ':' + str(args.port), '-n ' + 'airflow-webserver', '-p ' + str(pid), ] if args.access_logfile: run_args += ['--access-logfile', str(args.access_logfile)] if args.error_logfile: run_args += ['--error-logfile', str(args.error_logfile)] if args.daemon: run_args += ["-D"] module = "airflow.www.app:cached_app()".encode() run_args += [module] os.execvp( 'gunicorn', run_args )
def webserver(args): print(settings.HEADER) log_to_stdout() from airflow.www.app import cached_app app = cached_app(conf) threads = args.threads or conf.get('webserver', 'threads') if args.debug: print( "Starting the web server on port {0} and host {1}.".format( args.port, args.hostname)) app.run(debug=True, port=args.port, host=args.hostname) else: print( 'Running the Gunicorn server with {threads}' 'on host {args.hostname} and port ' '{args.port}...'.format(**locals())) sp = subprocess.Popen([ 'gunicorn', '-w', str(args.threads), '-t', '120', '-b', args.hostname + ':' + str(args.port), 'airflow.www.app:cached_app()']) sp.wait()
def webserver(args): print(settings.HEADER) from airflow.www.app import cached_app app = cached_app(configuration) workers = args.workers or configuration.get('webserver', 'workers') if args.debug: print( "Starting the web server on port {0} and host {1}.".format( args.port, args.hostname)) app.run(debug=True, port=args.port, host=args.hostname) else: print( 'Running the Gunicorn server with {workers} {args.workerclass}' 'workers on host {args.hostname} and port ' '{args.port}...'.format(**locals())) sp = subprocess.Popen([ 'gunicorn', '-w', str(args.workers), '-k', str(args.workerclass), '-t', '120', '-b', args.hostname + ':' + str(args.port), 'airflow.www.app:cached_app()']) sp.wait()
def webserver(args): print(settings.HEADER) log_to_stdout() from airflow.www.app import cached_app app = cached_app(configuration) workers = args.workers or configuration.get('webserver', 'workers') if args.debug: print( "Starting the web server on port {0} and host {1}.".format( args.port, args.hostname)) app.run(debug=True, port=args.port, host=args.hostname) else: print( 'Running the Gunicorn server with {workers} {args.workerclass}' 'workers on host {args.hostname} and port ' '{args.port}...'.format(**locals())) gunicorn = distutils.spawn.find_executable('gunicorn') os.execv(gunicorn, [ gunicorn, 'airflow.www.app:cached_app()', '-b', '%s:%d' % (args.hostname, args.port), '-w', str(args.workers), '-k', str(args.workerclass) ])
def webserver(args): print(settings.HEADER) from airflow.www.app import cached_app app = cached_app(conf) workers = args.workers or conf.get('webserver', 'workers') worker_timeout = (args.worker_timeout or conf.get('webserver', 'webserver_worker_timeout')) if args.debug: print( "Starting the web server on port {0} and host {1}.".format( args.port, args.hostname)) app.run(debug=True, port=args.port, host=args.hostname) else: pid, stdout, stderr, log_file = setup_locations("webserver", pid=args.pid) print( 'Running the Gunicorn server with {workers} {args.workerclass}' 'workers on host {args.hostname} and port ' '{args.port} with a timeout of {worker_timeout}...'.format(**locals())) run_args = ['gunicorn', '-w ' + str(args.workers), '-k ' + str(args.workerclass), '-t ' + str(args.worker_timeout), '-b ' + args.hostname + ':' + str(args.port), '-n ' + 'airflow-webserver', '-p ' + str(pid)] if args.daemon: run_args.append("-D") module = "airflow.www.app:cached_app()".encode() run_args.append(module) os.execvp( 'gunicorn', run_args )
def webserver(args): print(settings.HEADER) app = cached_app(conf) access_logfile = args.access_logfile or conf.get('webserver', 'access_logfile') error_logfile = args.error_logfile or conf.get('webserver', 'error_logfile') num_workers = args.workers or conf.get('webserver', 'workers') worker_timeout = (args.worker_timeout or conf.get('webserver', 'webserver_worker_timeout')) if args.debug: print( "Starting the web server on port {0} and host {1}.".format( args.port, args.hostname)) app.run(debug=True, port=args.port, host=args.hostname) else: pid, stdout, stderr, log_file = setup_locations("webserver", pid=args.pid) print( textwrap.dedent('''\ Running the Gunicorn Server with: Workers: {num_workers} {args.workerclass} Host: {args.hostname}:{args.port} Timeout: {worker_timeout} Logfiles: {access_logfile} {error_logfile} =================================================================\ '''.format(**locals()))) run_args = [ 'gunicorn', '-w', str(num_workers), '-k', str(args.workerclass), '-t', str(worker_timeout), '-b', args.hostname + ':' + str(args.port), '-n', 'airflow-webserver', '-p', str(pid), '-c', 'airflow.www.gunicorn_config' ] if args.access_logfile: run_args += ['--access-logfile', str(args.access_logfile)] if args.error_logfile: run_args += ['--error-logfile', str(args.error_logfile)] if args.daemon: run_args += ["-D"] run_args += ["airflow.www.app:cached_app()"] gunicorn_master_proc = subprocess.Popen(run_args) def kill_proc(dummy_signum, dummy_frame): gunicorn_master_proc.terminate() gunicorn_master_proc.wait() sys.exit(0) signal.signal(signal.SIGINT, kill_proc) signal.signal(signal.SIGTERM, kill_proc) # These run forever until SIG{INT, TERM, KILL, ...} signal is sent if conf.getint('webserver', 'worker_refresh_interval') > 0: restart_workers(gunicorn_master_proc, num_workers) else: while True: time.sleep(1)
def test_flask_blueprints(self): app = cached_app() self.assertIsInstance(app.blueprints['test_plugin'], Blueprint)