def test_enable_logparser(app, client): def json_loads_from_file(path): with io.open(path, 'r', encoding='utf-8') as f: return json.loads(f.read()) # In conftest.py: ENABLE_LOGPARSER=False assert not os.path.exists(app.config['STATS_JSON_PATH']) assert not os.path.exists(app.config['DEMO_JSON_PATH']) app.config['ENABLE_LOGPARSER'] = True app.config['ENABLE_EMAIL'] = False # ['username:[email protected]:6800', ] app.config['SCRAPYD_SERVERS'] = app.config['_SCRAPYD_SERVERS'] check_app_config(app.config) logparser_pid = app.config['LOGPARSER_PID'] assert isinstance(logparser_pid, int) and logparser_pid > 0 assert app.config['POLL_PID'] is None req(app, client, view='settings', kws=dict(node=1), ins='logparser_pid: %s' % logparser_pid) sleep() stats_json = json_loads_from_file(app.config['STATS_JSON_PATH']) assert stats_json['logparser_version'] == cst.LOGPARSER_VERSION assert cst.DEMO_JOBID in stats_json['datas'][cst.PROJECT][cst.SPIDER] demo_json = json_loads_from_file(app.config['DEMO_JSON_PATH']) assert demo_json['runtime'] == '0:01:08' assert demo_json['finish_reason'] == 'finished' assert demo_json['logparser_version'] == cst.LOGPARSER_VERSION
def test_check_app_config(app, client): cleantest(app, client) # In conftest.py: ENABLE_LOGPARSER=False assert not os.path.exists(app.config['STATS_JSON_PATH']) check_app_config(app.config) strings = [] assert app.config['LOGPARSER_PID'] is None strings.append('logparser_pid: None') poll_pid = app.config['POLL_PID'] if app.config.get('ENABLE_EMAIL', False): assert isinstance(poll_pid, int) and poll_pid > 0 else: assert poll_pid is None strings.append('poll_pid: %s' % poll_pid) req(app, client, view='settings', kws=dict(node=1), ins=strings) assert not os.path.exists(app.config['STATS_JSON_PATH']) # Test ENABLE_EMAIL = False if app.config.get('ENABLE_EMAIL', False): app.config['ENABLE_EMAIL'] = False check_app_config(app.config) assert app.config['LOGPARSER_PID'] is None assert app.config['POLL_PID'] is None req(app, client, view='settings', kws=dict(node=1), ins='poll_pid: None')
def test_check_app_config(app, client): cleantest(app, client) # In conftest.py: ENABLE_LOGPARSER=False assert not os.path.exists(app.config['STATS_JSON_PATH']) # ['username:[email protected]:6800', ] app.config['SCRAPYD_SERVERS'] = app.config['_SCRAPYD_SERVERS'] check_app_config(app.config) strings = [] assert app.config['LOGPARSER_PID'] is None strings.append('logparser_pid: None') poll_pid = app.config['POLL_PID'] if app.config.get('ENABLE_MONITOR', False): assert isinstance(poll_pid, int) and poll_pid > 0 else: assert poll_pid is None strings.append('poll_pid: %s' % poll_pid) req(app, client, view='settings', kws=dict(node=1), ins=strings) assert not os.path.exists(app.config['STATS_JSON_PATH']) # Test ENABLE_MONITOR = False if app.config.get('ENABLE_MONITOR', False): app.config['ENABLE_MONITOR'] = False # ['username:[email protected]:6800', ] app.config['SCRAPYD_SERVERS'] = app.config['_SCRAPYD_SERVERS'] check_app_config(app.config) assert app.config['LOGPARSER_PID'] is None assert app.config['POLL_PID'] is None req(app, client, view='settings', kws=dict(node=1), ins='poll_pid: None')
def test_ap_config(app): check_app_config(app.config)
def main(): apscheduler_logger.setLevel(logging.ERROR) # To hide warning logging in scheduler.py until app.run() main_pid = os.getpid() logger.info("ScrapydWeb version: %s", __version__) logger.info("Use 'scrapydweb -h' to get help") logger.info("Main pid: %s", main_pid) logger.debug("Loading default settings from %s", handle_slash(DEFAULT_SETTINGS_PY_PATH)) app = create_app() handle_metadata('main_pid', main_pid) # In handle_metadata(): with db.app.app_context(): app.config['MAIN_PID'] = main_pid app.config['DEFAULT_SETTINGS_PY_PATH'] = DEFAULT_SETTINGS_PY_PATH app.config['SCRAPYDWEB_SETTINGS_PY_PATH'] = os.path.join(os.getcwd(), SCRAPYDWEB_SETTINGS_PY) load_custom_settings(app.config) args = parse_args(app.config) # "scrapydweb -h" ends up here update_app_config(app.config, args) try: check_app_config(app.config) except AssertionError as err: logger.error("Check app config fail: ") sys.exit(u"\n{err}\n\nCheck and update your settings in {path}\n".format( err=err, path=handle_slash(app.config['SCRAPYDWEB_SETTINGS_PY_PATH']))) # https://stackoverflow.com/questions/34164464/flask-decorate-every-route-at-once @app.before_request def require_login(): if app.config.get('ENABLE_AUTH', False): auth = request.authorization USERNAME = str(app.config.get('USERNAME', '')) # May be 0 from config file PASSWORD = str(app.config.get('PASSWORD', '')) if not auth or not (auth.username == USERNAME and auth.password == PASSWORD): return authenticate() # MUST be commented out for released version # https://stackoverflow.com/questions/34066804/disabling-caching-in-flask # @app.after_request # def add_header(r): # r.headers['Pragma'] = 'no-cache' # r.headers['Expires'] = '0' # r.headers['Cache-Control'] = 'public, max-age=0' # return r @app.context_processor def inject_variable(): SCRAPYD_SERVERS = app.config.get('SCRAPYD_SERVERS', []) or ['127.0.0.1:6800'] SCRAPYD_SERVERS_PUBLIC_URLS = app.config.get('SCRAPYD_SERVERS_PUBLIC_URLS', None) return dict( SCRAPYD_SERVERS=SCRAPYD_SERVERS, SCRAPYD_SERVERS_AMOUNT=len(SCRAPYD_SERVERS), SCRAPYD_SERVERS_GROUPS=app.config.get('SCRAPYD_SERVERS_GROUPS', []) or [''], SCRAPYD_SERVERS_AUTHS=app.config.get('SCRAPYD_SERVERS_AUTHS', []) or [None], SCRAPYD_SERVERS_PUBLIC_URLS=SCRAPYD_SERVERS_PUBLIC_URLS or [None] * len(SCRAPYD_SERVERS), DAEMONSTATUS_REFRESH_INTERVAL=app.config.get('DAEMONSTATUS_REFRESH_INTERVAL', 10), ENABLE_AUTH=app.config.get('ENABLE_AUTH', False), SHOW_SCRAPYD_ITEMS=app.config.get('SHOW_SCRAPYD_ITEMS', True), ) # To solve https://github.com/my8100/scrapydweb/issues/17 # http://flask.pocoo.org/docs/1.0/cli/?highlight=flask_debug#environments # flask/helpers.py: get_env() The default is 'production' # On Windows, get/set/delete: set FLASK_ENV, set FLASK_ENV=production, set set FLASK_ENV= # if not os.environ.get('FLASK_ENV'): # os.environ['FLASK_ENV'] = 'development' # print("The environment variable 'FLASK_ENV' has been set to 'development'") # print("WARNING: Do not use the development server in a production. " # "Check out http://flask.pocoo.org/docs/1.0/deploying/") # http://flask.pocoo.org/docs/1.0/config/?highlight=flask_debug#environment-and-debug-features if app.config.get('DEBUG', False): os.environ['FLASK_DEBUG'] = '1' logger.info("Note that use_reloader is set to False in run.py") else: os.environ['FLASK_DEBUG'] = '0' # site-packages/flask/app.py # Threaded mode is enabled by default. # https://stackoverflow.com/a/28590266/10517783 to run in HTTP or HTTPS mode # site-packages/werkzeug/serving.py # https://stackoverflow.com/questions/13895176/sqlalchemy-and-sqlite-database-is-locked if app.config.get('ENABLE_HTTPS', False): protocol = 'https' context = (app.config['CERTIFICATE_FILEPATH'], app.config['PRIVATEKEY_FILEPATH']) else: protocol = 'http' context = None print("{star}Visit ScrapydWeb at {protocol}://127.0.0.1:{port} " "or {protocol}://IP-OF-THE-CURRENT-HOST:{port}{star}\n".format( star=STAR, protocol=protocol, port=app.config['SCRAPYDWEB_PORT'])) logger.info("For running Flask in production, check out http://flask.pocoo.org/docs/1.0/deploying/") apscheduler_logger.setLevel(logging.DEBUG) app.run(host=app.config['SCRAPYDWEB_BIND'], port=app.config['SCRAPYDWEB_PORT'], ssl_context=context, use_reloader=False)
def main(): main_pid = os.getpid() printf("Main pid: %s" % main_pid) printf("ScrapydWeb version: %s" % __version__) printf("Use the 'scrapydweb -h' command to get help") printf("Loading default settings from %s" % os.path.join(CWD, 'default_settings.py')) app = create_app() load_custom_config(app.config) args = parse_args(app.config) # "scrapydweb -h" ends up here update_app_config(app.config, args) # from pprint import pprint # pprint(app.config) try: check_app_config(app.config) except AssertionError as err: sys.exit("\n!!! %s\nCheck and update your settings in: %s" % (err, scrapydweb_settings_py_path)) if app.config.get('ENABLE_CACHE', True): caching_pid = init_caching(app.config, main_pid) else: caching_pid = None # https://stackoverflow.com/questions/34164464/flask-decorate-every-route-at-once @app.before_request def require_login(): if app.config.get('ENABLE_AUTH', False): auth = request.authorization USERNAME = str(app.config.get('USERNAME', '')) # May be 0 from config file PASSWORD = str(app.config.get('PASSWORD', '')) if not auth or not (auth.username == USERNAME and auth.password == PASSWORD): return authenticate() # Should be commented out for released version # https://stackoverflow.com/questions/34066804/disabling-caching-in-flask # @app.after_request # def add_header(r): # r.headers['Pragma'] = 'no-cache' # r.headers['Expires'] = '0' # r.headers['Cache-Control'] = 'public, max-age=0' # return r @app.context_processor def inject_variable(): return dict( main_pid=main_pid, caching_pid=caching_pid, CHECK_LATEST_VERSION_FREQ=100, scrapydweb_settings_py_path=scrapydweb_settings_py_path, ) printf( "Visit ScrapydWeb at http://127.0.0.1:{port} or http://{bind}:{port}". format(bind='IP-OF-CURRENT-HOST', port=app.config['SCRAPYDWEB_PORT'])) # site-packages/flask/app.py # def run(self, host=None, port=None, debug=None, load_dotenv=True, **options): # Threaded mode is enabled by default. app.run(host=app.config['SCRAPYDWEB_BIND'], port=app.config['SCRAPYDWEB_PORT']) # , debug=True)
def main(): main_pid = os.getpid() printf("ScrapydWeb version: %s" % __version__) printf("Use 'scrapydweb -h' to get help") printf("Main pid: %s" % main_pid) printf("Loading default settings from %s" % DEFAULT_SETTINGS_PY_PATH) app = create_app() app.config['MAIN_PID'] = main_pid app.config['DEFAULT_SETTINGS_PY_PATH'] = DEFAULT_SETTINGS_PY_PATH app.config['SCRAPYDWEB_SETTINGS_PY_PATH'] = os.path.join( os.getcwd(), SCRAPYDWEB_SETTINGS_PY) load_custom_settings(app.config) args = parse_args(app.config) # "scrapydweb -h" ends up here update_app_config(app.config, args) try: check_app_config(app.config) except AssertionError as err: sys.exit( "\n{alert}\n{err}\nCheck and update your settings in {path}\n{alert}" .format(alert=ALERT, err=err, path=app.config['SCRAPYDWEB_SETTINGS_PY_PATH'])) # https://stackoverflow.com/questions/34164464/flask-decorate-every-route-at-once @app.before_request def require_login(): if app.config.get('ENABLE_AUTH', False): auth = request.authorization USERNAME = str(app.config.get('USERNAME', '')) # May be 0 from config file PASSWORD = str(app.config.get('PASSWORD', '')) if not auth or not (auth.username == USERNAME and auth.password == PASSWORD): return authenticate() # MUST be commented out for released version # https://stackoverflow.com/questions/34066804/disabling-caching-in-flask # @app.after_request # def add_header(r): # r.headers['Pragma'] = 'no-cache' # r.headers['Expires'] = '0' # r.headers['Cache-Control'] = 'public, max-age=0' # return r @app.context_processor def inject_variable(): return dict( SCRAPYD_SERVERS=app.config.get('SCRAPYD_SERVERS', []) or ['127.0.0.1:6800'], SCRAPYD_SERVERS_AMOUNT=len( app.config.get('SCRAPYD_SERVERS', []) or ['127.0.0.1:6800']), SCRAPYD_SERVERS_GROUPS=app.config.get('SCRAPYD_SERVERS_GROUPS', []) or [''], SCRAPYD_SERVERS_AUTHS=app.config.get('SCRAPYD_SERVERS_AUTHS', []) or [None], DAEMONSTATUS_REFRESH_INTERVAL=app.config.get( 'DAEMONSTATUS_REFRESH_INTERVAL', 10), ENABLE_AUTH=app.config.get('ENABLE_AUTH', False), SHOW_SCRAPYD_ITEMS=app.config.get('SHOW_SCRAPYD_ITEMS', True), ) # To solve https://github.com/my8100/scrapydweb/issues/17 # http://flask.pocoo.org/docs/1.0/cli/?highlight=flask_debug#environments # flask/helpers.py: get_env() The default is 'production' # On Windows, get/set/delete: set FLASK_ENV, set FLASK_ENV=production, set set FLASK_ENV= # if not os.environ.get('FLASK_ENV'): # os.environ['FLASK_ENV'] = 'development' # printf("The environment variable 'FLASK_ENV' has been set to 'development'", warn=True) # printf("WARNING: Do not use the development server in a production. " # "Check out http://flask.pocoo.org/docs/1.0/deploying/", warn=True) # http://flask.pocoo.org/docs/1.0/config/?highlight=flask_debug#environment-and-debug-features if app.config.get('DEBUG', False): os.environ['FLASK_DEBUG'] = '1' printf( "It's not recommended to run ScrapydWeb in debug mode, set 'DEBUG = False' instead.", warn=True) else: os.environ['FLASK_DEBUG'] = '0' # site-packages/flask/app.py # Threaded mode is enabled by default. # https://stackoverflow.com/a/28590266/10517783 to run in HTTP or HTTPS mode # site-packages/werkzeug/serving.py if app.config.get('ENABLE_HTTPS', False): protocol = 'https' context = (app.config['CERTIFICATE_FILEPATH'], app.config['PRIVATEKEY_FILEPATH']) else: protocol = 'http' context = None print(STAR) printf( "Visit ScrapydWeb at {protocol}://127.0.0.1:{port} or {protocol}://IP-OF-THE-CURRENT-HOST:{port}" .format(protocol=protocol, port=app.config['SCRAPYDWEB_PORT'])) printf( "For running Flask in production, check out http://flask.pocoo.org/docs/1.0/deploying/", warn=True) print(STAR) app.run(host=app.config['SCRAPYDWEB_BIND'], port=app.config['SCRAPYDWEB_PORT'], ssl_context=context)
def main(): main_pid = os.getpid() printf("Main pid: %s" % main_pid) printf("scrapydweb version: %s" % __version__) printf("Run 'scrapydweb -h' to get help") printf("Loading default settings from %s" % os.path.join(CWD, 'default_settings.py')) app = create_app() load_custom_config(app.config) args = parse_args(app.config) # "scrapydweb -h" would end up here update_app_config(app.config, args) # from pprint import pprint # pprint(app.config) try: check_app_config(app.config) except AssertionError as err: sys.exit("\n!!! %s\nCheck out your settings in: %s" % (err, scrapydweb_settings_py_path)) if not app.config.get('DISABLE_CACHE', False): caching_pid = init_caching(app.config, main_pid) else: caching_pid = None REQUIRE_LOGIN = False if app.config.get('DISABLE_AUTH', True) else True USERNAME = str(app.config.get('USERNAME', '')) # May be 0 from config file PASSWORD = str(app.config.get('PASSWORD', '')) # https://stackoverflow.com/questions/34164464/flask-decorate-every-route-at-once @app.before_request def require_login(): if REQUIRE_LOGIN: auth = request.authorization if not auth or not (auth.username == USERNAME and auth.password == PASSWORD): return authenticate() @app.context_processor def inject_variable(): return dict( SCRAPYD_SERVERS=app.config['SCRAPYD_SERVERS'], SCRAPYD_SERVERS_AMOUNT=len(app.config['SCRAPYD_SERVERS']), SCRAPYD_SERVERS_GROUPS=app.config['SCRAPYD_SERVERS_GROUPS'], SCRAPYD_SERVERS_AUTHS=app.config['SCRAPYD_SERVERS_AUTHS'], PYTHON_VERSION='.'.join([str(n) for n in sys.version_info[:3]]), SCRAPYDWEB_VERSION=__version__, CHECK_LATEST_VERSION_FREQ=30, DEFAULT_LATEST_VERSION=DEFAULT_LATEST_VERSION, GITHUB_URL=__url__, SHOW_SCRAPYD_ITEMS=app.config.get('SHOW_SCRAPYD_ITEMS', True), DAEMONSTATUS_REFRESH_INTERVAL=int(app.config.get('DAEMONSTATUS_REFRESH_INTERVAL', 10)), REQUIRE_LOGIN=REQUIRE_LOGIN, scrapydweb_settings_py_path=scrapydweb_settings_py_path, main_pid=main_pid, caching_pid=caching_pid, ) printf("Visit ScrapydWeb at http://{bind}:{port} or http://127.0.0.1:{port}".format( bind='IP-OF-CURRENT-HOST', port=app.config['SCRAPYDWEB_PORT'])) # /site-packages/flask/app.py # def run(self, host=None, port=None, debug=None, load_dotenv=True, **options): # Threaded mode is enabled by default. app.run(host=app.config['SCRAPYDWEB_BIND'], port=app.config['SCRAPYDWEB_PORT']) # , debug=True)