def test_enable_logparser(app, client):
def json_loads_from_file(path):
with io.open(path, 'r', encoding='utf-8') as f:
return json.loads(f.read())
# In conftest.py: ENABLE_LOGPARSER=False
assert not os.path.exists(app.config['STATS_JSON_PATH'])
assert not os.path.exists(app.config['DEMO_JSON_PATH'])
app.config['ENABLE_LOGPARSER'] = True
app.config['ENABLE_EMAIL'] = False
# ['username:[email protected]:6800', ]
app.config['SCRAPYD_SERVERS'] = app.config['_SCRAPYD_SERVERS']
check_app_config(app.config)
logparser_pid = app.config['LOGPARSER_PID']
assert isinstance(logparser_pid, int) and logparser_pid > 0
assert app.config['POLL_PID'] is None
req(app, client, view='settings', kws=dict(node=1), ins='logparser_pid: %s' % logparser_pid)
sleep()
stats_json = json_loads_from_file(app.config['STATS_JSON_PATH'])
assert stats_json['logparser_version'] == cst.LOGPARSER_VERSION
assert cst.DEMO_JOBID in stats_json['datas'][cst.PROJECT][cst.SPIDER]
demo_json = json_loads_from_file(app.config['DEMO_JSON_PATH'])
assert demo_json['runtime'] == '0:01:08'
assert demo_json['finish_reason'] == 'finished'
assert demo_json['logparser_version'] == cst.LOGPARSER_VERSION
def test_check_app_config(app, client):
cleantest(app, client)
# In conftest.py: ENABLE_LOGPARSER=False
assert not os.path.exists(app.config['STATS_JSON_PATH'])
check_app_config(app.config)
strings = []
assert app.config['LOGPARSER_PID'] is None
strings.append('logparser_pid: None')
poll_pid = app.config['POLL_PID']
if app.config.get('ENABLE_EMAIL', False):
assert isinstance(poll_pid, int) and poll_pid > 0
else:
assert poll_pid is None
strings.append('poll_pid: %s' % poll_pid)
req(app, client, view='settings', kws=dict(node=1), ins=strings)
assert not os.path.exists(app.config['STATS_JSON_PATH'])
# Test ENABLE_EMAIL = False
if app.config.get('ENABLE_EMAIL', False):
app.config['ENABLE_EMAIL'] = False
check_app_config(app.config)
assert app.config['LOGPARSER_PID'] is None
assert app.config['POLL_PID'] is None
req(app,
client,
view='settings',
kws=dict(node=1),
ins='poll_pid: None')
def test_check_app_config(app, client):
cleantest(app, client)
# In conftest.py: ENABLE_LOGPARSER=False
assert not os.path.exists(app.config['STATS_JSON_PATH'])
# ['username:[email protected]:6800', ]
app.config['SCRAPYD_SERVERS'] = app.config['_SCRAPYD_SERVERS']
check_app_config(app.config)
strings = []
assert app.config['LOGPARSER_PID'] is None
strings.append('logparser_pid: None')
poll_pid = app.config['POLL_PID']
if app.config.get('ENABLE_MONITOR', False):
assert isinstance(poll_pid, int) and poll_pid > 0
else:
assert poll_pid is None
strings.append('poll_pid: %s' % poll_pid)
req(app, client, view='settings', kws=dict(node=1), ins=strings)
assert not os.path.exists(app.config['STATS_JSON_PATH'])
# Test ENABLE_MONITOR = False
if app.config.get('ENABLE_MONITOR', False):
app.config['ENABLE_MONITOR'] = False
# ['username:[email protected]:6800', ]
app.config['SCRAPYD_SERVERS'] = app.config['_SCRAPYD_SERVERS']
check_app_config(app.config)
assert app.config['LOGPARSER_PID'] is None
assert app.config['POLL_PID'] is None
req(app,
client,
view='settings',
kws=dict(node=1),
ins='poll_pid: None')
def test_ap_config(app):
check_app_config(app.config)
def main():
apscheduler_logger.setLevel(logging.ERROR) # To hide warning logging in scheduler.py until app.run()
main_pid = os.getpid()
logger.info("ScrapydWeb version: %s", __version__)
logger.info("Use 'scrapydweb -h' to get help")
logger.info("Main pid: %s", main_pid)
logger.debug("Loading default settings from %s", handle_slash(DEFAULT_SETTINGS_PY_PATH))
app = create_app()
handle_metadata('main_pid', main_pid) # In handle_metadata(): with db.app.app_context():
app.config['MAIN_PID'] = main_pid
app.config['DEFAULT_SETTINGS_PY_PATH'] = DEFAULT_SETTINGS_PY_PATH
app.config['SCRAPYDWEB_SETTINGS_PY_PATH'] = os.path.join(os.getcwd(), SCRAPYDWEB_SETTINGS_PY)
load_custom_settings(app.config)
args = parse_args(app.config)
# "scrapydweb -h" ends up here
update_app_config(app.config, args)
try:
check_app_config(app.config)
except AssertionError as err:
logger.error("Check app config fail: ")
sys.exit(u"\n{err}\n\nCheck and update your settings in {path}\n".format(
err=err, path=handle_slash(app.config['SCRAPYDWEB_SETTINGS_PY_PATH'])))
# https://stackoverflow.com/questions/34164464/flask-decorate-every-route-at-once
@app.before_request
def require_login():
if app.config.get('ENABLE_AUTH', False):
auth = request.authorization
USERNAME = str(app.config.get('USERNAME', '')) # May be 0 from config file
PASSWORD = str(app.config.get('PASSWORD', ''))
if not auth or not (auth.username == USERNAME and auth.password == PASSWORD):
return authenticate()
# MUST be commented out for released version
# https://stackoverflow.com/questions/34066804/disabling-caching-in-flask
# @app.after_request
# def add_header(r):
# r.headers['Pragma'] = 'no-cache'
# r.headers['Expires'] = '0'
# r.headers['Cache-Control'] = 'public, max-age=0'
# return r
@app.context_processor
def inject_variable():
SCRAPYD_SERVERS = app.config.get('SCRAPYD_SERVERS', []) or ['127.0.0.1:6800']
SCRAPYD_SERVERS_PUBLIC_URLS = app.config.get('SCRAPYD_SERVERS_PUBLIC_URLS', None)
return dict(
SCRAPYD_SERVERS=SCRAPYD_SERVERS,
SCRAPYD_SERVERS_AMOUNT=len(SCRAPYD_SERVERS),
SCRAPYD_SERVERS_GROUPS=app.config.get('SCRAPYD_SERVERS_GROUPS', []) or [''],
SCRAPYD_SERVERS_AUTHS=app.config.get('SCRAPYD_SERVERS_AUTHS', []) or [None],
SCRAPYD_SERVERS_PUBLIC_URLS=SCRAPYD_SERVERS_PUBLIC_URLS or [None] * len(SCRAPYD_SERVERS),
DAEMONSTATUS_REFRESH_INTERVAL=app.config.get('DAEMONSTATUS_REFRESH_INTERVAL', 10),
ENABLE_AUTH=app.config.get('ENABLE_AUTH', False),
SHOW_SCRAPYD_ITEMS=app.config.get('SHOW_SCRAPYD_ITEMS', True),
)
# To solve https://github.com/my8100/scrapydweb/issues/17
# http://flask.pocoo.org/docs/1.0/cli/?highlight=flask_debug#environments
# flask/helpers.py: get_env() The default is 'production'
# On Windows, get/set/delete: set FLASK_ENV, set FLASK_ENV=production, set set FLASK_ENV=
# if not os.environ.get('FLASK_ENV'):
# os.environ['FLASK_ENV'] = 'development'
# print("The environment variable 'FLASK_ENV' has been set to 'development'")
# print("WARNING: Do not use the development server in a production. "
# "Check out http://flask.pocoo.org/docs/1.0/deploying/")
# http://flask.pocoo.org/docs/1.0/config/?highlight=flask_debug#environment-and-debug-features
if app.config.get('DEBUG', False):
os.environ['FLASK_DEBUG'] = '1'
logger.info("Note that use_reloader is set to False in run.py")
else:
os.environ['FLASK_DEBUG'] = '0'
# site-packages/flask/app.py
# Threaded mode is enabled by default.
# https://stackoverflow.com/a/28590266/10517783 to run in HTTP or HTTPS mode
# site-packages/werkzeug/serving.py
# https://stackoverflow.com/questions/13895176/sqlalchemy-and-sqlite-database-is-locked
if app.config.get('ENABLE_HTTPS', False):
protocol = 'https'
context = (app.config['CERTIFICATE_FILEPATH'], app.config['PRIVATEKEY_FILEPATH'])
else:
protocol = 'http'
context = None
print("{star}Visit ScrapydWeb at {protocol}://127.0.0.1:{port} "
"or {protocol}://IP-OF-THE-CURRENT-HOST:{port}{star}\n".format(
star=STAR, protocol=protocol, port=app.config['SCRAPYDWEB_PORT']))
logger.info("For running Flask in production, check out http://flask.pocoo.org/docs/1.0/deploying/")
apscheduler_logger.setLevel(logging.DEBUG)
app.run(host=app.config['SCRAPYDWEB_BIND'], port=app.config['SCRAPYDWEB_PORT'],
ssl_context=context, use_reloader=False)
def main():
main_pid = os.getpid()
printf("Main pid: %s" % main_pid)
printf("ScrapydWeb version: %s" % __version__)
printf("Use the 'scrapydweb -h' command to get help")
printf("Loading default settings from %s" %
os.path.join(CWD, 'default_settings.py'))
app = create_app()
load_custom_config(app.config)
args = parse_args(app.config)
# "scrapydweb -h" ends up here
update_app_config(app.config, args)
# from pprint import pprint
# pprint(app.config)
try:
check_app_config(app.config)
except AssertionError as err:
sys.exit("\n!!! %s\nCheck and update your settings in: %s" %
(err, scrapydweb_settings_py_path))
if app.config.get('ENABLE_CACHE', True):
caching_pid = init_caching(app.config, main_pid)
else:
caching_pid = None
# https://stackoverflow.com/questions/34164464/flask-decorate-every-route-at-once
@app.before_request
def require_login():
if app.config.get('ENABLE_AUTH', False):
auth = request.authorization
USERNAME = str(app.config.get('USERNAME',
'')) # May be 0 from config file
PASSWORD = str(app.config.get('PASSWORD', ''))
if not auth or not (auth.username == USERNAME
and auth.password == PASSWORD):
return authenticate()
# Should be commented out for released version
# https://stackoverflow.com/questions/34066804/disabling-caching-in-flask
# @app.after_request
# def add_header(r):
# r.headers['Pragma'] = 'no-cache'
# r.headers['Expires'] = '0'
# r.headers['Cache-Control'] = 'public, max-age=0'
# return r
@app.context_processor
def inject_variable():
return dict(
main_pid=main_pid,
caching_pid=caching_pid,
CHECK_LATEST_VERSION_FREQ=100,
scrapydweb_settings_py_path=scrapydweb_settings_py_path,
)
printf(
"Visit ScrapydWeb at http://127.0.0.1:{port} or http://{bind}:{port}".
format(bind='IP-OF-CURRENT-HOST', port=app.config['SCRAPYDWEB_PORT']))
# site-packages/flask/app.py
# def run(self, host=None, port=None, debug=None, load_dotenv=True, **options):
# Threaded mode is enabled by default.
app.run(host=app.config['SCRAPYDWEB_BIND'],
port=app.config['SCRAPYDWEB_PORT']) # , debug=True)
def main():
main_pid = os.getpid()
printf("ScrapydWeb version: %s" % __version__)
printf("Use 'scrapydweb -h' to get help")
printf("Main pid: %s" % main_pid)
printf("Loading default settings from %s" % DEFAULT_SETTINGS_PY_PATH)
app = create_app()
app.config['MAIN_PID'] = main_pid
app.config['DEFAULT_SETTINGS_PY_PATH'] = DEFAULT_SETTINGS_PY_PATH
app.config['SCRAPYDWEB_SETTINGS_PY_PATH'] = os.path.join(
os.getcwd(), SCRAPYDWEB_SETTINGS_PY)
load_custom_settings(app.config)
args = parse_args(app.config)
# "scrapydweb -h" ends up here
update_app_config(app.config, args)
try:
check_app_config(app.config)
except AssertionError as err:
sys.exit(
"\n{alert}\n{err}\nCheck and update your settings in {path}\n{alert}"
.format(alert=ALERT,
err=err,
path=app.config['SCRAPYDWEB_SETTINGS_PY_PATH']))
# https://stackoverflow.com/questions/34164464/flask-decorate-every-route-at-once
@app.before_request
def require_login():
if app.config.get('ENABLE_AUTH', False):
auth = request.authorization
USERNAME = str(app.config.get('USERNAME',
'')) # May be 0 from config file
PASSWORD = str(app.config.get('PASSWORD', ''))
if not auth or not (auth.username == USERNAME
and auth.password == PASSWORD):
return authenticate()
# MUST be commented out for released version
# https://stackoverflow.com/questions/34066804/disabling-caching-in-flask
# @app.after_request
# def add_header(r):
# r.headers['Pragma'] = 'no-cache'
# r.headers['Expires'] = '0'
# r.headers['Cache-Control'] = 'public, max-age=0'
# return r
@app.context_processor
def inject_variable():
return dict(
SCRAPYD_SERVERS=app.config.get('SCRAPYD_SERVERS', [])
or ['127.0.0.1:6800'],
SCRAPYD_SERVERS_AMOUNT=len(
app.config.get('SCRAPYD_SERVERS', []) or ['127.0.0.1:6800']),
SCRAPYD_SERVERS_GROUPS=app.config.get('SCRAPYD_SERVERS_GROUPS', [])
or [''],
SCRAPYD_SERVERS_AUTHS=app.config.get('SCRAPYD_SERVERS_AUTHS', [])
or [None],
DAEMONSTATUS_REFRESH_INTERVAL=app.config.get(
'DAEMONSTATUS_REFRESH_INTERVAL', 10),
ENABLE_AUTH=app.config.get('ENABLE_AUTH', False),
SHOW_SCRAPYD_ITEMS=app.config.get('SHOW_SCRAPYD_ITEMS', True),
)
# To solve https://github.com/my8100/scrapydweb/issues/17
# http://flask.pocoo.org/docs/1.0/cli/?highlight=flask_debug#environments
# flask/helpers.py: get_env() The default is 'production'
# On Windows, get/set/delete: set FLASK_ENV, set FLASK_ENV=production, set set FLASK_ENV=
# if not os.environ.get('FLASK_ENV'):
# os.environ['FLASK_ENV'] = 'development'
# printf("The environment variable 'FLASK_ENV' has been set to 'development'", warn=True)
# printf("WARNING: Do not use the development server in a production. "
# "Check out http://flask.pocoo.org/docs/1.0/deploying/", warn=True)
# http://flask.pocoo.org/docs/1.0/config/?highlight=flask_debug#environment-and-debug-features
if app.config.get('DEBUG', False):
os.environ['FLASK_DEBUG'] = '1'
printf(
"It's not recommended to run ScrapydWeb in debug mode, set 'DEBUG = False' instead.",
warn=True)
else:
os.environ['FLASK_DEBUG'] = '0'
# site-packages/flask/app.py
# Threaded mode is enabled by default.
# https://stackoverflow.com/a/28590266/10517783 to run in HTTP or HTTPS mode
# site-packages/werkzeug/serving.py
if app.config.get('ENABLE_HTTPS', False):
protocol = 'https'
context = (app.config['CERTIFICATE_FILEPATH'],
app.config['PRIVATEKEY_FILEPATH'])
else:
protocol = 'http'
context = None
print(STAR)
printf(
"Visit ScrapydWeb at {protocol}://127.0.0.1:{port} or {protocol}://IP-OF-THE-CURRENT-HOST:{port}"
.format(protocol=protocol, port=app.config['SCRAPYDWEB_PORT']))
printf(
"For running Flask in production, check out http://flask.pocoo.org/docs/1.0/deploying/",
warn=True)
print(STAR)
app.run(host=app.config['SCRAPYDWEB_BIND'],
port=app.config['SCRAPYDWEB_PORT'],
ssl_context=context)
def main():
main_pid = os.getpid()
printf("Main pid: %s" % main_pid)
printf("scrapydweb version: %s" % __version__)
printf("Run 'scrapydweb -h' to get help")
printf("Loading default settings from %s" % os.path.join(CWD, 'default_settings.py'))
app = create_app()
load_custom_config(app.config)
args = parse_args(app.config)
# "scrapydweb -h" would end up here
update_app_config(app.config, args)
# from pprint import pprint
# pprint(app.config)
try:
check_app_config(app.config)
except AssertionError as err:
sys.exit("\n!!! %s\nCheck out your settings in: %s" % (err, scrapydweb_settings_py_path))
if not app.config.get('DISABLE_CACHE', False):
caching_pid = init_caching(app.config, main_pid)
else:
caching_pid = None
REQUIRE_LOGIN = False if app.config.get('DISABLE_AUTH', True) else True
USERNAME = str(app.config.get('USERNAME', '')) # May be 0 from config file
PASSWORD = str(app.config.get('PASSWORD', ''))
# https://stackoverflow.com/questions/34164464/flask-decorate-every-route-at-once
@app.before_request
def require_login():
if REQUIRE_LOGIN:
auth = request.authorization
if not auth or not (auth.username == USERNAME and auth.password == PASSWORD):
return authenticate()
@app.context_processor
def inject_variable():
return dict(
SCRAPYD_SERVERS=app.config['SCRAPYD_SERVERS'],
SCRAPYD_SERVERS_AMOUNT=len(app.config['SCRAPYD_SERVERS']),
SCRAPYD_SERVERS_GROUPS=app.config['SCRAPYD_SERVERS_GROUPS'],
SCRAPYD_SERVERS_AUTHS=app.config['SCRAPYD_SERVERS_AUTHS'],
PYTHON_VERSION='.'.join([str(n) for n in sys.version_info[:3]]),
SCRAPYDWEB_VERSION=__version__,
CHECK_LATEST_VERSION_FREQ=30,
DEFAULT_LATEST_VERSION=DEFAULT_LATEST_VERSION,
GITHUB_URL=__url__,
SHOW_SCRAPYD_ITEMS=app.config.get('SHOW_SCRAPYD_ITEMS', True),
DAEMONSTATUS_REFRESH_INTERVAL=int(app.config.get('DAEMONSTATUS_REFRESH_INTERVAL', 10)),
REQUIRE_LOGIN=REQUIRE_LOGIN,
scrapydweb_settings_py_path=scrapydweb_settings_py_path,
main_pid=main_pid,
caching_pid=caching_pid,
)
printf("Visit ScrapydWeb at http://{bind}:{port} or http://127.0.0.1:{port}".format(
bind='IP-OF-CURRENT-HOST', port=app.config['SCRAPYDWEB_PORT']))
# /site-packages/flask/app.py
# def run(self, host=None, port=None, debug=None, load_dotenv=True, **options):
# Threaded mode is enabled by default.
app.run(host=app.config['SCRAPYDWEB_BIND'], port=app.config['SCRAPYDWEB_PORT']) # , debug=True)
