def app():
config = dict(
TESTING=True,
SCRAPYD_SERVERS=[custom_settings['_SCRAPYD_SERVER'], 'not-exist:6801'],
SCRAPYD_SERVERS_AUTHS=[
custom_settings['_SCRAPYD_SERVER_AUTH'], ('username', 'password')
],
SCRAPYD_SERVERS_GROUPS=['', 'Scrapyd-group'],
SCRAPY_PROJECTS_DIR=os.path.join(CWD, 'data'),
SCRAPYD_LOGS_DIR='',
EMAIL_WORKING_DAYS=list(range(1, 8)),
EMAIL_WORKING_HOURS=list(range(24)),
ON_JOB_FINISHED=True,
LOG_CRITICAL_THRESHOLD=1,
LOG_CRITICAL_TRIGGER_FORCESTOP=True)
config.update(custom_settings)
app = create_app(config)
@app.context_processor
def inject_variable():
return dict(CHECK_LATEST_VERSION_FREQ=100)
yield app
def app():
config = dict(TESTING=True,
SCRAPYD_SERVERS=['127.0.0.1:6800'],
SCRAPYD_SERVERS_AUTHS=[None],
SCRAPY_PROJECTS_DIR=os.path.join(CWD, 'data'),
SCRAPYD_LOGS_DIR='')
# For test_a_factory.py and test_email() in test_log.py
config.update(email_settings)
app = create_app(config)
@app.context_processor
def inject_variable():
return dict(
SCRAPYD_SERVERS=app.config['SCRAPYD_SERVERS'],
SCRAPYD_SERVERS_GROUPS=['Scrapyd-group'] *
len(app.config['SCRAPYD_SERVERS']),
SCRAPYD_SERVERS_AUTHS=[('Scrapyd-username', 'Scrapyd-password')] *
len(app.config['SCRAPYD_SERVERS']),
DEFAULT_LATEST_VERSION=DEFAULT_LATEST_VERSION,
DAEMONSTATUS_REFRESH_INTERVAL=0,
SCRAPYD_SERVERS_AMOUNT=len(app.config['SCRAPYD_SERVERS']))
yield app
def app():
fake_server = 'scrapydweb-fake-domain.com:443'
SCRAPYD_SERVERS = [custom_settings['_SCRAPYD_SERVER'], fake_server]
if custom_settings['_SCRAPYD_SERVER_AUTH']:
username, password = custom_settings['_SCRAPYD_SERVER_AUTH']
authed_server = '%s:%s@%s' % (username, password,
custom_settings['_SCRAPYD_SERVER'])
_SCRAPYD_SERVERS = [authed_server, fake_server]
else:
_SCRAPYD_SERVERS = SCRAPYD_SERVERS
config = dict(
TESTING=True,
# SERVER_NAME='127.0.0.1:5000', # http://flask.pocoo.org/docs/0.12/config/#builtin-configuration-values
DEFAULT_SETTINGS_PY_PATH='',
SCRAPYDWEB_SETTINGS_PY_PATH='',
MAIN_PID=os.getpid(),
LOGPARSER_PID=0,
POLL_PID=0,
SCRAPYD_SERVERS=SCRAPYD_SERVERS,
_SCRAPYD_SERVERS=_SCRAPYD_SERVERS,
LOCAL_SCRAPYD_SERVER=custom_settings['_SCRAPYD_SERVER'],
SCRAPYD_SERVERS_AUTHS=[
custom_settings['_SCRAPYD_SERVER_AUTH'],
('username', '123456abcdef')
],
SCRAPYD_SERVERS_GROUPS=['', 'Scrapyd-group'],
SCRAPY_PROJECTS_DIR=os.path.join(cst.ROOT_DIR, 'data'),
ENABLE_LOGPARSER=False,
ALERT_WORKING_DAYS=list(range(1, 8)),
ALERT_WORKING_HOURS=list(range(24)),
VERBOSE=True,
)
config.update(custom_settings)
app = create_app(config)
@app.context_processor
def inject_variable():
SCRAPYD_SERVERS = app.config.get('SCRAPYD_SERVERS',
[]) or ['127.0.0.1:6800']
return dict(
SCRAPYD_SERVERS=SCRAPYD_SERVERS,
SCRAPYD_SERVERS_AMOUNT=len(SCRAPYD_SERVERS),
SCRAPYD_SERVERS_GROUPS=app.config.get('SCRAPYD_SERVERS_GROUPS', [])
or [''],
SCRAPYD_SERVERS_AUTHS=app.config.get('SCRAPYD_SERVERS_AUTHS', [])
or [None],
SCRAPYD_SERVERS_PUBLIC_URLS=[''] * len(SCRAPYD_SERVERS),
DAEMONSTATUS_REFRESH_INTERVAL=app.config.get(
'DAEMONSTATUS_REFRESH_INTERVAL', 10),
ENABLE_AUTH=app.config.get('ENABLE_AUTH', False),
SHOW_SCRAPYD_ITEMS=app.config.get('SHOW_SCRAPYD_ITEMS', True),
)
yield app
def app():
app = create_app({
'TESTING': True,
'SCRAPYD_SERVERS': ['127.0.0.1:6800']
})
@app.context_processor
def inject_variable():
return {
'SCRAPYD_SERVERS': app.config['SCRAPYD_SERVERS'],
'SCRAPYD_SERVERS_GROUP': ["" for s in app.config['SCRAPYD_SERVERS']],
'DEFAULT_LATEST_VERSION': DEFAULT_LATEST_VERSION,
}
yield app
def main():
print(">>> scrapydweb version: %s" % __version__)
print(">>> Run 'scrapydweb -h' to get help")
print(">>> Run ScrapydWeb with argument '-ss 127.0.0.1 -ss 192.168.0.101:12345@group1' "
"to set any number of Scrapyd servers to control.")
print(">>> Run ScrapydWeb with argument '--scrapyd_logs_dir SCRAPYD_LOGS_DIR' to speed up loading utf8 and stats html")
print(">>> Run ScrapydWeb with argument '--disable_cache' to disable caching utf8 and stats files in the background periodically")
print(">>> Using default settings from %s" % os.path.join(CWD, 'default_settings.py'))
app = create_app()
scrapydweb_settings_py = find_scrapydweb_settings_py()
if scrapydweb_settings_py:
print(">>> Overriding custom settings from %s" % scrapydweb_settings_py)
app.config.from_pyfile(scrapydweb_settings_py)
else:
try:
copyfile(os.path.join(CWD, 'default_settings.py'), os.path.join('.', 'scrapydweb_settings.py'))
print(">>> The config file 'scrapydweb_settings.py' is copied to your working directory, "
"and you may custom settings with it")
except:
print(">>> You may copy the file 'default_settings.py' from above path to your working directory, "
"and rename it as 'scrapydweb_settings.py' to custom settings")
args = parse_args(app.config)
check_args(args)
update_app_config(app.config, args)
# print(app.config)
if not app.config['DISABLE_CACHE']:
start_caching(app.config)
print('>>> Visit ScrapydWeb at http://{host}:{port} or http://127.0.0.1:{port}'.format(
host="IP-OF-THE-HOST-WHERE-SCRAPYDWEB-RUNS-ON", port=app.config['SCRAPYDWEB_PORT']))
@app.context_processor
def inject_variable():
return {
'SCRAPYD_SERVERS': app.config['SCRAPYD_SERVERS'],
'SCRAPYD_SERVERS_GROUP': app.config['SCRAPYD_SERVERS_GROUP'],
'DEFAULT_LATEST_VERSION': DEFAULT_LATEST_VERSION,
}
# /site-packages/flask/app.py
# run(host=None, port=None, debug=None, load_dotenv=True, **options)
# Threaded mode is enabled by default.
app.run(host=app.config['SCRAPYDWEB_HOST'], port=app.config['SCRAPYDWEB_PORT']) # , debug=True)
def app():
app = create_app({'TESTING': True, 'SCRAPYD_SERVERS': ['127.0.0.1:6800']})
@app.context_processor
def inject_variable():
return {
'SCRAPYD_SERVERS':
app.config['SCRAPYD_SERVERS'],
'SCRAPYD_SERVERS_GROUPS':
['fakegroup' for s in app.config['SCRAPYD_SERVERS']],
'SCRAPYD_SERVERS_AUTHS': [('fakeusername', 'fakepassword')
for s in app.config['SCRAPYD_SERVERS']],
'DEFAULT_LATEST_VERSION':
DEFAULT_LATEST_VERSION,
}
yield app
def app():
app = create_app({
'TESTING': True,
'SCRAPYD_SERVERS': ['127.0.0.1:6800'],
'SCRAPY_PROJECTS_DIR': os.path.join(CWD, 'data'),
})
@app.context_processor
def inject_variable():
return {
'SCRAPYD_SERVERS': app.config['SCRAPYD_SERVERS'],
'SCRAPYD_SERVERS_GROUPS': ['fakegroup' for s in app.config['SCRAPYD_SERVERS']],
'SCRAPYD_SERVERS_AUTHS': [('fakeusername', 'fakepassword') for s in app.config['SCRAPYD_SERVERS']],
'DEFAULT_LATEST_VERSION': DEFAULT_LATEST_VERSION,
'DAEMONSTATUS_REFRESH_INTERVAL': 0,
}
yield app
def app():
config = dict(
TESTING=True,
# SERVER_NAME='127.0.0.1:5000', # http://flask.pocoo.org/docs/0.12/config/#builtin-configuration-values
DEFAULT_SETTINGS_PY_PATH='',
SCRAPYDWEB_SETTINGS_PY_PATH='',
MAIN_PID=os.getpid(),
LOGPARSER_PID=0,
POLL_PID=0,
SCRAPYD_SERVERS=[custom_settings['_SCRAPYD_SERVER'], 'not-exist:6801'],
LOCAL_SCRAPYD_SERVER=custom_settings['_SCRAPYD_SERVER'],
SCRAPYD_SERVERS_AUTHS=[
custom_settings['_SCRAPYD_SERVER_AUTH'], ('username', 'password')
],
SCRAPYD_SERVERS_GROUPS=['', 'Scrapyd-group'],
SCRAPY_PROJECTS_DIR=os.path.join(cst.CWD, 'data'),
ENABLE_LOGPARSER=False,
EMAIL_WORKING_DAYS=list(range(1, 8)),
EMAIL_WORKING_HOURS=list(range(24)),
VERBOSE=True)
config.update(custom_settings)
app = create_app(config)
@app.context_processor
def inject_variable():
return dict(
SCRAPYD_SERVERS=app.config.get('SCRAPYD_SERVERS', [])
or ['127.0.0.1:6800'],
SCRAPYD_SERVERS_AMOUNT=len(
app.config.get('SCRAPYD_SERVERS', []) or ['127.0.0.1:6800']),
SCRAPYD_SERVERS_GROUPS=app.config.get('SCRAPYD_SERVERS_GROUPS', [])
or [''],
SCRAPYD_SERVERS_AUTHS=app.config.get('SCRAPYD_SERVERS_AUTHS', [])
or [None],
DAEMONSTATUS_REFRESH_INTERVAL=app.config.get(
'DAEMONSTATUS_REFRESH_INTERVAL', 10),
ENABLE_AUTH=app.config.get('ENABLE_AUTH', False),
SHOW_SCRAPYD_ITEMS=app.config.get('SHOW_SCRAPYD_ITEMS', True),
)
yield app
def app():
config = dict(
TESTING=True,
SCRAPYD_SERVERS=[custom_settings['_SCRAPYD_SERVER'], 'not-exist:6801'],
SCRAPYD_SERVERS_AUTHS=[
custom_settings['_SCRAPYD_SERVER_AUTH'], ('username', 'password')
],
SCRAPYD_SERVERS_GROUPS=['', 'Scrapyd-group'],
SCRAPY_PROJECTS_DIR=os.path.join(CWD, 'data'),
SCRAPYD_LOGS_DIR='',
VERBOSE=True,
EMAIL_WORKING_DAYS=list(range(1, 8)),
EMAIL_WORKING_HOURS=list(range(24)),
# ON_JOB_FINISHED=True,
# LOG_CRITICAL_THRESHOLD=1,
# LOG_CRITICAL_TRIGGER_FORCESTOP=True
)
config.update(custom_settings)
app = create_app(config)
@app.context_processor
def inject_variable():
return dict(
SCRAPYD_SERVERS=app.config.get('SCRAPYD_SERVERS', [])
or ['127.0.0.1:6800'],
SCRAPYD_SERVERS_AMOUNT=len(
app.config.get('SCRAPYD_SERVERS', []) or ['127.0.0.1:6800']),
SCRAPYD_SERVERS_GROUPS=app.config.get('SCRAPYD_SERVERS_GROUPS', [])
or [''],
SCRAPYD_SERVERS_AUTHS=app.config.get('SCRAPYD_SERVERS_AUTHS', [])
or [None],
DAEMONSTATUS_REFRESH_INTERVAL=app.config.get(
'DAEMONSTATUS_REFRESH_INTERVAL', 10),
ENABLE_AUTH=app.config.get('ENABLE_AUTH', False),
SHOW_SCRAPYD_ITEMS=app.config.get('SHOW_SCRAPYD_ITEMS', True),
)
yield app
def test_config():
assert not create_app().testing
assert create_app({'TESTING': True}).testing
def main():
print(">>> scrapydweb version: %s" % __version__)
print(">>> Run 'scrapydweb -h' to get help")
print(">>> Loading default settings from %s" %
os.path.join(CWD, 'default_settings.py'))
app = create_app()
scrapydweb_settings_py = find_scrapydweb_settings_py()
if scrapydweb_settings_py:
print(">>> Overriding custom settings from %s" %
scrapydweb_settings_py)
app.config.from_pyfile(scrapydweb_settings_py)
else:
try:
copyfile(os.path.join(CWD, 'default_settings.py'),
os.path.join('.', SCRAPYDWEB_SETTINGS_PY))
print(
">>> The config file '%s' is copied to your working directory, "
"and you may custom settings with it" % SCRAPYDWEB_SETTINGS_PY)
except:
print(
">>> You may copy the file 'default_settings.py' from above path to your working directory, "
"and rename it to '%s' to custom settings" %
SCRAPYDWEB_SETTINGS_PY)
args = parse_args(app.config)
check_args(args)
update_app_config(app.config, args)
# print(app.config)
if not app.config['DISABLE_CACHE']:
import atexit
proc = start_caching(app.config)
print(
">>> Caching utf8 and stats files in the background with pid: %s" %
proc.pid)
atexit.register(kill_child, proc)
print(
">>> Visit ScrapydWeb at http://{host}:{port} or http://127.0.0.1:{port}"
.format(host='IP-OF-THE-HOST-WHERE-SCRAPYDWEB-RUNS-ON',
port=app.config['SCRAPYDWEB_PORT']))
username = app.config.get('USERNAME', '')
password = app.config.get('PASSWORD', '')
@app.context_processor
def inject_variable():
return {
'SCRAPYD_SERVERS': app.config['SCRAPYD_SERVERS'],
'SCRAPYD_SERVERS_GROUPS': app.config['SCRAPYD_SERVERS_GROUPS'],
'SCRAPYD_SERVERS_AUTHS': app.config['SCRAPYD_SERVERS_AUTHS'],
'DEFAULT_LATEST_VERSION': DEFAULT_LATEST_VERSION,
'GITHUB_URL': __url__,
'REQUIRE_LOGIN': True if username and password else False,
'HIDE_SCRAPYD_ITEMS': app.config.get('HIDE_SCRAPYD_ITEMS', False),
}
# https://stackoverflow.com/questions/34164464/flask-decorate-every-route-at-once
@app.before_request
def require_login():
if request.form:
app.logger.debug(json_dumps(request.form))
if username and password:
auth = request.authorization
if not auth or not (auth.username == username
and auth.password == password):
return authenticate()
# /site-packages/flask/app.py
# run(host=None, port=None, debug=None, load_dotenv=True, **options)
# Threaded mode is enabled by default.
app.run(host=app.config['SCRAPYDWEB_HOST'],
port=app.config['SCRAPYDWEB_PORT']) # , debug=True)
def main():
apscheduler_logger.setLevel(logging.ERROR) # To hide warning logging in scheduler.py until app.run()
main_pid = os.getpid()
logger.info("ScrapydWeb version: %s", __version__)
logger.info("Use 'scrapydweb -h' to get help")
logger.info("Main pid: %s", main_pid)
logger.debug("Loading default settings from %s", handle_slash(DEFAULT_SETTINGS_PY_PATH))
app = create_app()
handle_metadata('main_pid', main_pid) # In handle_metadata(): with db.app.app_context():
app.config['MAIN_PID'] = main_pid
app.config['DEFAULT_SETTINGS_PY_PATH'] = DEFAULT_SETTINGS_PY_PATH
app.config['SCRAPYDWEB_SETTINGS_PY_PATH'] = os.path.join(os.getcwd(), SCRAPYDWEB_SETTINGS_PY)
load_custom_settings(app.config)
args = parse_args(app.config)
# "scrapydweb -h" ends up here
update_app_config(app.config, args)
try:
check_app_config(app.config)
except AssertionError as err:
logger.error("Check app config fail: ")
sys.exit(u"\n{err}\n\nCheck and update your settings in {path}\n".format(
err=err, path=handle_slash(app.config['SCRAPYDWEB_SETTINGS_PY_PATH'])))
# https://stackoverflow.com/questions/34164464/flask-decorate-every-route-at-once
@app.before_request
def require_login():
if app.config.get('ENABLE_AUTH', False):
auth = request.authorization
USERNAME = str(app.config.get('USERNAME', '')) # May be 0 from config file
PASSWORD = str(app.config.get('PASSWORD', ''))
if not auth or not (auth.username == USERNAME and auth.password == PASSWORD):
return authenticate()
# MUST be commented out for released version
# https://stackoverflow.com/questions/34066804/disabling-caching-in-flask
# @app.after_request
# def add_header(r):
# r.headers['Pragma'] = 'no-cache'
# r.headers['Expires'] = '0'
# r.headers['Cache-Control'] = 'public, max-age=0'
# return r
@app.context_processor
def inject_variable():
SCRAPYD_SERVERS = app.config.get('SCRAPYD_SERVERS', []) or ['127.0.0.1:6800']
SCRAPYD_SERVERS_PUBLIC_URLS = app.config.get('SCRAPYD_SERVERS_PUBLIC_URLS', None)
return dict(
SCRAPYD_SERVERS=SCRAPYD_SERVERS,
SCRAPYD_SERVERS_AMOUNT=len(SCRAPYD_SERVERS),
SCRAPYD_SERVERS_GROUPS=app.config.get('SCRAPYD_SERVERS_GROUPS', []) or [''],
SCRAPYD_SERVERS_AUTHS=app.config.get('SCRAPYD_SERVERS_AUTHS', []) or [None],
SCRAPYD_SERVERS_PUBLIC_URLS=SCRAPYD_SERVERS_PUBLIC_URLS or [None] * len(SCRAPYD_SERVERS),
DAEMONSTATUS_REFRESH_INTERVAL=app.config.get('DAEMONSTATUS_REFRESH_INTERVAL', 10),
ENABLE_AUTH=app.config.get('ENABLE_AUTH', False),
SHOW_SCRAPYD_ITEMS=app.config.get('SHOW_SCRAPYD_ITEMS', True),
)
# To solve https://github.com/my8100/scrapydweb/issues/17
# http://flask.pocoo.org/docs/1.0/cli/?highlight=flask_debug#environments
# flask/helpers.py: get_env() The default is 'production'
# On Windows, get/set/delete: set FLASK_ENV, set FLASK_ENV=production, set set FLASK_ENV=
# if not os.environ.get('FLASK_ENV'):
# os.environ['FLASK_ENV'] = 'development'
# print("The environment variable 'FLASK_ENV' has been set to 'development'")
# print("WARNING: Do not use the development server in a production. "
# "Check out http://flask.pocoo.org/docs/1.0/deploying/")
# http://flask.pocoo.org/docs/1.0/config/?highlight=flask_debug#environment-and-debug-features
if app.config.get('DEBUG', False):
os.environ['FLASK_DEBUG'] = '1'
logger.info("Note that use_reloader is set to False in run.py")
else:
os.environ['FLASK_DEBUG'] = '0'
# site-packages/flask/app.py
# Threaded mode is enabled by default.
# https://stackoverflow.com/a/28590266/10517783 to run in HTTP or HTTPS mode
# site-packages/werkzeug/serving.py
# https://stackoverflow.com/questions/13895176/sqlalchemy-and-sqlite-database-is-locked
if app.config.get('ENABLE_HTTPS', False):
protocol = 'https'
context = (app.config['CERTIFICATE_FILEPATH'], app.config['PRIVATEKEY_FILEPATH'])
else:
protocol = 'http'
context = None
print("{star}Visit ScrapydWeb at {protocol}://127.0.0.1:{port} "
"or {protocol}://IP-OF-THE-CURRENT-HOST:{port}{star}\n".format(
star=STAR, protocol=protocol, port=app.config['SCRAPYDWEB_PORT']))
logger.info("For running Flask in production, check out http://flask.pocoo.org/docs/1.0/deploying/")
apscheduler_logger.setLevel(logging.DEBUG)
app.run(host=app.config['SCRAPYDWEB_BIND'], port=app.config['SCRAPYDWEB_PORT'],
ssl_context=context, use_reloader=False)
def main():
main_pid = os.getpid()
printf("Main pid: %s" % main_pid)
printf("ScrapydWeb version: %s" % __version__)
printf("Use the 'scrapydweb -h' command to get help")
printf("Loading default settings from %s" %
os.path.join(CWD, 'default_settings.py'))
app = create_app()
load_custom_config(app.config)
args = parse_args(app.config)
# "scrapydweb -h" ends up here
update_app_config(app.config, args)
# from pprint import pprint
# pprint(app.config)
try:
check_app_config(app.config)
except AssertionError as err:
sys.exit("\n!!! %s\nCheck and update your settings in: %s" %
(err, scrapydweb_settings_py_path))
if app.config.get('ENABLE_CACHE', True):
caching_pid = init_caching(app.config, main_pid)
else:
caching_pid = None
# https://stackoverflow.com/questions/34164464/flask-decorate-every-route-at-once
@app.before_request
def require_login():
if app.config.get('ENABLE_AUTH', False):
auth = request.authorization
USERNAME = str(app.config.get('USERNAME',
'')) # May be 0 from config file
PASSWORD = str(app.config.get('PASSWORD', ''))
if not auth or not (auth.username == USERNAME
and auth.password == PASSWORD):
return authenticate()
# Should be commented out for released version
# https://stackoverflow.com/questions/34066804/disabling-caching-in-flask
# @app.after_request
# def add_header(r):
# r.headers['Pragma'] = 'no-cache'
# r.headers['Expires'] = '0'
# r.headers['Cache-Control'] = 'public, max-age=0'
# return r
@app.context_processor
def inject_variable():
return dict(
main_pid=main_pid,
caching_pid=caching_pid,
CHECK_LATEST_VERSION_FREQ=100,
scrapydweb_settings_py_path=scrapydweb_settings_py_path,
)
printf(
"Visit ScrapydWeb at http://127.0.0.1:{port} or http://{bind}:{port}".
format(bind='IP-OF-CURRENT-HOST', port=app.config['SCRAPYDWEB_PORT']))
# site-packages/flask/app.py
# def run(self, host=None, port=None, debug=None, load_dotenv=True, **options):
# Threaded mode is enabled by default.
app.run(host=app.config['SCRAPYDWEB_BIND'],
port=app.config['SCRAPYDWEB_PORT']) # , debug=True)
def main():
main_pid = os.getpid()
printf("ScrapydWeb version: %s" % __version__)
printf("Use 'scrapydweb -h' to get help")
printf("Main pid: %s" % main_pid)
printf("Loading default settings from %s" % DEFAULT_SETTINGS_PY_PATH)
app = create_app()
app.config['MAIN_PID'] = main_pid
app.config['DEFAULT_SETTINGS_PY_PATH'] = DEFAULT_SETTINGS_PY_PATH
app.config['SCRAPYDWEB_SETTINGS_PY_PATH'] = os.path.join(
os.getcwd(), SCRAPYDWEB_SETTINGS_PY)
load_custom_settings(app.config)
args = parse_args(app.config)
# "scrapydweb -h" ends up here
update_app_config(app.config, args)
try:
check_app_config(app.config)
except AssertionError as err:
sys.exit(
"\n{alert}\n{err}\nCheck and update your settings in {path}\n{alert}"
.format(alert=ALERT,
err=err,
path=app.config['SCRAPYDWEB_SETTINGS_PY_PATH']))
# https://stackoverflow.com/questions/34164464/flask-decorate-every-route-at-once
@app.before_request
def require_login():
if app.config.get('ENABLE_AUTH', False):
auth = request.authorization
USERNAME = str(app.config.get('USERNAME',
'')) # May be 0 from config file
PASSWORD = str(app.config.get('PASSWORD', ''))
if not auth or not (auth.username == USERNAME
and auth.password == PASSWORD):
return authenticate()
# MUST be commented out for released version
# https://stackoverflow.com/questions/34066804/disabling-caching-in-flask
# @app.after_request
# def add_header(r):
# r.headers['Pragma'] = 'no-cache'
# r.headers['Expires'] = '0'
# r.headers['Cache-Control'] = 'public, max-age=0'
# return r
@app.context_processor
def inject_variable():
return dict(
SCRAPYD_SERVERS=app.config.get('SCRAPYD_SERVERS', [])
or ['127.0.0.1:6800'],
SCRAPYD_SERVERS_AMOUNT=len(
app.config.get('SCRAPYD_SERVERS', []) or ['127.0.0.1:6800']),
SCRAPYD_SERVERS_GROUPS=app.config.get('SCRAPYD_SERVERS_GROUPS', [])
or [''],
SCRAPYD_SERVERS_AUTHS=app.config.get('SCRAPYD_SERVERS_AUTHS', [])
or [None],
DAEMONSTATUS_REFRESH_INTERVAL=app.config.get(
'DAEMONSTATUS_REFRESH_INTERVAL', 10),
ENABLE_AUTH=app.config.get('ENABLE_AUTH', False),
SHOW_SCRAPYD_ITEMS=app.config.get('SHOW_SCRAPYD_ITEMS', True),
)
# To solve https://github.com/my8100/scrapydweb/issues/17
# http://flask.pocoo.org/docs/1.0/cli/?highlight=flask_debug#environments
# flask/helpers.py: get_env() The default is 'production'
# On Windows, get/set/delete: set FLASK_ENV, set FLASK_ENV=production, set set FLASK_ENV=
# if not os.environ.get('FLASK_ENV'):
# os.environ['FLASK_ENV'] = 'development'
# printf("The environment variable 'FLASK_ENV' has been set to 'development'", warn=True)
# printf("WARNING: Do not use the development server in a production. "
# "Check out http://flask.pocoo.org/docs/1.0/deploying/", warn=True)
# http://flask.pocoo.org/docs/1.0/config/?highlight=flask_debug#environment-and-debug-features
if app.config.get('DEBUG', False):
os.environ['FLASK_DEBUG'] = '1'
printf(
"It's not recommended to run ScrapydWeb in debug mode, set 'DEBUG = False' instead.",
warn=True)
else:
os.environ['FLASK_DEBUG'] = '0'
# site-packages/flask/app.py
# Threaded mode is enabled by default.
# https://stackoverflow.com/a/28590266/10517783 to run in HTTP or HTTPS mode
# site-packages/werkzeug/serving.py
if app.config.get('ENABLE_HTTPS', False):
protocol = 'https'
context = (app.config['CERTIFICATE_FILEPATH'],
app.config['PRIVATEKEY_FILEPATH'])
else:
protocol = 'http'
context = None
print(STAR)
printf(
"Visit ScrapydWeb at {protocol}://127.0.0.1:{port} or {protocol}://IP-OF-THE-CURRENT-HOST:{port}"
.format(protocol=protocol, port=app.config['SCRAPYDWEB_PORT']))
printf(
"For running Flask in production, check out http://flask.pocoo.org/docs/1.0/deploying/",
warn=True)
print(STAR)
app.run(host=app.config['SCRAPYDWEB_BIND'],
port=app.config['SCRAPYDWEB_PORT'],
ssl_context=context)
def main():
main_pid = os.getpid()
printf("Main pid: %s" % main_pid)
printf("scrapydweb version: %s" % __version__)
printf("Run 'scrapydweb -h' to get help")
printf("Loading default settings from %s" % os.path.join(CWD, 'default_settings.py'))
app = create_app()
load_custom_config(app.config)
args = parse_args(app.config)
# "scrapydweb -h" would end up here
update_app_config(app.config, args)
# from pprint import pprint
# pprint(app.config)
try:
check_app_config(app.config)
except AssertionError as err:
sys.exit("\n!!! %s\nCheck out your settings in: %s" % (err, scrapydweb_settings_py_path))
if not app.config.get('DISABLE_CACHE', False):
caching_pid = init_caching(app.config, main_pid)
else:
caching_pid = None
REQUIRE_LOGIN = False if app.config.get('DISABLE_AUTH', True) else True
USERNAME = str(app.config.get('USERNAME', '')) # May be 0 from config file
PASSWORD = str(app.config.get('PASSWORD', ''))
# https://stackoverflow.com/questions/34164464/flask-decorate-every-route-at-once
@app.before_request
def require_login():
if REQUIRE_LOGIN:
auth = request.authorization
if not auth or not (auth.username == USERNAME and auth.password == PASSWORD):
return authenticate()
@app.context_processor
def inject_variable():
return dict(
SCRAPYD_SERVERS=app.config['SCRAPYD_SERVERS'],
SCRAPYD_SERVERS_AMOUNT=len(app.config['SCRAPYD_SERVERS']),
SCRAPYD_SERVERS_GROUPS=app.config['SCRAPYD_SERVERS_GROUPS'],
SCRAPYD_SERVERS_AUTHS=app.config['SCRAPYD_SERVERS_AUTHS'],
PYTHON_VERSION='.'.join([str(n) for n in sys.version_info[:3]]),
SCRAPYDWEB_VERSION=__version__,
CHECK_LATEST_VERSION_FREQ=30,
DEFAULT_LATEST_VERSION=DEFAULT_LATEST_VERSION,
GITHUB_URL=__url__,
SHOW_SCRAPYD_ITEMS=app.config.get('SHOW_SCRAPYD_ITEMS', True),
DAEMONSTATUS_REFRESH_INTERVAL=int(app.config.get('DAEMONSTATUS_REFRESH_INTERVAL', 10)),
REQUIRE_LOGIN=REQUIRE_LOGIN,
scrapydweb_settings_py_path=scrapydweb_settings_py_path,
main_pid=main_pid,
caching_pid=caching_pid,
)
printf("Visit ScrapydWeb at http://{bind}:{port} or http://127.0.0.1:{port}".format(
bind='IP-OF-CURRENT-HOST', port=app.config['SCRAPYDWEB_PORT']))
# /site-packages/flask/app.py
# def run(self, host=None, port=None, debug=None, load_dotenv=True, **options):
# Threaded mode is enabled by default.
app.run(host=app.config['SCRAPYDWEB_BIND'], port=app.config['SCRAPYDWEB_PORT']) # , debug=True)
import sys
import os
import logging
sys.path.insert(0, '/var/www/html/scrapydweb')
sys.path.append('/var/www/html/scrapydweb/my_env/lib/python3.8/site-packages')
sys.path.append('/var/www/html')
from scrapydweb import create_app
from scrapydweb.common import handle_metadata, authenticate
from flask import request
application = create_app()
application.config[
'DEFAULT_SETTINGS_PY_PATH'] = '/var/www/html/scrapydweb/default_settings.py'
application.config[
'SCRAPYDWEB_SETTINGS_PY_PATH'] = '/var/www/html/scrapydweb/default_settings.py'
main_pid = os.getpid()
application.config['MAIN_PID'] = main_pid
application.config['LOGPARSER_PID'] = main_pid
application.config['POLL_PID'] = main_pid
logger = logging.getLogger(__name__)
apscheduler_logger = logging.getLogger('apscheduler')
handle_metadata('main_pid', main_pid)
@application.context_processor
def inject_variable():
