def _get_mapping(self, dst):
from dallinger.config import initialize_experiment_package
initialize_experiment_package(dst)
from dallinger.experiment import load
exp_class = load()
extra_files = getattr(exp_class, "extra_files", None)
if extra_files is None:
try:
from dallinger_experiment.experiment import extra_files
except ImportError:
try:
from dallinger_experiment.dallinger_experiment import extra_files
except ImportError:
pass
if extra_files is not None:
for src, filename in extra_files():
filename = filename.lstrip("/")
if os.path.isdir(src):
for dirpath, dirnames, filenames in os.walk(src,
topdown=True):
for fn in filenames:
dst_fileparts = [dst, filename] + dirnames + [fn]
dst_filepath = os.path.join(*dst_fileparts)
yield (
os.path.join(dirpath, fn),
dst_filepath,
)
else:
dst_filepath = os.path.join(dst, filename)
yield (src, dst_filepath)
def main():
import gevent.monkey
gevent.monkey.patch_all()
from gevent.queue import LifoQueue
# These imports are inside the __main__ block
# to make sure that we only import from rq_gevent_worker
# (which has the side effect of applying gevent monkey patches)
# in the worker process. This way other processes can import the
# redis connection without that side effect.
import os
from redis import BlockingConnectionPool, StrictRedis
from rq import Queue, Connection
from dallinger.heroku.rq_gevent_worker import GeventWorker as Worker
from dallinger.config import initialize_experiment_package
initialize_experiment_package(os.getcwd())
import logging
logging.basicConfig(format="%(asctime)s %(message)s", level=logging.DEBUG)
redis_url = os.getenv("REDIS_URL", "redis://localhost:6379")
# Specify queue class for improved performance with gevent.
# see http://carsonip.me/posts/10x-faster-python-gevent-redis-connection-pool/
redis_pool = BlockingConnectionPool.from_url(redis_url,
queue_class=LifoQueue)
redis_conn = StrictRedis(connection_pool=redis_pool)
with Connection(redis_conn):
worker = Worker(list(map(Queue, listen)))
worker.work()
def verify_experiment_module(verbose):
"""Perform basic sanity checks on experiment.py.
"""
ok = True
if not os.path.exists("experiment.py"):
return False
# Bootstrap a package in a temp directory and make it importable:
temp_package_name = "TEMP_VERIFICATION_PACKAGE"
tmp = tempfile.mkdtemp()
clone_dir = os.path.join(tmp, temp_package_name)
to_ignore = shutil.ignore_patterns(
os.path.join(".git", "*"), "*.db", "snapshots", "data", "server.log"
)
shutil.copytree(os.getcwd(), clone_dir, ignore=to_ignore)
initialize_experiment_package(clone_dir)
from dallinger_experiment import experiment
if clone_dir not in experiment.__file__:
raise ImportError("Checking the wrong experiment.py... aborting.")
classes = inspect.getmembers(experiment, inspect.isclass)
exps = [c for c in classes if (c[1].__bases__[0].__name__ in "Experiment")]
# Clean up:
for entry in [k for k in sys.modules if temp_package_name in k]:
del sys.modules[entry]
# Run checks:
if len(exps) == 0:
log(
"✗ experiment.py does not define an experiment class.",
delay=0,
chevrons=False,
verbose=verbose,
)
ok = False
elif len(exps) == 1:
log(
"✓ experiment.py defines 1 experiment",
delay=0,
chevrons=False,
verbose=verbose,
)
else:
log(
"✗ experiment.py defines more than one experiment class.",
delay=0,
chevrons=False,
verbose=verbose,
)
ok = False
return ok
def load():
"""Load the active experiment."""
initialize_experiment_package(os.getcwd())
try:
try:
from dallinger_experiment import experiment
except ImportError:
from dallinger_experiment import dallinger_experiment as experiment
classes = inspect.getmembers(experiment, inspect.isclass)
for name, c in classes:
if 'Experiment' in c.__bases__[0].__name__:
return c
else:
raise ImportError
except ImportError:
logger.error('Could not import experiment.')
raise
def load():
"""Load the active experiment."""
initialize_experiment_package(os.getcwd())
try:
try:
from dallinger_experiment import experiment
except ImportError:
from dallinger_experiment import dallinger_experiment as experiment
classes = inspect.getmembers(experiment, is_experiment_class)
preferred_class = os.environ.get("EXPERIMENT_CLASS_NAME", None)
if preferred_class is not None:
try:
return dict(classes)[preferred_class]
except KeyError:
raise ImportError(
"No experiment named {} was found".format(preferred_class)
)
if len(classes) > 1:
for name, c in classes:
if "Experiment" in c.__bases__[0].__name__:
warnings.warn(
UserWarning(
"More than one potential experiment class found but no EXPERIMENT_CLASS_NAME environment variable. Picking {} from {}.".format(
name, [n for (n, cls) in classes]
)
),
stacklevel=3,
)
return c
raise ImportError(
"No direct experiment subclass found in {}".format(
[n for (n, cls) in classes]
)
)
elif len(classes) == 0:
raise ImportError("No experiment classes found")
else:
return classes[0][1]
except ImportError:
logger.error("Could not import experiment.")
raise
def main():
import gevent.monkey
gevent.monkey.patch_all()
from gevent.queue import LifoQueue
# These imports are inside the __main__ block
# to make sure that we only import from rq_gevent_worker
# (which has the side effect of applying gevent monkey patches)
# in the worker process. This way other processes can import the
# redis connection without that side effect.
import logging
import os
from redis import BlockingConnectionPool, StrictRedis
from rq import Queue, Connection
from six.moves.urllib.parse import urlparse
from dallinger.heroku.rq_gevent_worker import GeventWorker as Worker
from dallinger.config import initialize_experiment_package
initialize_experiment_package(os.getcwd())
logging.basicConfig(format="%(asctime)s %(message)s", level=logging.DEBUG)
redis_url = os.getenv("REDIS_URL", "redis://localhost:6379")
# Specify queue class for improved performance with gevent.
# see http://carsonip.me/posts/10x-faster-python-gevent-redis-connection-pool/
connection_args = {
"url": redis_url,
"queue_class": LifoQueue,
}
# Since we are generally running on Heroku, and configuring SSL certificates
# is challenging, we disable cert requirements on secure connections.
if urlparse(redis_url).scheme == "rediss":
connection_args["ssl_cert_reqs"] = None
redis_pool = BlockingConnectionPool.from_url(**connection_args)
redis_conn = StrictRedis(connection_pool=redis_pool)
with Connection(redis_conn):
worker = Worker(list(map(Queue, listen)))
worker.work()
def init_db(drop_all=False, bind=engine):
"""Initialize the database, optionally dropping existing tables."""
# To create the db structure according to the experiment configuration
# we need to import the experiment code, so that sqlalchemy has a chance
# to update its metadata
initialize_experiment_package(os.getcwd())
try:
from dallinger_experiment import experiment # noqa: F401
except ImportError:
pass
try:
if drop_all:
Base.metadata.drop_all(bind=bind)
Base.metadata.create_all(bind=bind)
except OperationalError as err:
msg = 'password authentication failed for user "dallinger"'
if msg in err.message:
sys.stderr.write(db_user_warning)
raise
return session
listen = ['high', 'default', 'low']
redis_url = os.getenv('REDIS_URL', 'redis://localhost:6379')
conn = redis.from_url(redis_url)
if __name__ == '__main__': # pragma: nocover
# Make sure gevent patches are applied early.
import gevent.monkey
gevent.monkey.patch_all()
# These imports are inside the __main__ block
# to make sure that we only import from rq_gevent_worker
# (which has the side effect of applying gevent monkey patches)
# in the worker process. This way other processes can import the
# redis connection without that side effect.
from rq import (
Queue,
Connection
)
from dallinger.heroku.rq_gevent_worker import GeventWorker as Worker
from dallinger.config import initialize_experiment_package
initialize_experiment_package(os.getcwd())
import logging
logging.basicConfig(format='%(asctime)s %(message)s', level=logging.INFO)
with Connection(conn):
worker = Worker(list(map(Queue, listen)))
worker.work()
def verify_package(verbose=True):
"""Ensure the package has a config file and a valid experiment file."""
is_passing = True
# Check for existence of required files.
required_files = [
"config.txt",
"experiment.py",
]
for f in required_files:
if os.path.exists(f):
log("✓ {} is PRESENT".format(f), chevrons=False, verbose=verbose)
else:
log("✗ {} is MISSING".format(f), chevrons=False, verbose=verbose)
is_passing = False
# Check the experiment file.
if os.path.exists("experiment.py"):
# Check if the experiment file has exactly one Experiment class.
tmp = tempfile.mkdtemp()
clone_dir = os.path.join(tmp, 'temp_exp_package')
to_ignore = shutil.ignore_patterns(os.path.join(".git", "*"), "*.db",
"snapshots", "data", "server.log")
shutil.copytree(os.getcwd(), clone_dir, ignore=to_ignore)
initialize_experiment_package(clone_dir)
from dallinger_experiment import experiment
classes = inspect.getmembers(experiment, inspect.isclass)
exps = [
c for c in classes if (c[1].__bases__[0].__name__ in "Experiment")
]
if len(exps) == 0:
log("✗ experiment.py does not define an experiment class.",
delay=0,
chevrons=False,
verbose=verbose)
is_passing = False
elif len(exps) == 1:
log("✓ experiment.py defines 1 experiment",
delay=0,
chevrons=False,
verbose=verbose)
else:
log("✗ experiment.py defines more than one experiment class.",
delay=0,
chevrons=False,
verbose=verbose)
config = get_config()
if not config.ready:
config.load()
# Check base_payment is correct
base_pay = config.get('base_payment')
dollarFormat = "{:.2f}".format(base_pay)
if base_pay <= 0:
log("✗ base_payment must be positive value in config.txt.",
delay=0,
chevrons=False,
verbose=verbose)
is_passing = False
if float(dollarFormat) != float(base_pay):
log("✗ base_payment must be in [dollars].[cents] format in config.txt. Try changing "
"{0} to {1}.".format(base_pay, dollarFormat),
delay=0,
chevrons=False,
verbose=verbose)
is_passing = False
# Check front-end files do not exist
files = [
os.path.join("templates", "complete.html"),
os.path.join("templates", "error.html"),
os.path.join("templates", "error-complete.html"),
os.path.join("templates", "launch.html"),
os.path.join("templates", "thanks.html"),
os.path.join("static", "css", "dallinger.css"),
os.path.join("static", "scripts", "dallinger.js"),
os.path.join("static", "scripts", "dallinger2.js"),
os.path.join("static", "scripts", "reqwest.min.js"),
os.path.join("static", "scripts", "tracker.js"),
os.path.join("static", "robots.txt")
]
for f in files:
if os.path.exists(f):
log("✗ {} OVERWRITES shared frontend files inserted at run-time".
format(f),
delay=0,
chevrons=False,
verbose=verbose)
log("✓ no file conflicts", delay=0, chevrons=False, verbose=verbose)
return is_passing
