def _get_mapping(self, dst): from dallinger.config import initialize_experiment_package initialize_experiment_package(dst) from dallinger.experiment import load exp_class = load() extra_files = getattr(exp_class, "extra_files", None) if extra_files is None: try: from dallinger_experiment.experiment import extra_files except ImportError: try: from dallinger_experiment.dallinger_experiment import extra_files except ImportError: pass if extra_files is not None: for src, filename in extra_files(): filename = filename.lstrip("/") if os.path.isdir(src): for dirpath, dirnames, filenames in os.walk(src, topdown=True): for fn in filenames: dst_fileparts = [dst, filename] + dirnames + [fn] dst_filepath = os.path.join(*dst_fileparts) yield ( os.path.join(dirpath, fn), dst_filepath, ) else: dst_filepath = os.path.join(dst, filename) yield (src, dst_filepath)
def main(): import gevent.monkey gevent.monkey.patch_all() from gevent.queue import LifoQueue # These imports are inside the __main__ block # to make sure that we only import from rq_gevent_worker # (which has the side effect of applying gevent monkey patches) # in the worker process. This way other processes can import the # redis connection without that side effect. import os from redis import BlockingConnectionPool, StrictRedis from rq import Queue, Connection from dallinger.heroku.rq_gevent_worker import GeventWorker as Worker from dallinger.config import initialize_experiment_package initialize_experiment_package(os.getcwd()) import logging logging.basicConfig(format="%(asctime)s %(message)s", level=logging.DEBUG) redis_url = os.getenv("REDIS_URL", "redis://localhost:6379") # Specify queue class for improved performance with gevent. # see http://carsonip.me/posts/10x-faster-python-gevent-redis-connection-pool/ redis_pool = BlockingConnectionPool.from_url(redis_url, queue_class=LifoQueue) redis_conn = StrictRedis(connection_pool=redis_pool) with Connection(redis_conn): worker = Worker(list(map(Queue, listen))) worker.work()
def verify_experiment_module(verbose): """Perform basic sanity checks on experiment.py. """ ok = True if not os.path.exists("experiment.py"): return False # Bootstrap a package in a temp directory and make it importable: temp_package_name = "TEMP_VERIFICATION_PACKAGE" tmp = tempfile.mkdtemp() clone_dir = os.path.join(tmp, temp_package_name) to_ignore = shutil.ignore_patterns( os.path.join(".git", "*"), "*.db", "snapshots", "data", "server.log" ) shutil.copytree(os.getcwd(), clone_dir, ignore=to_ignore) initialize_experiment_package(clone_dir) from dallinger_experiment import experiment if clone_dir not in experiment.__file__: raise ImportError("Checking the wrong experiment.py... aborting.") classes = inspect.getmembers(experiment, inspect.isclass) exps = [c for c in classes if (c[1].__bases__[0].__name__ in "Experiment")] # Clean up: for entry in [k for k in sys.modules if temp_package_name in k]: del sys.modules[entry] # Run checks: if len(exps) == 0: log( "✗ experiment.py does not define an experiment class.", delay=0, chevrons=False, verbose=verbose, ) ok = False elif len(exps) == 1: log( "✓ experiment.py defines 1 experiment", delay=0, chevrons=False, verbose=verbose, ) else: log( "✗ experiment.py defines more than one experiment class.", delay=0, chevrons=False, verbose=verbose, ) ok = False return ok
def load(): """Load the active experiment.""" initialize_experiment_package(os.getcwd()) try: try: from dallinger_experiment import experiment except ImportError: from dallinger_experiment import dallinger_experiment as experiment classes = inspect.getmembers(experiment, inspect.isclass) for name, c in classes: if 'Experiment' in c.__bases__[0].__name__: return c else: raise ImportError except ImportError: logger.error('Could not import experiment.') raise
def load(): """Load the active experiment.""" initialize_experiment_package(os.getcwd()) try: try: from dallinger_experiment import experiment except ImportError: from dallinger_experiment import dallinger_experiment as experiment classes = inspect.getmembers(experiment, is_experiment_class) preferred_class = os.environ.get("EXPERIMENT_CLASS_NAME", None) if preferred_class is not None: try: return dict(classes)[preferred_class] except KeyError: raise ImportError( "No experiment named {} was found".format(preferred_class) ) if len(classes) > 1: for name, c in classes: if "Experiment" in c.__bases__[0].__name__: warnings.warn( UserWarning( "More than one potential experiment class found but no EXPERIMENT_CLASS_NAME environment variable. Picking {} from {}.".format( name, [n for (n, cls) in classes] ) ), stacklevel=3, ) return c raise ImportError( "No direct experiment subclass found in {}".format( [n for (n, cls) in classes] ) ) elif len(classes) == 0: raise ImportError("No experiment classes found") else: return classes[0][1] except ImportError: logger.error("Could not import experiment.") raise
def main(): import gevent.monkey gevent.monkey.patch_all() from gevent.queue import LifoQueue # These imports are inside the __main__ block # to make sure that we only import from rq_gevent_worker # (which has the side effect of applying gevent monkey patches) # in the worker process. This way other processes can import the # redis connection without that side effect. import logging import os from redis import BlockingConnectionPool, StrictRedis from rq import Queue, Connection from six.moves.urllib.parse import urlparse from dallinger.heroku.rq_gevent_worker import GeventWorker as Worker from dallinger.config import initialize_experiment_package initialize_experiment_package(os.getcwd()) logging.basicConfig(format="%(asctime)s %(message)s", level=logging.DEBUG) redis_url = os.getenv("REDIS_URL", "redis://localhost:6379") # Specify queue class for improved performance with gevent. # see http://carsonip.me/posts/10x-faster-python-gevent-redis-connection-pool/ connection_args = { "url": redis_url, "queue_class": LifoQueue, } # Since we are generally running on Heroku, and configuring SSL certificates # is challenging, we disable cert requirements on secure connections. if urlparse(redis_url).scheme == "rediss": connection_args["ssl_cert_reqs"] = None redis_pool = BlockingConnectionPool.from_url(**connection_args) redis_conn = StrictRedis(connection_pool=redis_pool) with Connection(redis_conn): worker = Worker(list(map(Queue, listen))) worker.work()
def init_db(drop_all=False, bind=engine): """Initialize the database, optionally dropping existing tables.""" # To create the db structure according to the experiment configuration # we need to import the experiment code, so that sqlalchemy has a chance # to update its metadata initialize_experiment_package(os.getcwd()) try: from dallinger_experiment import experiment # noqa: F401 except ImportError: pass try: if drop_all: Base.metadata.drop_all(bind=bind) Base.metadata.create_all(bind=bind) except OperationalError as err: msg = 'password authentication failed for user "dallinger"' if msg in err.message: sys.stderr.write(db_user_warning) raise return session
listen = ['high', 'default', 'low'] redis_url = os.getenv('REDIS_URL', 'redis://localhost:6379') conn = redis.from_url(redis_url) if __name__ == '__main__': # pragma: nocover # Make sure gevent patches are applied early. import gevent.monkey gevent.monkey.patch_all() # These imports are inside the __main__ block # to make sure that we only import from rq_gevent_worker # (which has the side effect of applying gevent monkey patches) # in the worker process. This way other processes can import the # redis connection without that side effect. from rq import ( Queue, Connection ) from dallinger.heroku.rq_gevent_worker import GeventWorker as Worker from dallinger.config import initialize_experiment_package initialize_experiment_package(os.getcwd()) import logging logging.basicConfig(format='%(asctime)s %(message)s', level=logging.INFO) with Connection(conn): worker = Worker(list(map(Queue, listen))) worker.work()
def verify_package(verbose=True): """Ensure the package has a config file and a valid experiment file.""" is_passing = True # Check for existence of required files. required_files = [ "config.txt", "experiment.py", ] for f in required_files: if os.path.exists(f): log("✓ {} is PRESENT".format(f), chevrons=False, verbose=verbose) else: log("✗ {} is MISSING".format(f), chevrons=False, verbose=verbose) is_passing = False # Check the experiment file. if os.path.exists("experiment.py"): # Check if the experiment file has exactly one Experiment class. tmp = tempfile.mkdtemp() clone_dir = os.path.join(tmp, 'temp_exp_package') to_ignore = shutil.ignore_patterns(os.path.join(".git", "*"), "*.db", "snapshots", "data", "server.log") shutil.copytree(os.getcwd(), clone_dir, ignore=to_ignore) initialize_experiment_package(clone_dir) from dallinger_experiment import experiment classes = inspect.getmembers(experiment, inspect.isclass) exps = [ c for c in classes if (c[1].__bases__[0].__name__ in "Experiment") ] if len(exps) == 0: log("✗ experiment.py does not define an experiment class.", delay=0, chevrons=False, verbose=verbose) is_passing = False elif len(exps) == 1: log("✓ experiment.py defines 1 experiment", delay=0, chevrons=False, verbose=verbose) else: log("✗ experiment.py defines more than one experiment class.", delay=0, chevrons=False, verbose=verbose) config = get_config() if not config.ready: config.load() # Check base_payment is correct base_pay = config.get('base_payment') dollarFormat = "{:.2f}".format(base_pay) if base_pay <= 0: log("✗ base_payment must be positive value in config.txt.", delay=0, chevrons=False, verbose=verbose) is_passing = False if float(dollarFormat) != float(base_pay): log("✗ base_payment must be in [dollars].[cents] format in config.txt. Try changing " "{0} to {1}.".format(base_pay, dollarFormat), delay=0, chevrons=False, verbose=verbose) is_passing = False # Check front-end files do not exist files = [ os.path.join("templates", "complete.html"), os.path.join("templates", "error.html"), os.path.join("templates", "error-complete.html"), os.path.join("templates", "launch.html"), os.path.join("templates", "thanks.html"), os.path.join("static", "css", "dallinger.css"), os.path.join("static", "scripts", "dallinger.js"), os.path.join("static", "scripts", "dallinger2.js"), os.path.join("static", "scripts", "reqwest.min.js"), os.path.join("static", "scripts", "tracker.js"), os.path.join("static", "robots.txt") ] for f in files: if os.path.exists(f): log("✗ {} OVERWRITES shared frontend files inserted at run-time". format(f), delay=0, chevrons=False, verbose=verbose) log("✓ no file conflicts", delay=0, chevrons=False, verbose=verbose) return is_passing