def selective_copy_to(self, destination):
"""Write files declared in extra_files from the source directory
to another directory.
"""
for from_path, to_path in self._get_mapping(destination):
target_folder = os.path.dirname(to_path)
ensure_directory(target_folder)
shutil.copyfile(from_path, to_path)
def selective_copy_to(self, destination):
"""Write files from the source directory to another directory, skipping
files excluded by the general exclusion_policy, plus any files
ignored by git configuration.
"""
for from_path, to_path in self._get_mapping(destination):
target_folder = os.path.dirname(to_path)
ensure_directory(target_folder)
shutil.copyfile(from_path, to_path)
def selective_copy_to(self, destination):
"""Write files from the source directory to another directory, skipping
files excluded by the general exclusion_policy, plus any files
ignored by git configuration.
"""
for path in self.files:
subpath = os.path.relpath(path, start=self.root)
target_folder = os.path.join(destination, os.path.dirname(subpath))
ensure_directory(target_folder)
shutil.copy2(path, target_folder)
def assemble_experiment_temp_dir(config):
"""Create a temp directory from which to run an experiment.
The new directory will include:
- Copies of custom experiment files which don't match the exclusion policy
- Templates and static resources from Dallinger
- An export of the loaded configuration
- Heroku-specific files (Procile, runtime.txt) from Dallinger
Assumes the experiment root directory is the current working directory.
Returns the absolute path of the new directory.
"""
app_id = config.get("id")
dst = os.path.join(tempfile.mkdtemp(), app_id)
# Copy local experiment files, minus some
ExperimentFileSource(os.getcwd()).selective_copy_to(dst)
# Export the loaded configuration
config.write(filter_sensitive=True, directory=dst)
# Save the experiment id
with open(os.path.join(dst, "experiment_id.txt"), "w") as file:
file.write(app_id)
# Copy Dallinger files
dallinger_root = dallinger_package_path()
ensure_directory(os.path.join(dst, "static", "scripts"))
ensure_directory(os.path.join(dst, "static", "css"))
frontend_files = [
os.path.join("static", "css", "dallinger.css"),
os.path.join("static", "scripts", "dallinger2.js"),
os.path.join("static", "scripts", "reqwest.min.js"),
os.path.join("static", "scripts", "require.js"),
os.path.join("static", "scripts", "reconnecting-websocket.js"),
os.path.join("static", "scripts", "spin.min.js"),
os.path.join("static", "scripts", "tracker.js"),
os.path.join("static", "scripts", "store+json2.min.js"),
os.path.join("templates", "error.html"),
os.path.join("templates", "error-complete.html"),
os.path.join("templates", "launch.html"),
os.path.join("templates", "complete.html"),
os.path.join("templates", "questionnaire.html"),
os.path.join("templates", "thanks.html"),
os.path.join("templates", "waiting.html"),
os.path.join("static", "robots.txt"),
]
frontend_dirs = [os.path.join("templates", "base")]
for filename in frontend_files:
src = os.path.join(dallinger_root, "frontend", filename)
dst_filepath = os.path.join(dst, filename)
if not os.path.exists(dst_filepath):
shutil.copy(src, dst_filepath)
for filename in frontend_dirs:
src = os.path.join(dallinger_root, "frontend", filename)
dst_filepath = os.path.join(dst, filename)
if not os.path.exists(dst_filepath):
shutil.copytree(src, dst_filepath)
# Copy Heroku files
heroku_files = ["Procfile"]
for filename in heroku_files:
src = os.path.join(dallinger_root, "heroku", filename)
shutil.copy(src, os.path.join(dst, filename))
# Write out a runtime.txt file based on configuration
pyversion = config.get("heroku_python_version")
with open(os.path.join(dst, "runtime.txt"), "w") as file:
file.write("python-{}".format(pyversion))
if not config.get("clock_on"):
# If the clock process has been disabled, overwrite the Procfile:
src = os.path.join(dallinger_root, "heroku", "Procfile_no_clock")
shutil.copy(src, os.path.join(dst, "Procfile"))
return dst
def setup_experiment(debug=True, verbose=False, app=None, exp_config=None):
"""Check the app and, if compatible with Dallinger, freeze its state."""
log(header, chevrons=False)
# Verify that the Postgres server is running.
try:
psycopg2.connect(database="x", user="******", password="******")
except psycopg2.OperationalError as e:
if "could not connect to server" in str(e):
raise RuntimeError("The Postgres server isn't running.")
# Load configuration.
config = get_config()
if not config.ready:
config.load()
# Check that the demo-specific requirements are satisfied.
try:
with open("requirements.txt", "r") as f:
dependencies = [r for r in f.readlines() if r[:3] != "-e "]
except (OSError, IOError):
dependencies = []
pkg_resources.require(dependencies)
# Generate a unique id for this experiment.
from dallinger.experiment import Experiment
generated_uid = public_id = Experiment.make_uuid()
# If the user provided an app name, use it everywhere that's user-facing.
if app:
public_id = str(app)
log("Experiment id is " + public_id + "")
# Copy this directory into a temporary folder, ignoring .git
dst = os.path.join(tempfile.mkdtemp(), public_id)
to_ignore = shutil.ignore_patterns(os.path.join(".git", "*"), "*.db",
"snapshots", "data", "server.log")
shutil.copytree(os.getcwd(), dst, ignore=to_ignore)
click.echo(dst)
# Save the experiment id
with open(os.path.join(dst, "experiment_id.txt"), "w") as file:
file.write(generated_uid)
# Change directory to the temporary folder.
cwd = os.getcwd()
os.chdir(dst)
# Write the custom config
if exp_config:
config.extend(exp_config)
config.extend({'id': unicode(generated_uid)})
config.write(filter_sensitive=True)
# Zip up the temporary directory and place it in the cwd.
if not debug:
log("Freezing the experiment package...")
shutil.make_archive(
os.path.join(cwd, "snapshots", public_id + "-code"), "zip", dst)
# Check directories.
ensure_directory(os.path.join("static", "scripts"))
ensure_directory(os.path.join("templates", "default"))
ensure_directory(os.path.join("static", "css"))
# Rename experiment.py for backwards compatibility.
os.rename(os.path.join(dst, "experiment.py"),
os.path.join(dst, "dallinger_experiment.py"))
# Get dallinger package location.
from pkg_resources import get_distribution
dist = get_distribution('dallinger')
src_base = os.path.join(dist.location, dist.project_name)
heroku_files = [
"Procfile",
"launch.py",
"worker.py",
"clock.py",
"runtime.txt",
]
for filename in heroku_files:
src = os.path.join(src_base, "heroku", filename)
shutil.copy(src, os.path.join(dst, filename))
clock_on = config.get('clock_on', False)
# If the clock process has been disabled, overwrite the Procfile.
if not clock_on:
src = os.path.join(src_base, "heroku", "Procfile_no_clock")
shutil.copy(src, os.path.join(dst, "Procfile"))
frontend_files = [
os.path.join("static", "css", "dallinger.css"),
os.path.join("static", "scripts", "dallinger.js"),
os.path.join("static", "scripts", "dallinger2.js"),
os.path.join("static", "scripts", "reqwest.min.js"),
os.path.join("static", "scripts", "require.js"),
os.path.join("static", "scripts", "reconnecting-websocket.js"),
os.path.join("static", "scripts", "spin.min.js"),
os.path.join("static", "scripts", "tracker.js"),
os.path.join("templates", "error.html"),
os.path.join("templates", "launch.html"),
os.path.join("templates", "complete.html"),
os.path.join("templates", "questionnaire.html"),
os.path.join("templates", "thanks.html"),
os.path.join("templates", "waiting.html"),
os.path.join("static", "robots.txt")
]
frontend_dirs = [
os.path.join("templates", "base"),
]
for filename in frontend_files:
src = os.path.join(src_base, "frontend", filename)
dst_filepath = os.path.join(dst, filename)
if not os.path.exists(dst_filepath):
shutil.copy(src, dst_filepath)
for filename in frontend_dirs:
src = os.path.join(src_base, "frontend", filename)
dst_filepath = os.path.join(dst, filename)
if not os.path.exists(dst_filepath):
shutil.copytree(src, dst_filepath)
time.sleep(0.25)
os.chdir(cwd)
return (public_id, dst)
def assemble_experiment_temp_dir(log, config, for_remote=False):
"""Create a temp directory from which to run an experiment.
If for_remote is set to True the preparation includes bundling
the local dallinger version if it was installed in editable mode.
The new directory will include:
- Copies of custom experiment files which don't match the exclusion policy
- Templates and static resources from Dallinger
- An export of the loaded configuration
- Heroku-specific files (Procile, runtime.txt) from Dallinger
- A requirements.txt file with the contents from the constraints.txt file
in the experiment (Dallinger should have generated one with pip-compile
if needed by the time we reach this code)
- A dallinger zip (only if dallinger is installed in editable mode)
Assumes the experiment root directory is the current working directory.
Returns the absolute path of the new directory.
"""
exp_id = config.get("id")
dst = os.path.join(tempfile.mkdtemp(), exp_id)
# Copy local experiment files, minus some
ExperimentFileSource(os.getcwd()).selective_copy_to(dst)
# Export the loaded configuration
config.write(filter_sensitive=True, directory=dst)
# Save the experiment id
with open(os.path.join(dst, "experiment_id.txt"), "w") as file:
file.write(exp_id)
# Copy Dallinger files
dallinger_root = dallinger_package_path()
ensure_directory(os.path.join(dst, "static", "scripts"))
ensure_directory(os.path.join(dst, "static", "css"))
frontend_files = [
os.path.join("static", "css", "bootstrap.min.css"),
os.path.join("static", "css", "dallinger.css"),
os.path.join("static", "css", "dashboard.css"),
os.path.join("static", "scripts", "jquery-3.5.1.min.js"),
os.path.join("static", "scripts", "popper.min.js"),
os.path.join("static", "scripts", "bootstrap.min.js"),
os.path.join("static", "scripts", "clipboard.min.js"),
os.path.join("static", "scripts", "dallinger2.js"),
os.path.join("static", "scripts", "network-monitor.js"),
os.path.join("static", "scripts", "reqwest.min.js"),
os.path.join("static", "scripts", "require.js"),
os.path.join("static", "scripts", "reconnecting-websocket.js"),
os.path.join("static", "scripts", "spin.min.js"),
os.path.join("static", "scripts", "tracker.js"),
os.path.join("static", "scripts", "store+json2.min.js"),
os.path.join("templates", "error.html"),
os.path.join("templates", "error-complete.html"),
os.path.join("templates", "launch.html"),
os.path.join("templates", "questionnaire.html"),
os.path.join("templates", "exit_recruiter.html"),
os.path.join("templates", "exit_recruiter_mturk.html"),
os.path.join("templates", "waiting.html"),
os.path.join("templates", "login.html"),
os.path.join("templates", "dashboard_lifecycle.html"),
os.path.join("templates", "dashboard_database.html"),
os.path.join("templates", "dashboard_heroku.html"),
os.path.join("templates", "dashboard_home.html"),
os.path.join("templates", "dashboard_monitor.html"),
os.path.join("templates", "dashboard_mturk.html"),
os.path.join("static", "robots.txt"),
]
frontend_dirs = [os.path.join("templates", "base")]
for filename in frontend_files:
src = os.path.join(dallinger_root, "frontend", filename)
dst_filepath = os.path.join(dst, filename)
if not os.path.exists(dst_filepath):
shutil.copy(src, dst_filepath)
for filename in frontend_dirs:
src = os.path.join(dallinger_root, "frontend", filename)
dst_filepath = os.path.join(dst, filename)
if not os.path.exists(dst_filepath):
shutil.copytree(src, dst_filepath)
# Copy Heroku files
heroku_files = ["Procfile"]
for filename in heroku_files:
src = os.path.join(dallinger_root, "heroku", filename)
shutil.copy(src, os.path.join(dst, filename))
# Write out a runtime.txt file based on configuration
pyversion = config.get("heroku_python_version")
with open(os.path.join(dst, "runtime.txt"), "w") as file:
file.write("python-{}".format(pyversion))
if not config.get("clock_on"):
# If the clock process has been disabled, overwrite the Procfile:
src = os.path.join(dallinger_root, "heroku", "Procfile_no_clock")
shutil.copy(src, os.path.join(dst, "Procfile"))
ExplicitFileSource(os.getcwd()).selective_copy_to(dst)
requirements_path = Path(dst) / "requirements.txt"
# Overwrite the requirements.txt file with the contents of the constraints.txt file
(Path(dst) / "constraints.txt").replace(requirements_path)
if for_remote:
dallinger_path = get_editable_dallinger_path()
if dallinger_path:
log("Dallinger is installed as an editable package, "
"and so will be copied and deployed in its current state, "
"ignoring the dallinger version specified in your experiment's "
"requirements.txt file!")
egg_name = build_and_place(dallinger_path, dst)
# Replace the line about dallinger in requirements.txt so that
# it refers to the just generated package
constraints_text = requirements_path.read_text()
new_constraints_text = re.sub("dallinger==.*", f"file:{egg_name}",
constraints_text)
requirements_path.write_text(new_constraints_text)
return dst
