def selective_copy_to(self, destination): """Write files declared in extra_files from the source directory to another directory. """ for from_path, to_path in self._get_mapping(destination): target_folder = os.path.dirname(to_path) ensure_directory(target_folder) shutil.copyfile(from_path, to_path)
def selective_copy_to(self, destination): """Write files from the source directory to another directory, skipping files excluded by the general exclusion_policy, plus any files ignored by git configuration. """ for from_path, to_path in self._get_mapping(destination): target_folder = os.path.dirname(to_path) ensure_directory(target_folder) shutil.copyfile(from_path, to_path)
def selective_copy_to(self, destination): """Write files from the source directory to another directory, skipping files excluded by the general exclusion_policy, plus any files ignored by git configuration. """ for path in self.files: subpath = os.path.relpath(path, start=self.root) target_folder = os.path.join(destination, os.path.dirname(subpath)) ensure_directory(target_folder) shutil.copy2(path, target_folder)
def assemble_experiment_temp_dir(config): """Create a temp directory from which to run an experiment. The new directory will include: - Copies of custom experiment files which don't match the exclusion policy - Templates and static resources from Dallinger - An export of the loaded configuration - Heroku-specific files (Procile, runtime.txt) from Dallinger Assumes the experiment root directory is the current working directory. Returns the absolute path of the new directory. """ app_id = config.get("id") dst = os.path.join(tempfile.mkdtemp(), app_id) # Copy local experiment files, minus some ExperimentFileSource(os.getcwd()).selective_copy_to(dst) # Export the loaded configuration config.write(filter_sensitive=True, directory=dst) # Save the experiment id with open(os.path.join(dst, "experiment_id.txt"), "w") as file: file.write(app_id) # Copy Dallinger files dallinger_root = dallinger_package_path() ensure_directory(os.path.join(dst, "static", "scripts")) ensure_directory(os.path.join(dst, "static", "css")) frontend_files = [ os.path.join("static", "css", "dallinger.css"), os.path.join("static", "scripts", "dallinger2.js"), os.path.join("static", "scripts", "reqwest.min.js"), os.path.join("static", "scripts", "require.js"), os.path.join("static", "scripts", "reconnecting-websocket.js"), os.path.join("static", "scripts", "spin.min.js"), os.path.join("static", "scripts", "tracker.js"), os.path.join("static", "scripts", "store+json2.min.js"), os.path.join("templates", "error.html"), os.path.join("templates", "error-complete.html"), os.path.join("templates", "launch.html"), os.path.join("templates", "complete.html"), os.path.join("templates", "questionnaire.html"), os.path.join("templates", "thanks.html"), os.path.join("templates", "waiting.html"), os.path.join("static", "robots.txt"), ] frontend_dirs = [os.path.join("templates", "base")] for filename in frontend_files: src = os.path.join(dallinger_root, "frontend", filename) dst_filepath = os.path.join(dst, filename) if not os.path.exists(dst_filepath): shutil.copy(src, dst_filepath) for filename in frontend_dirs: src = os.path.join(dallinger_root, "frontend", filename) dst_filepath = os.path.join(dst, filename) if not os.path.exists(dst_filepath): shutil.copytree(src, dst_filepath) # Copy Heroku files heroku_files = ["Procfile"] for filename in heroku_files: src = os.path.join(dallinger_root, "heroku", filename) shutil.copy(src, os.path.join(dst, filename)) # Write out a runtime.txt file based on configuration pyversion = config.get("heroku_python_version") with open(os.path.join(dst, "runtime.txt"), "w") as file: file.write("python-{}".format(pyversion)) if not config.get("clock_on"): # If the clock process has been disabled, overwrite the Procfile: src = os.path.join(dallinger_root, "heroku", "Procfile_no_clock") shutil.copy(src, os.path.join(dst, "Procfile")) return dst
def setup_experiment(debug=True, verbose=False, app=None, exp_config=None): """Check the app and, if compatible with Dallinger, freeze its state.""" log(header, chevrons=False) # Verify that the Postgres server is running. try: psycopg2.connect(database="x", user="******", password="******") except psycopg2.OperationalError as e: if "could not connect to server" in str(e): raise RuntimeError("The Postgres server isn't running.") # Load configuration. config = get_config() if not config.ready: config.load() # Check that the demo-specific requirements are satisfied. try: with open("requirements.txt", "r") as f: dependencies = [r for r in f.readlines() if r[:3] != "-e "] except (OSError, IOError): dependencies = [] pkg_resources.require(dependencies) # Generate a unique id for this experiment. from dallinger.experiment import Experiment generated_uid = public_id = Experiment.make_uuid() # If the user provided an app name, use it everywhere that's user-facing. if app: public_id = str(app) log("Experiment id is " + public_id + "") # Copy this directory into a temporary folder, ignoring .git dst = os.path.join(tempfile.mkdtemp(), public_id) to_ignore = shutil.ignore_patterns(os.path.join(".git", "*"), "*.db", "snapshots", "data", "server.log") shutil.copytree(os.getcwd(), dst, ignore=to_ignore) click.echo(dst) # Save the experiment id with open(os.path.join(dst, "experiment_id.txt"), "w") as file: file.write(generated_uid) # Change directory to the temporary folder. cwd = os.getcwd() os.chdir(dst) # Write the custom config if exp_config: config.extend(exp_config) config.extend({'id': unicode(generated_uid)}) config.write(filter_sensitive=True) # Zip up the temporary directory and place it in the cwd. if not debug: log("Freezing the experiment package...") shutil.make_archive( os.path.join(cwd, "snapshots", public_id + "-code"), "zip", dst) # Check directories. ensure_directory(os.path.join("static", "scripts")) ensure_directory(os.path.join("templates", "default")) ensure_directory(os.path.join("static", "css")) # Rename experiment.py for backwards compatibility. os.rename(os.path.join(dst, "experiment.py"), os.path.join(dst, "dallinger_experiment.py")) # Get dallinger package location. from pkg_resources import get_distribution dist = get_distribution('dallinger') src_base = os.path.join(dist.location, dist.project_name) heroku_files = [ "Procfile", "launch.py", "worker.py", "clock.py", "runtime.txt", ] for filename in heroku_files: src = os.path.join(src_base, "heroku", filename) shutil.copy(src, os.path.join(dst, filename)) clock_on = config.get('clock_on', False) # If the clock process has been disabled, overwrite the Procfile. if not clock_on: src = os.path.join(src_base, "heroku", "Procfile_no_clock") shutil.copy(src, os.path.join(dst, "Procfile")) frontend_files = [ os.path.join("static", "css", "dallinger.css"), os.path.join("static", "scripts", "dallinger.js"), os.path.join("static", "scripts", "dallinger2.js"), os.path.join("static", "scripts", "reqwest.min.js"), os.path.join("static", "scripts", "require.js"), os.path.join("static", "scripts", "reconnecting-websocket.js"), os.path.join("static", "scripts", "spin.min.js"), os.path.join("static", "scripts", "tracker.js"), os.path.join("templates", "error.html"), os.path.join("templates", "launch.html"), os.path.join("templates", "complete.html"), os.path.join("templates", "questionnaire.html"), os.path.join("templates", "thanks.html"), os.path.join("templates", "waiting.html"), os.path.join("static", "robots.txt") ] frontend_dirs = [ os.path.join("templates", "base"), ] for filename in frontend_files: src = os.path.join(src_base, "frontend", filename) dst_filepath = os.path.join(dst, filename) if not os.path.exists(dst_filepath): shutil.copy(src, dst_filepath) for filename in frontend_dirs: src = os.path.join(src_base, "frontend", filename) dst_filepath = os.path.join(dst, filename) if not os.path.exists(dst_filepath): shutil.copytree(src, dst_filepath) time.sleep(0.25) os.chdir(cwd) return (public_id, dst)
def assemble_experiment_temp_dir(log, config, for_remote=False): """Create a temp directory from which to run an experiment. If for_remote is set to True the preparation includes bundling the local dallinger version if it was installed in editable mode. The new directory will include: - Copies of custom experiment files which don't match the exclusion policy - Templates and static resources from Dallinger - An export of the loaded configuration - Heroku-specific files (Procile, runtime.txt) from Dallinger - A requirements.txt file with the contents from the constraints.txt file in the experiment (Dallinger should have generated one with pip-compile if needed by the time we reach this code) - A dallinger zip (only if dallinger is installed in editable mode) Assumes the experiment root directory is the current working directory. Returns the absolute path of the new directory. """ exp_id = config.get("id") dst = os.path.join(tempfile.mkdtemp(), exp_id) # Copy local experiment files, minus some ExperimentFileSource(os.getcwd()).selective_copy_to(dst) # Export the loaded configuration config.write(filter_sensitive=True, directory=dst) # Save the experiment id with open(os.path.join(dst, "experiment_id.txt"), "w") as file: file.write(exp_id) # Copy Dallinger files dallinger_root = dallinger_package_path() ensure_directory(os.path.join(dst, "static", "scripts")) ensure_directory(os.path.join(dst, "static", "css")) frontend_files = [ os.path.join("static", "css", "bootstrap.min.css"), os.path.join("static", "css", "dallinger.css"), os.path.join("static", "css", "dashboard.css"), os.path.join("static", "scripts", "jquery-3.5.1.min.js"), os.path.join("static", "scripts", "popper.min.js"), os.path.join("static", "scripts", "bootstrap.min.js"), os.path.join("static", "scripts", "clipboard.min.js"), os.path.join("static", "scripts", "dallinger2.js"), os.path.join("static", "scripts", "network-monitor.js"), os.path.join("static", "scripts", "reqwest.min.js"), os.path.join("static", "scripts", "require.js"), os.path.join("static", "scripts", "reconnecting-websocket.js"), os.path.join("static", "scripts", "spin.min.js"), os.path.join("static", "scripts", "tracker.js"), os.path.join("static", "scripts", "store+json2.min.js"), os.path.join("templates", "error.html"), os.path.join("templates", "error-complete.html"), os.path.join("templates", "launch.html"), os.path.join("templates", "questionnaire.html"), os.path.join("templates", "exit_recruiter.html"), os.path.join("templates", "exit_recruiter_mturk.html"), os.path.join("templates", "waiting.html"), os.path.join("templates", "login.html"), os.path.join("templates", "dashboard_lifecycle.html"), os.path.join("templates", "dashboard_database.html"), os.path.join("templates", "dashboard_heroku.html"), os.path.join("templates", "dashboard_home.html"), os.path.join("templates", "dashboard_monitor.html"), os.path.join("templates", "dashboard_mturk.html"), os.path.join("static", "robots.txt"), ] frontend_dirs = [os.path.join("templates", "base")] for filename in frontend_files: src = os.path.join(dallinger_root, "frontend", filename) dst_filepath = os.path.join(dst, filename) if not os.path.exists(dst_filepath): shutil.copy(src, dst_filepath) for filename in frontend_dirs: src = os.path.join(dallinger_root, "frontend", filename) dst_filepath = os.path.join(dst, filename) if not os.path.exists(dst_filepath): shutil.copytree(src, dst_filepath) # Copy Heroku files heroku_files = ["Procfile"] for filename in heroku_files: src = os.path.join(dallinger_root, "heroku", filename) shutil.copy(src, os.path.join(dst, filename)) # Write out a runtime.txt file based on configuration pyversion = config.get("heroku_python_version") with open(os.path.join(dst, "runtime.txt"), "w") as file: file.write("python-{}".format(pyversion)) if not config.get("clock_on"): # If the clock process has been disabled, overwrite the Procfile: src = os.path.join(dallinger_root, "heroku", "Procfile_no_clock") shutil.copy(src, os.path.join(dst, "Procfile")) ExplicitFileSource(os.getcwd()).selective_copy_to(dst) requirements_path = Path(dst) / "requirements.txt" # Overwrite the requirements.txt file with the contents of the constraints.txt file (Path(dst) / "constraints.txt").replace(requirements_path) if for_remote: dallinger_path = get_editable_dallinger_path() if dallinger_path: log("Dallinger is installed as an editable package, " "and so will be copied and deployed in its current state, " "ignoring the dallinger version specified in your experiment's " "requirements.txt file!") egg_name = build_and_place(dallinger_path, dst) # Replace the line about dallinger in requirements.txt so that # it refers to the just generated package constraints_text = requirements_path.read_text() new_constraints_text = re.sub("dallinger==.*", f"file:{egg_name}", constraints_text) requirements_path.write_text(new_constraints_text) return dst