def temp_repo(self, in_tempdir, stub_config): from dallinger.utils import GitClient stub_config.write() config = {'user.name': 'Test User', 'user.email': '*****@*****.**'} git = GitClient() git.init(config=config) git.add("--all") git.commit("Test Repo")
def test_tests(self): from dallinger.utils import GitClient from dallinger.utils import run_command original_dir = os.getcwd() git = GitClient() gudir = git.clone('https://github.com/Dallinger/Griduniverse.git') os.chdir(gudir) cmd = [sys.executable, 'setup.py', 'develop'] st = run_command(cmd, sys.stdout) cmd = [sys.executable, '-m', 'pytest'] st = run_command(cmd, sys.stdout, ignore_errors=True) os.chdir(original_dir) shutil.rmtree(gudir, ignore_errors=True) print("Grid Universe tests finished running.") if st != 0: print("Some tests failed.") assert st == 0
class ExperimentFileSource(object): """Treat an experiment directory as a potential source of files for copying to a temp directory as part of a deployment (debug or otherwise). """ def __init__(self, root_dir="."): self.root = root_dir self.git = GitClient() @property def files(self): """A Set of all files copyable in the source directory, accounting for exclusions. """ return {path for path in self._walk()} @property def size(self): """Combined size of all files, accounting for exclusions. """ return sum([os.path.getsize(path) for path in self._walk()]) def selective_copy_to(self, destination): """Write files from the source directory to another directory, skipping files excluded by the general exclusion_policy, plus any files ignored by git configuration. """ for path in self.files: subpath = os.path.relpath(path, start=self.root) target_folder = os.path.join(destination, os.path.dirname(subpath)) ensure_directory(target_folder) shutil.copy2(path, target_folder) def _walk(self): exclusions = exclusion_policy() git_files = set([os.path.join(self.root, f) for f in self.git.files()]) for dirpath, dirnames, filenames in os.walk(self.root, topdown=True): current_exclusions = exclusions(dirpath, os.listdir(dirpath)) # Modifying dirnames in-place will prune the subsequent files and # directories visited by os.walk. This is only possible when # topdown = True dirnames[:] = [d for d in dirnames if d not in current_exclusions] legit_files = { os.path.join(dirpath, f) for f in filenames if f not in current_exclusions and os.path.join(dirpath, f) } if git_files: legit_files = legit_files.intersection(git_files) for legit in legit_files: yield legit
def git(self): from dallinger.utils import GitClient git = GitClient() return git
def __init__(self, root_dir="."): self.root = root_dir self.git = GitClient()
def deploy_sandbox_shared_setup(log, verbose=True, app=None, exp_config=None): """Set up Git, push to Heroku, and launch the app.""" if verbose: out = None else: out = open(os.devnull, "w") config = get_config() if not config.ready: config.load() heroku.sanity_check(config) (id, tmp) = setup_experiment(log, debug=False, app=app, exp_config=exp_config) # Register the experiment using all configured registration services. if config.get("mode") == "live": log("Registering the experiment on configured services...") registration.register(id, snapshot=None) # Log in to Heroku if we aren't already. log("Making sure that you are logged in to Heroku.") heroku.log_in() config.set("heroku_auth_token", heroku.auth_token()) log("", chevrons=False) # Change to temporary directory. cwd = os.getcwd() os.chdir(tmp) # Commit Heroku-specific files to tmp folder's git repo. git = GitClient(output=out) git.init() git.add("--all") git.commit('"Experiment {}"'.format(id)) # Initialize the app on Heroku. log("Initializing app on Heroku...") team = config.get("heroku_team", None) heroku_app = HerokuApp(dallinger_uid=id, output=out, team=team) heroku_app.bootstrap() heroku_app.buildpack( "https://github.com/stomita/heroku-buildpack-phantomjs") # Set up add-ons and AWS environment variables. database_size = config.get("database_size") redis_size = config.get("redis_size") addons = [ "heroku-postgresql:{}".format(quote(database_size)), "heroku-redis:{}".format(quote(redis_size)), "papertrail", ] if config.get("sentry"): addons.append("sentry") for name in addons: heroku_app.addon(name) heroku_config = { "aws_access_key_id": config["aws_access_key_id"], "aws_secret_access_key": config["aws_secret_access_key"], "aws_region": config["aws_region"], "auto_recruit": config["auto_recruit"], "smtp_username": config["smtp_username"], "smtp_password": config["smtp_password"], "whimsical": config["whimsical"], } heroku_app.set_multiple(**heroku_config) # Wait for Redis database to be ready. log("Waiting for Redis...") ready = False while not ready: try: r = redis.from_url(heroku_app.redis_url) r.set("foo", "bar") ready = True except (ValueError, redis.exceptions.ConnectionError): time.sleep(2) log("Saving the URL of the postgres database...") config.extend({"database_url": heroku_app.db_url}) config.write() git.add("config.txt") time.sleep(0.25) git.commit("Save URL for database") time.sleep(0.25) # Launch the Heroku app. log("Pushing code to Heroku...") git.push(remote="heroku", branch="HEAD:master") log("Scaling up the dynos...") size = config.get("dyno_type") for process in ["web", "worker"]: qty = config.get("num_dynos_" + process) heroku_app.scale_up_dyno(process, qty, size) if config.get("clock_on"): heroku_app.scale_up_dyno("clock", 1, size) time.sleep(8) # Launch the experiment. log("Launching the experiment on the remote server and starting recruitment..." ) launch_data = _handle_launch_data("{}/launch".format(heroku_app.url), error=log) result = { "app_name": heroku_app.name, "app_home": heroku_app.url, "recruitment_msg": launch_data.get("recruitment_msg", None), } log("Experiment details:") log("App home: {}".format(result["app_home"]), chevrons=False) log("Recruiter info:") log(result["recruitment_msg"], chevrons=False) # Return to the branch whence we came. os.chdir(cwd) log("Completed deployment of experiment " + id + ".") return result
def deploy_sandbox_shared_setup(verbose=True, app=None, exp_config=None): """Set up Git, push to Heroku, and launch the app.""" if verbose: out = None else: out = open(os.devnull, 'w') (id, tmp) = setup_experiment(debug=False, verbose=verbose, app=app, exp_config=exp_config) config = get_config() # We know it's ready; setup_experiment() does this. # Register the experiment using all configured registration services. if config.get("mode") == u"live": log("Registering the experiment on configured services...") registration.register(id, snapshot=None) # Log in to Heroku if we aren't already. log("Making sure that you are logged in to Heroku.") heroku.log_in() config.set("heroku_auth_token", heroku.auth_token()) click.echo("") # Change to temporary directory. cwd = os.getcwd() os.chdir(tmp) # Commit Heroku-specific files to tmp folder's git repo. git = GitClient(output=out) git.init() git.add("--all") git.commit('"Experiment {}"'.format(id)) # Initialize the app on Heroku. log("Initializing app on Heroku...") team = config.get("heroku_team", '').strip() or None heroku_app = HerokuApp(dallinger_uid=id, output=out, team=team) heroku_app.bootstrap() heroku_app.buildpack( "https://github.com/stomita/heroku-buildpack-phantomjs") # Set up add-ons and AWS environment variables. database_size = config.get('database_size') addons = [ "heroku-postgresql:{}".format(quote(database_size)), "heroku-redis:premium-0", "papertrail" ] if config.get("sentry", False): addons.append("sentry") for name in addons: heroku_app.addon(name) heroku_config = { "aws_access_key_id": config["aws_access_key_id"], "aws_secret_access_key": config["aws_secret_access_key"], "aws_region": config["aws_region"], "auto_recruit": config["auto_recruit"], "dallinger_email_username": config["dallinger_email_address"], "dallinger_email_key": config["dallinger_email_password"], "whimsical": config["whimsical"], } for k, v in sorted(heroku_config.items()): # sorted for testablility heroku_app.set(k, v) # Wait for Redis database to be ready. log("Waiting for Redis...") ready = False while not ready: r = redis.from_url(heroku_app.redis_url) try: r.set("foo", "bar") ready = True except redis.exceptions.ConnectionError: time.sleep(2) log("Saving the URL of the postgres database...") # Set the notification URL and database URL in the config file. config.extend({ "notification_url": heroku_app.url + u"/notifications", "database_url": heroku_app.db_url, }) config.write() git.add("config.txt") time.sleep(0.25) git.commit("Save URLs for database and notifications") time.sleep(0.25) # Launch the Heroku app. log("Pushing code to Heroku...") git.push(remote="heroku", branch="HEAD:master") log("Scaling up the dynos...") size = config.get("dyno_type") for process in ["web", "worker"]: qty = config.get("num_dynos_" + process) heroku_app.scale_up_dyno(process, qty, size) if config.get("clock_on"): heroku_app.scale_up_dyno("clock", 1, size) time.sleep(8) # Launch the experiment. log("Launching the experiment on the remote server and starting recruitment..." ) launch_data = _handle_launch_data('{}/launch'.format(heroku_app.url)) result = { 'app_name': heroku_app.name, 'app_home': heroku_app.url, 'recruitment_msg': launch_data.get('recruitment_msg', None), } log("Experiment details:") log("App home: {}".format(result['app_home']), chevrons=False) log("Recruiter info:") log(result['recruitment_msg'], chevrons=False) # Return to the branch whence we came. os.chdir(cwd) log("Completed deployment of experiment " + id + ".") return result
class ExperimentFileSource(object): """Treat an experiment directory as a potential source of files for copying to a temp directory as part of a deployment (debug or otherwise). """ def __init__(self, root_dir="."): self.root = root_dir self.git = GitClient() @property def files(self): """A Set of all files copyable in the source directory, accounting for exclusions. """ return {path for path in self._walk()} @property def size(self): """Combined size of all files, accounting for exclusions. """ return sum([os.path.getsize(path) for path in self._walk()]) def selective_copy_to(self, destination): """Write files from the source directory to another directory, skipping files excluded by the general exclusion_policy, plus any files ignored by git configuration. """ for path in self.files: subpath = os.path.relpath(path, start=self.root) target_folder = os.path.join(destination, os.path.dirname(subpath)) ensure_directory(target_folder) shutil.copy2(path, target_folder) def _walk(self): # The GitClient and os.walk may return different representations of the # same unicode characters, so we use unicodedata.normalize() for # comparisons: # list(name_from_git) # ['å', ' ', 'f', 'i', 'l', 'e', '.', 't', 'x', 't'] # list(from_os_walk) # ['a', '̊', ' ', 'f', 'i', 'l', 'e', '.', 't', 'x', 't'] exclusions = exclusion_policy() git_files = { os.path.join(self.root, normalize("NFC", f)) for f in self.git.files() } for dirpath, dirnames, filenames in os.walk(self.root, topdown=True): current_exclusions = exclusions(dirpath, os.listdir(dirpath)) # Modifying dirnames in-place will prune the subsequent files and # directories visited by os.walk. This is only possible when # topdown = True dirnames[:] = [d for d in dirnames if d not in current_exclusions] legit_files = { os.path.join(dirpath, f) for f in filenames if f not in current_exclusions } if git_files: normalized = { normalize("NFC", six.text_type(f)): f for f in legit_files } legit_files = {v for k, v in normalized.items() if k in git_files} for legit in legit_files: yield legit
def git(self): from dallinger.utils import GitClient return GitClient()
def deploy_sandbox_shared_setup(log, verbose=True, app=None, exp_config=None, prelaunch_actions=None): """Set up Git, push to Heroku, and launch the app.""" if verbose: out = None else: out = open(os.devnull, "w") config = get_config() if not config.ready: config.load() heroku.sanity_check(config) (heroku_app_id, tmp) = setup_experiment(log, debug=False, app=app, exp_config=exp_config) # Register the experiment using all configured registration services. if config.get("mode") == "live": log("Registering the experiment on configured services...") registration.register(heroku_app_id, snapshot=None) # Log in to Heroku if we aren't already. log("Making sure that you are logged in to Heroku.") heroku.log_in() config.set("heroku_auth_token", heroku.auth_token()) log("", chevrons=False) # Change to temporary directory. cwd = os.getcwd() os.chdir(tmp) # Commit Heroku-specific files to tmp folder's git repo. git = GitClient(output=out) git.init() git.add("--all") git.commit('"Experiment {}"'.format(heroku_app_id)) # Initialize the app on Heroku. log("Initializing app on Heroku...") team = config.get("heroku_team", None) heroku_app = HerokuApp(dallinger_uid=heroku_app_id, output=out, team=team) heroku_app.bootstrap() heroku_app.buildpack( "https://github.com/stomita/heroku-buildpack-phantomjs") heroku_app.set("PYTHON_NO_SQLITE3", "true") # Set up add-ons and AWS environment variables. database_size = config.get("database_size") redis_size = config.get("redis_size") addons = [ "heroku-postgresql:{}".format(quote(database_size)), "heroku-redis:{}".format(quote(redis_size)), "papertrail", ] if config.get("sentry"): addons.append("sentry") for name in addons: heroku_app.addon(name) heroku_config = { "aws_access_key_id": config["aws_access_key_id"], "aws_secret_access_key": config["aws_secret_access_key"], "aws_region": config["aws_region"], "auto_recruit": config["auto_recruit"], "smtp_username": config["smtp_username"], "smtp_password": config["smtp_password"], "whimsical": config["whimsical"], "FLASK_SECRET_KEY": codecs.encode(os.urandom(16), "hex").decode("ascii"), } # Set up the preferred class as an environment variable, if one is set # This is needed before the config is parsed, but we also store it in the # config to make things easier for recording into bundles. preferred_class = config.get("EXPERIMENT_CLASS_NAME", None) if preferred_class: heroku_config["EXPERIMENT_CLASS_NAME"] = preferred_class heroku_app.set_multiple(**heroku_config) # Wait for Redis database to be ready. log("Waiting for Redis...", nl=False) ready = False while not ready: try: r = connect_to_redis(url=heroku_app.redis_url) r.set("foo", "bar") ready = True log("\n✓ connected at {}".format(heroku_app.redis_url), chevrons=False) except (ValueError, redis.exceptions.ConnectionError): time.sleep(2) log(".", chevrons=False, nl=False) log("Saving the URL of the postgres database...") config.extend({"database_url": heroku_app.db_url}) config.write() git.add("config.txt") git.commit("Save URL for database") log("Generating dashboard links...") heroku_addons = heroku_app.addon_parameters() heroku_addons = json.dumps(heroku_addons) if six.PY2: heroku_addons = heroku_addons.decode("utf-8") config.extend({"infrastructure_debug_details": heroku_addons}) config.write() git.add("config.txt") git.commit("Save URLs for heroku addon management") # Launch the Heroku app. log("Pushing code to Heroku...") git.push(remote="heroku", branch="HEAD:master") log("Scaling up the dynos...") default_size = config.get("dyno_type") for process in ["web", "worker"]: size = config.get("dyno_type_" + process, default_size) qty = config.get("num_dynos_" + process) heroku_app.scale_up_dyno(process, qty, size) if config.get("clock_on"): heroku_app.scale_up_dyno("clock", 1, size) if prelaunch_actions is not None: for task in prelaunch_actions: task(heroku_app, config) # Launch the experiment. log("Launching the experiment on the remote server and starting recruitment..." ) launch_url = "{}/launch".format(heroku_app.url) log("Calling {}".format(launch_url), chevrons=False) launch_data = _handle_launch_data(launch_url, error=log) result = { "app_name": heroku_app.name, "app_home": heroku_app.url, "dashboard_url": "{}/dashboard/".format(heroku_app.url), "recruitment_msg": launch_data.get("recruitment_msg", None), } log("Experiment details:") log("App home: {}".format(result["app_home"]), chevrons=False) log("Dashboard URL: {}".format(result["dashboard_url"]), chevrons=False) log("Dashboard user: {}".format(config.get("dashboard_user")), chevrons=False) log( "Dashboard password: {}".format(config.get("dashboard_password")), chevrons=False, ) log("Recruiter info:") log(result["recruitment_msg"], chevrons=False) # Return to the branch whence we came. os.chdir(cwd) log("Completed Heroku deployment of experiment ID {} using app ID {}.". format(config.get("id"), heroku_app_id)) return result