def temp_repo(self, in_tempdir, stub_config):
from dallinger.utils import GitClient
stub_config.write()
config = {'user.name': 'Test User', 'user.email': '*****@*****.**'}
git = GitClient()
git.init(config=config)
git.add("--all")
git.commit("Test Repo")
def test_tests(self):
from dallinger.utils import GitClient
from dallinger.utils import run_command
original_dir = os.getcwd()
git = GitClient()
gudir = git.clone('https://github.com/Dallinger/Griduniverse.git')
os.chdir(gudir)
cmd = [sys.executable, 'setup.py', 'develop']
st = run_command(cmd, sys.stdout)
cmd = [sys.executable, '-m', 'pytest']
st = run_command(cmd, sys.stdout, ignore_errors=True)
os.chdir(original_dir)
shutil.rmtree(gudir, ignore_errors=True)
print("Grid Universe tests finished running.")
if st != 0:
print("Some tests failed.")
assert st == 0
class ExperimentFileSource(object):
"""Treat an experiment directory as a potential source of files for
copying to a temp directory as part of a deployment (debug or otherwise).
"""
def __init__(self, root_dir="."):
self.root = root_dir
self.git = GitClient()
@property
def files(self):
"""A Set of all files copyable in the source directory, accounting for
exclusions.
"""
return {path for path in self._walk()}
@property
def size(self):
"""Combined size of all files, accounting for exclusions.
"""
return sum([os.path.getsize(path) for path in self._walk()])
def selective_copy_to(self, destination):
"""Write files from the source directory to another directory, skipping
files excluded by the general exclusion_policy, plus any files
ignored by git configuration.
"""
for path in self.files:
subpath = os.path.relpath(path, start=self.root)
target_folder = os.path.join(destination, os.path.dirname(subpath))
ensure_directory(target_folder)
shutil.copy2(path, target_folder)
def _walk(self):
exclusions = exclusion_policy()
git_files = set([os.path.join(self.root, f) for f in self.git.files()])
for dirpath, dirnames, filenames in os.walk(self.root, topdown=True):
current_exclusions = exclusions(dirpath, os.listdir(dirpath))
# Modifying dirnames in-place will prune the subsequent files and
# directories visited by os.walk. This is only possible when
# topdown = True
dirnames[:] = [d for d in dirnames if d not in current_exclusions]
legit_files = {
os.path.join(dirpath, f)
for f in filenames
if f not in current_exclusions and os.path.join(dirpath, f)
}
if git_files:
legit_files = legit_files.intersection(git_files)
for legit in legit_files:
yield legit
def git(self):
from dallinger.utils import GitClient
git = GitClient()
return git
def __init__(self, root_dir="."):
self.root = root_dir
self.git = GitClient()
def deploy_sandbox_shared_setup(log, verbose=True, app=None, exp_config=None):
"""Set up Git, push to Heroku, and launch the app."""
if verbose:
out = None
else:
out = open(os.devnull, "w")
config = get_config()
if not config.ready:
config.load()
heroku.sanity_check(config)
(id, tmp) = setup_experiment(log,
debug=False,
app=app,
exp_config=exp_config)
# Register the experiment using all configured registration services.
if config.get("mode") == "live":
log("Registering the experiment on configured services...")
registration.register(id, snapshot=None)
# Log in to Heroku if we aren't already.
log("Making sure that you are logged in to Heroku.")
heroku.log_in()
config.set("heroku_auth_token", heroku.auth_token())
log("", chevrons=False)
# Change to temporary directory.
cwd = os.getcwd()
os.chdir(tmp)
# Commit Heroku-specific files to tmp folder's git repo.
git = GitClient(output=out)
git.init()
git.add("--all")
git.commit('"Experiment {}"'.format(id))
# Initialize the app on Heroku.
log("Initializing app on Heroku...")
team = config.get("heroku_team", None)
heroku_app = HerokuApp(dallinger_uid=id, output=out, team=team)
heroku_app.bootstrap()
heroku_app.buildpack(
"https://github.com/stomita/heroku-buildpack-phantomjs")
# Set up add-ons and AWS environment variables.
database_size = config.get("database_size")
redis_size = config.get("redis_size")
addons = [
"heroku-postgresql:{}".format(quote(database_size)),
"heroku-redis:{}".format(quote(redis_size)),
"papertrail",
]
if config.get("sentry"):
addons.append("sentry")
for name in addons:
heroku_app.addon(name)
heroku_config = {
"aws_access_key_id": config["aws_access_key_id"],
"aws_secret_access_key": config["aws_secret_access_key"],
"aws_region": config["aws_region"],
"auto_recruit": config["auto_recruit"],
"smtp_username": config["smtp_username"],
"smtp_password": config["smtp_password"],
"whimsical": config["whimsical"],
}
heroku_app.set_multiple(**heroku_config)
# Wait for Redis database to be ready.
log("Waiting for Redis...")
ready = False
while not ready:
try:
r = redis.from_url(heroku_app.redis_url)
r.set("foo", "bar")
ready = True
except (ValueError, redis.exceptions.ConnectionError):
time.sleep(2)
log("Saving the URL of the postgres database...")
config.extend({"database_url": heroku_app.db_url})
config.write()
git.add("config.txt")
time.sleep(0.25)
git.commit("Save URL for database")
time.sleep(0.25)
# Launch the Heroku app.
log("Pushing code to Heroku...")
git.push(remote="heroku", branch="HEAD:master")
log("Scaling up the dynos...")
size = config.get("dyno_type")
for process in ["web", "worker"]:
qty = config.get("num_dynos_" + process)
heroku_app.scale_up_dyno(process, qty, size)
if config.get("clock_on"):
heroku_app.scale_up_dyno("clock", 1, size)
time.sleep(8)
# Launch the experiment.
log("Launching the experiment on the remote server and starting recruitment..."
)
launch_data = _handle_launch_data("{}/launch".format(heroku_app.url),
error=log)
result = {
"app_name": heroku_app.name,
"app_home": heroku_app.url,
"recruitment_msg": launch_data.get("recruitment_msg", None),
}
log("Experiment details:")
log("App home: {}".format(result["app_home"]), chevrons=False)
log("Recruiter info:")
log(result["recruitment_msg"], chevrons=False)
# Return to the branch whence we came.
os.chdir(cwd)
log("Completed deployment of experiment " + id + ".")
return result
def deploy_sandbox_shared_setup(verbose=True, app=None, exp_config=None):
"""Set up Git, push to Heroku, and launch the app."""
if verbose:
out = None
else:
out = open(os.devnull, 'w')
(id, tmp) = setup_experiment(debug=False,
verbose=verbose,
app=app,
exp_config=exp_config)
config = get_config() # We know it's ready; setup_experiment() does this.
# Register the experiment using all configured registration services.
if config.get("mode") == u"live":
log("Registering the experiment on configured services...")
registration.register(id, snapshot=None)
# Log in to Heroku if we aren't already.
log("Making sure that you are logged in to Heroku.")
heroku.log_in()
config.set("heroku_auth_token", heroku.auth_token())
click.echo("")
# Change to temporary directory.
cwd = os.getcwd()
os.chdir(tmp)
# Commit Heroku-specific files to tmp folder's git repo.
git = GitClient(output=out)
git.init()
git.add("--all")
git.commit('"Experiment {}"'.format(id))
# Initialize the app on Heroku.
log("Initializing app on Heroku...")
team = config.get("heroku_team", '').strip() or None
heroku_app = HerokuApp(dallinger_uid=id, output=out, team=team)
heroku_app.bootstrap()
heroku_app.buildpack(
"https://github.com/stomita/heroku-buildpack-phantomjs")
# Set up add-ons and AWS environment variables.
database_size = config.get('database_size')
addons = [
"heroku-postgresql:{}".format(quote(database_size)),
"heroku-redis:premium-0", "papertrail"
]
if config.get("sentry", False):
addons.append("sentry")
for name in addons:
heroku_app.addon(name)
heroku_config = {
"aws_access_key_id": config["aws_access_key_id"],
"aws_secret_access_key": config["aws_secret_access_key"],
"aws_region": config["aws_region"],
"auto_recruit": config["auto_recruit"],
"dallinger_email_username": config["dallinger_email_address"],
"dallinger_email_key": config["dallinger_email_password"],
"whimsical": config["whimsical"],
}
for k, v in sorted(heroku_config.items()): # sorted for testablility
heroku_app.set(k, v)
# Wait for Redis database to be ready.
log("Waiting for Redis...")
ready = False
while not ready:
r = redis.from_url(heroku_app.redis_url)
try:
r.set("foo", "bar")
ready = True
except redis.exceptions.ConnectionError:
time.sleep(2)
log("Saving the URL of the postgres database...")
# Set the notification URL and database URL in the config file.
config.extend({
"notification_url": heroku_app.url + u"/notifications",
"database_url": heroku_app.db_url,
})
config.write()
git.add("config.txt")
time.sleep(0.25)
git.commit("Save URLs for database and notifications")
time.sleep(0.25)
# Launch the Heroku app.
log("Pushing code to Heroku...")
git.push(remote="heroku", branch="HEAD:master")
log("Scaling up the dynos...")
size = config.get("dyno_type")
for process in ["web", "worker"]:
qty = config.get("num_dynos_" + process)
heroku_app.scale_up_dyno(process, qty, size)
if config.get("clock_on"):
heroku_app.scale_up_dyno("clock", 1, size)
time.sleep(8)
# Launch the experiment.
log("Launching the experiment on the remote server and starting recruitment..."
)
launch_data = _handle_launch_data('{}/launch'.format(heroku_app.url))
result = {
'app_name': heroku_app.name,
'app_home': heroku_app.url,
'recruitment_msg': launch_data.get('recruitment_msg', None),
}
log("Experiment details:")
log("App home: {}".format(result['app_home']), chevrons=False)
log("Recruiter info:")
log(result['recruitment_msg'], chevrons=False)
# Return to the branch whence we came.
os.chdir(cwd)
log("Completed deployment of experiment " + id + ".")
return result
class ExperimentFileSource(object):
"""Treat an experiment directory as a potential source of files for
copying to a temp directory as part of a deployment (debug or otherwise).
"""
def __init__(self, root_dir="."):
self.root = root_dir
self.git = GitClient()
@property
def files(self):
"""A Set of all files copyable in the source directory, accounting for
exclusions.
"""
return {path for path in self._walk()}
@property
def size(self):
"""Combined size of all files, accounting for exclusions.
"""
return sum([os.path.getsize(path) for path in self._walk()])
def selective_copy_to(self, destination):
"""Write files from the source directory to another directory, skipping
files excluded by the general exclusion_policy, plus any files
ignored by git configuration.
"""
for path in self.files:
subpath = os.path.relpath(path, start=self.root)
target_folder = os.path.join(destination, os.path.dirname(subpath))
ensure_directory(target_folder)
shutil.copy2(path, target_folder)
def _walk(self):
# The GitClient and os.walk may return different representations of the
# same unicode characters, so we use unicodedata.normalize() for
# comparisons:
# list(name_from_git)
# ['å', ' ', 'f', 'i', 'l', 'e', '.', 't', 'x', 't']
# list(from_os_walk)
# ['a', '̊', ' ', 'f', 'i', 'l', 'e', '.', 't', 'x', 't']
exclusions = exclusion_policy()
git_files = {
os.path.join(self.root, normalize("NFC", f)) for f in self.git.files()
}
for dirpath, dirnames, filenames in os.walk(self.root, topdown=True):
current_exclusions = exclusions(dirpath, os.listdir(dirpath))
# Modifying dirnames in-place will prune the subsequent files and
# directories visited by os.walk. This is only possible when
# topdown = True
dirnames[:] = [d for d in dirnames if d not in current_exclusions]
legit_files = {
os.path.join(dirpath, f)
for f in filenames
if f not in current_exclusions
}
if git_files:
normalized = {
normalize("NFC", six.text_type(f)): f for f in legit_files
}
legit_files = {v for k, v in normalized.items() if k in git_files}
for legit in legit_files:
yield legit
def git(self):
from dallinger.utils import GitClient
return GitClient()
def deploy_sandbox_shared_setup(log,
verbose=True,
app=None,
exp_config=None,
prelaunch_actions=None):
"""Set up Git, push to Heroku, and launch the app."""
if verbose:
out = None
else:
out = open(os.devnull, "w")
config = get_config()
if not config.ready:
config.load()
heroku.sanity_check(config)
(heroku_app_id, tmp) = setup_experiment(log,
debug=False,
app=app,
exp_config=exp_config)
# Register the experiment using all configured registration services.
if config.get("mode") == "live":
log("Registering the experiment on configured services...")
registration.register(heroku_app_id, snapshot=None)
# Log in to Heroku if we aren't already.
log("Making sure that you are logged in to Heroku.")
heroku.log_in()
config.set("heroku_auth_token", heroku.auth_token())
log("", chevrons=False)
# Change to temporary directory.
cwd = os.getcwd()
os.chdir(tmp)
# Commit Heroku-specific files to tmp folder's git repo.
git = GitClient(output=out)
git.init()
git.add("--all")
git.commit('"Experiment {}"'.format(heroku_app_id))
# Initialize the app on Heroku.
log("Initializing app on Heroku...")
team = config.get("heroku_team", None)
heroku_app = HerokuApp(dallinger_uid=heroku_app_id, output=out, team=team)
heroku_app.bootstrap()
heroku_app.buildpack(
"https://github.com/stomita/heroku-buildpack-phantomjs")
heroku_app.set("PYTHON_NO_SQLITE3", "true")
# Set up add-ons and AWS environment variables.
database_size = config.get("database_size")
redis_size = config.get("redis_size")
addons = [
"heroku-postgresql:{}".format(quote(database_size)),
"heroku-redis:{}".format(quote(redis_size)),
"papertrail",
]
if config.get("sentry"):
addons.append("sentry")
for name in addons:
heroku_app.addon(name)
heroku_config = {
"aws_access_key_id": config["aws_access_key_id"],
"aws_secret_access_key": config["aws_secret_access_key"],
"aws_region": config["aws_region"],
"auto_recruit": config["auto_recruit"],
"smtp_username": config["smtp_username"],
"smtp_password": config["smtp_password"],
"whimsical": config["whimsical"],
"FLASK_SECRET_KEY": codecs.encode(os.urandom(16),
"hex").decode("ascii"),
}
# Set up the preferred class as an environment variable, if one is set
# This is needed before the config is parsed, but we also store it in the
# config to make things easier for recording into bundles.
preferred_class = config.get("EXPERIMENT_CLASS_NAME", None)
if preferred_class:
heroku_config["EXPERIMENT_CLASS_NAME"] = preferred_class
heroku_app.set_multiple(**heroku_config)
# Wait for Redis database to be ready.
log("Waiting for Redis...", nl=False)
ready = False
while not ready:
try:
r = connect_to_redis(url=heroku_app.redis_url)
r.set("foo", "bar")
ready = True
log("\n✓ connected at {}".format(heroku_app.redis_url),
chevrons=False)
except (ValueError, redis.exceptions.ConnectionError):
time.sleep(2)
log(".", chevrons=False, nl=False)
log("Saving the URL of the postgres database...")
config.extend({"database_url": heroku_app.db_url})
config.write()
git.add("config.txt")
git.commit("Save URL for database")
log("Generating dashboard links...")
heroku_addons = heroku_app.addon_parameters()
heroku_addons = json.dumps(heroku_addons)
if six.PY2:
heroku_addons = heroku_addons.decode("utf-8")
config.extend({"infrastructure_debug_details": heroku_addons})
config.write()
git.add("config.txt")
git.commit("Save URLs for heroku addon management")
# Launch the Heroku app.
log("Pushing code to Heroku...")
git.push(remote="heroku", branch="HEAD:master")
log("Scaling up the dynos...")
default_size = config.get("dyno_type")
for process in ["web", "worker"]:
size = config.get("dyno_type_" + process, default_size)
qty = config.get("num_dynos_" + process)
heroku_app.scale_up_dyno(process, qty, size)
if config.get("clock_on"):
heroku_app.scale_up_dyno("clock", 1, size)
if prelaunch_actions is not None:
for task in prelaunch_actions:
task(heroku_app, config)
# Launch the experiment.
log("Launching the experiment on the remote server and starting recruitment..."
)
launch_url = "{}/launch".format(heroku_app.url)
log("Calling {}".format(launch_url), chevrons=False)
launch_data = _handle_launch_data(launch_url, error=log)
result = {
"app_name": heroku_app.name,
"app_home": heroku_app.url,
"dashboard_url": "{}/dashboard/".format(heroku_app.url),
"recruitment_msg": launch_data.get("recruitment_msg", None),
}
log("Experiment details:")
log("App home: {}".format(result["app_home"]), chevrons=False)
log("Dashboard URL: {}".format(result["dashboard_url"]), chevrons=False)
log("Dashboard user: {}".format(config.get("dashboard_user")),
chevrons=False)
log(
"Dashboard password: {}".format(config.get("dashboard_password")),
chevrons=False,
)
log("Recruiter info:")
log(result["recruitment_msg"], chevrons=False)
# Return to the branch whence we came.
os.chdir(cwd)
log("Completed Heroku deployment of experiment ID {} using app ID {}.".
format(config.get("id"), heroku_app_id))
return result
