def command(self, task_input, task_output, extra_task_args=[]):
def quote_and_join(arguments):
return " ".join([quote(arg) for arg in arguments])
pieces = []
run_pip = quote(os.path.join(self.path_prefix, 'pip'))
run_task = quote(
os.path.join(self.path_prefix, '_kubeface-run-task'))
kubeface_install_command = self.kubeface_install_command.format(
pip=run_pip)
if self.kubeface_install_policy == 'if-not-present':
# From: http://stackoverflow.com/questions/592620/check-if-a-program-exists-from-a-bash-script
pieces.append("command -v %s || { %s ; } " % (
run_task,
kubeface_install_command))
elif self.kubeface_install_policy == 'always':
pieces.append(kubeface_install_command)
if self.pip_packages:
pieces.append("%s install %s" % (
run_pip,
quote_and_join(self.pip_packages)))
pieces.append(
run_task +
" " +
quote_and_join([
task_input,
task_output,
"--verbose",
] + extra_task_args))
result = " && ".join(pieces)
return result
def _request(reddit_session, page_url, params=None, url_data=None, timeout=45):
"""
Actually make the request.
"""
page_url = quote(page_url.encode("utf-8"), ":/")
if url_data:
page_url += "?" + urlencode(url_data)
encoded_params = None
if params:
if params is True:
params = {}
params.setdefault("api_type", "json")
if reddit_session.modhash:
params.setdefault("uh", reddit_session.modhash)
params = dict([k, v.encode("utf-8")] for k, v in six.iteritems(params))
encoded_params = urlencode(params).encode("utf-8")
request = Request(page_url, data=encoded_params, headers=reddit_session.DEFAULT_HEADERS)
if reddit_session.config.log_requests >= 1:
sys.stderr.write("retrieving: %s\n" % page_url)
if reddit_session.config.log_requests >= 2:
sys.stderr.write("data: %s\n" % (encoded_params or "None"))
# pylint: disable-msg=W0212
response = reddit_session._opener.open(request, timeout=timeout)
return response.read()
def show_argv(*argv):
r"""
Join -- and possibly escape & quote -- the elements of ``argv`` into a
pure-ASCII form suitable for passing directly to a \*nix shell.
Nonprintable characters are escaped in a format that is Bash-specific but
should still be understandable to users of other shells.
The elements of ``argv`` are assumed to have been taken directly from
`sys.argv` (possibly with a detour through `argparse`); specifically, in
Python 2, they are assumed to be byte strings (encoding irrelevant), and in
Python 3, they are assumed to be text strings decoded with `os.fsdecode`.
"""
# Just using `repr` for this won't work, as the quotes it adds around
# simple (e.g., alphanumeric) arguments are unnecessary, and the double
# quotes it puts around strings like `"'$HOME'"` are just plain wrong.
shown = ''
assigning = True
for a in argv:
if isinstance(a, text_type):
a = os.fsencode(a)
a = a.decode('iso-8859-1')
if re.search(r'[^\x20-\x7E]', a):
a = "$'" + a.translate(bash_slash) + "'"
assigning = False
else:
a = quote(a)
if assigning and re.match(r'^[A-Za-z_]\w*=', a):
a = "'" + a + "'"
else:
assigning = False
if shown:
shown += ' '
shown += a
return shown
def inline(env):
results = []
for key, value in env.items():
if value is None:
value = ''
if not getattr(value, 'safe', False):
value = quote(value)
results.append('%s=%s' % (key, value))
return ', '.join(results)
def extract_tarball(session, tarball, target_dir):
"""
Extract tarball to given dir from a session.
:param session: session to run the command on
:type session: ShellSession
:param str tarball: full path of tar file that should be extracted
:param str target_dir: name of directory where tarball should be extracted
:raises: :py:class:`RuntimeError` if tar command returned non-null
"""
cmd = 'tar -C {} --strip-components=1 -xapf {}' \
.format(quote(target_dir), quote(tarball))
status, output = session.cmd_status_output(cmd)
if output:
output = output.strip()
if status != 0:
raise RuntimeError('Failed to extract {} to {}: {}'.format(
tarball, target_dir, output))
def set(self, key, value):
"""Configure an app key/value pair"""
cmd = [
"heroku", "config:set", "{}={}".format(key, quote(str(value))),
"--app", self.name
]
if self._is_sensitive_key(key):
self._run_quiet(cmd)
else:
self._run(cmd)
def copy(session, source, target):
"""
Copy file or directory from source to target.
:param session: session to run the command on
:type session: ShellSession
:param str source: full path of existing file or directory
:param str target: full path of new file or directory
:raises: :py:class:`RuntimeError` if cp returns non-zero exit status
Calls `cp source target` through a session. See `man cp` for what source
and target can be and what behavior to expect.
"""
cmd = 'cp {} {}'.format(quote(source), quote(target))
status, output = session.cmd_status_output(cmd)
if status != 0:
raise RuntimeError('Failed to cp {} to {}: {}'.format(
source, target,
output.strip() if output else ''))
def __str__(self):
flags = []
if not self.check:
flags.append('nocheck')
if not self.optimize:
flags.append('noopt')
if not self.strip:
flags.append('nostrip')
if not self.docs:
flags.append('nodocs')
if self.parallel > 1:
flags.append('parallel=%i' % self.parallel)
return quote(' '.join(flags))
def _request(reddit_session, page_url, params=None, url_data=None, timeout=45):
page_url = quote(page_url.encode('utf-8'), ':/')
if url_data:
page_url += '?' + urlencode(url_data)
encoded_params = None
if params:
params = dict([k, v.encode('utf-8')] for k, v in six.iteritems(params))
encoded_params = urlencode(params).encode('utf-8')
request = Request(page_url, data=encoded_params,
headers=reddit_session.DEFAULT_HEADERS)
# pylint: disable-msg=W0212
response = reddit_session._opener.open(request, timeout=timeout)
return response.read()
def get_pretty_cmd(self):
if not self.call:
return None
# We want to have pretty callable parameters
arguments = list()
for p in self.call.params:
if callable(p):
try:
arguments.append(text_type(p()))
except IOError:
arguments.append(repr(p))
else:
arguments.append(quote(text_type(p)))
return "%s %s" % (self.call.cmd, " ".join(arguments))
def set_multiple(self, **kwargs):
"""Configure multiple app key/value pairs"""
quiet = False
if not kwargs:
return
cmd = ["heroku", "config:set"]
for k in sorted(kwargs):
cmd.append("{}={}".format(k, quote(str(kwargs[k]))))
if self._is_sensitive_key(k):
quiet = True
cmd.extend(["--app", self.name])
if quiet:
self._run_quiet(cmd)
else:
self._run(cmd)
def get_pretty_cmd(self):
if not self.call:
return None
# We want to have pretty callable parameters
arguments = list()
for p in self.call.params:
if callable(p):
try:
arguments.append(inspect.getsource(p))
except IOError:
arguments.append(repr(p))
else:
arguments.append(quote(text_type(p)))
return "%s %s" % (self.call.cmd, " ".join(arguments))
def rmtree(session, path):
"""
Remove a directory including its contents.
:param session: session to run the command on
:type session: ShellSession
:param str path: directory to remove
:raises: :py:class:`RuntimeError` if deletion fails
Calls `rm -rf`.
"""
cmd = 'rm -rf ' + quote(path)
status, output = session.cmd_status_output(cmd)
if status != 0:
raise RuntimeError('Failed to remove {}: {}'.format(
path,
output.strip() if output else ''))
def _request(reddit_session, page_url, params=None, url_data=None, timeout=45):
page_url = quote(page_url.encode('utf-8'), ':/')
if url_data:
page_url += '?' + urlencode(url_data)
encoded_params = None
if params:
if params is True:
params = {}
params.setdefault('api_type', 'json')
if reddit_session.modhash:
params.setdefault('uh', reddit_session.modhash)
params = dict([k, v.encode('utf-8')] for k, v in six.iteritems(params))
encoded_params = urlencode(params).encode('utf-8')
request = Request(page_url, data=encoded_params,
headers=reddit_session.DEFAULT_HEADERS)
# pylint: disable-msg=W0212
response = reddit_session._opener.open(request, timeout=timeout)
return response.read()
def cat(session, filename):
"""
Get contents of a text file from a session.
:param session: session to run the command on
:type session: ShellSession
:param str filename: full path of file
:returns: file contents
:rtype: str
:raises: :py:class:`RuntimeError` if cat command fails
Should only be used for very short files without tabs or other fancy
contents. Otherwise better download file or use some other method.
"""
cmd = 'cat ' + quote(filename)
status, output = session.cmd_status_output(cmd)
if output:
output = output.strip()
if status != 0:
raise RuntimeError('Failed to cat {}: {}'.format(filename, output))
return output
def hash_file(session, filename, method='md5'):
"""
Calculate hash of given file from a session.
:param session: session to run the command on
:type session: ShellSession
:param str filename: full path of file that should be hashed
:param str method: method for hashing
:returns: hash of file, a hex number
:rtype: str
:raises: :py:class:`RuntimeError` if hash command failed (e.g. file not
found) or resulting output does not have expected format
:raises: :py:class:`ValueError` if given method is not (yet) supported
"""
if method == 'md5':
cmd = 'md5sum'
expect_len = 32
else:
raise ValueError('only method "md5" supported yet')
# run cmd on shell
status, output = session.cmd_status_output(cmd + ' ' + quote(filename))
if output:
output = output.strip()
if status != 0:
raise RuntimeError('Could not hash {} using {}: {}'.format(
filename, cmd, output))
# parse output
hash_str = output.split(maxsplit=1)[0].lower()
# check length and all chars are hex
if expect_len and len(hash_str) != expect_len:
raise RuntimeError(
'Resulting hash string has unexpected length {}: {}'.format(
len(hash_str), hash_str))
if hash_str.strip('0123456789abcdef'):
raise RuntimeError(
'Resulting hash string has unexpected characters: ' + hash_str)
return hash_str
def ls(session, dir_name): # pylint: disable=C0103
"""
Run `ls` in given directory through a session.
:param session: session to run the command on
:type session: ShellSession
:param str dir_name: name of directory
:returns: names of files in dir (without path components)
:rtype: [str]
Just like :py:func:`os.listdir`, does not include file names starting with
dot (`'.'`)
"""
cmd = 'ls -1UNq {}'.format(quote(dir_name))
status, output = session.cmd_status_output(cmd)
if output:
output = output.strip()
if status == 2: # probably just nothing found
return []
if status != 0:
raise RuntimeError('Failed to ls {}: {}'.format(dir_name, output))
return output.splitlines()
def _get_run_command(entrypoint_command):
formatted_command = []
for cmd in entrypoint_command:
formatted_command.extend([quote(s) for s in split(cmd)])
return formatted_command
def deploy_sandbox_shared_setup(log, verbose=True, app=None, exp_config=None):
"""Set up Git, push to Heroku, and launch the app."""
if verbose:
out = None
else:
out = open(os.devnull, "w")
config = get_config()
if not config.ready:
config.load()
heroku.sanity_check(config)
(id, tmp) = setup_experiment(log,
debug=False,
app=app,
exp_config=exp_config)
# Register the experiment using all configured registration services.
if config.get("mode") == "live":
log("Registering the experiment on configured services...")
registration.register(id, snapshot=None)
# Log in to Heroku if we aren't already.
log("Making sure that you are logged in to Heroku.")
heroku.log_in()
config.set("heroku_auth_token", heroku.auth_token())
log("", chevrons=False)
# Change to temporary directory.
cwd = os.getcwd()
os.chdir(tmp)
# Commit Heroku-specific files to tmp folder's git repo.
git = GitClient(output=out)
git.init()
git.add("--all")
git.commit('"Experiment {}"'.format(id))
# Initialize the app on Heroku.
log("Initializing app on Heroku...")
team = config.get("heroku_team", None)
heroku_app = HerokuApp(dallinger_uid=id, output=out, team=team)
heroku_app.bootstrap()
heroku_app.buildpack(
"https://github.com/stomita/heroku-buildpack-phantomjs")
# Set up add-ons and AWS environment variables.
database_size = config.get("database_size")
redis_size = config.get("redis_size")
addons = [
"heroku-postgresql:{}".format(quote(database_size)),
"heroku-redis:{}".format(quote(redis_size)),
"papertrail",
]
if config.get("sentry"):
addons.append("sentry")
for name in addons:
heroku_app.addon(name)
heroku_config = {
"aws_access_key_id": config["aws_access_key_id"],
"aws_secret_access_key": config["aws_secret_access_key"],
"aws_region": config["aws_region"],
"auto_recruit": config["auto_recruit"],
"smtp_username": config["smtp_username"],
"smtp_password": config["smtp_password"],
"whimsical": config["whimsical"],
}
heroku_app.set_multiple(**heroku_config)
# Wait for Redis database to be ready.
log("Waiting for Redis...")
ready = False
while not ready:
try:
r = redis.from_url(heroku_app.redis_url)
r.set("foo", "bar")
ready = True
except (ValueError, redis.exceptions.ConnectionError):
time.sleep(2)
log("Saving the URL of the postgres database...")
config.extend({"database_url": heroku_app.db_url})
config.write()
git.add("config.txt")
time.sleep(0.25)
git.commit("Save URL for database")
time.sleep(0.25)
# Launch the Heroku app.
log("Pushing code to Heroku...")
git.push(remote="heroku", branch="HEAD:master")
log("Scaling up the dynos...")
size = config.get("dyno_type")
for process in ["web", "worker"]:
qty = config.get("num_dynos_" + process)
heroku_app.scale_up_dyno(process, qty, size)
if config.get("clock_on"):
heroku_app.scale_up_dyno("clock", 1, size)
time.sleep(8)
# Launch the experiment.
log("Launching the experiment on the remote server and starting recruitment..."
)
launch_data = _handle_launch_data("{}/launch".format(heroku_app.url),
error=log)
result = {
"app_name": heroku_app.name,
"app_home": heroku_app.url,
"recruitment_msg": launch_data.get("recruitment_msg", None),
}
log("Experiment details:")
log("App home: {}".format(result["app_home"]), chevrons=False)
log("Recruiter info:")
log(result["recruitment_msg"], chevrons=False)
# Return to the branch whence we came.
os.chdir(cwd)
log("Completed deployment of experiment " + id + ".")
return result
def quote_and_join(arguments):
return " ".join([quote(arg) for arg in arguments])
def main(args, logfp):
"""
Create genomes and reads for a multiple infection detection experiment.
@param args: A namespace instance, as returned by parse_args
@param logfp: A file object to write log information to.
"""
print('Invocation arguments', args, file=logfp)
qOutputDir = quote(args.outputDir)
genome1 = join(qOutputDir, 'genome-1.fasta')
genome2 = join(qOutputDir, 'genome-2.fasta')
genome2locations = join(qOutputDir, 'genome-2.locations')
reads1 = join(qOutputDir, 'reads-1.fastq')
reads2 = join(qOutputDir, 'reads-2.fastq')
reads12 = join(qOutputDir, 'reads-12.fastq')
executor = Executor(args.dryRun)
if args.genome1Filename:
executor.execute('ln -s %s %s' %
(quote(args.genome1Filename), genome1))
else:
if args.genomeLength < 1:
print('Random initial genome length must be > 0.', file=sys.stderr)
sys.exit(3)
print('Writing random starting genome of length %d to %s' %
(args.genomeLength, genome1),
file=logfp)
if not args.dryRun:
sequence = ''.join(
[choice('ACGT') for _ in range(args.genomeLength)])
with open(genome1, 'w') as fp:
print('>genome-1\n%s' % sequence, file=fp)
if args.genome2Filename:
executor.execute('ln -s %s %s' %
(quote(args.genome2Filename), genome2))
else:
# Make a second genome using the given mutation rate. Print its
# mutated locations to a file.
(genome1read, ) = list(FastaReads(genome1))
offsets = mutateRead(genome1read, args.genome2MutationRate)
with open(genome2locations, 'w') as fp:
print('\n'.join(str(offset + 1) for offset in sorted(offsets)),
file=fp)
genome1read.id = 'genome-2'
Reads([genome1read]).save(genome2)
cmdPrefix = ('create-reads.py --maxReadLength %d --minReadLength %d '
'--meanLength %d --sdLength %d --rate %f ' %
(args.maxReadLength, args.minReadLength, args.meanReadLength,
args.sdReadLength, args.readMutationRate))
for info in [{
'reads': reads1,
'fasta': genome1,
'number': 1,
'count': args.genome1ReadCount or args.readCount,
}, {
'reads': reads2,
'fasta': genome2,
'number': 2,
'count': args.genome2ReadCount or args.readCount,
}]:
executor.execute(cmdPrefix +
('--idPrefix genome-%(number)d-read- '
'--count %(count)d < %(fasta)s > %(reads)s' % info))
executor.execute('cat %s %s > %s' % (reads1, reads2, reads12))
print('\n'.join(executor.log), file=logfp)
def _modify_instruction_label_env(self, instruction, instr_key,
instr_value):
"""
set <INSTRUCTION> instr_key to instr_value
:param instr_key: str, label key
:param instr_value: str or None, new label/env value or None to remove
"""
if instruction == 'LABEL':
instructions = self.labels
elif instruction == 'ENV':
instructions = self.envs
elif instruction == 'ARG':
instructions = self.args
else:
raise ValueError("Unknown instruction '%s'" % instruction)
if instr_key not in instructions:
raise KeyError('%s not in %ss' % (instr_key, instruction))
# extract target instructions from the final stage only
candidates = []
for insn in self.structure:
if insn['instruction'] == 'FROM':
candidates = []
if insn['instruction'] == instruction:
candidates.append(insn)
# Find where in the file to put the changes
content = startline = endline = None
for candidate in candidates:
words = list(WordSplitter(candidate['value']).split(dequote=False))
# LABEL/ENV/ARG syntax is one of two types:
if '=' not in words[0]: # LABEL/ENV/ARG name value
# Remove quotes from key name and see if it's the one
# we're looking for.
if WordSplitter(words[0]).dequote() == instr_key:
if instr_value is None:
# Delete this line altogether
content = None
else:
# Adjust label/env value
words[1:] = [quote(instr_value)]
# Now reconstruct the line
content = " ".join([instruction] + words) + '\n'
startline = candidate['startline']
endline = candidate['endline']
break
else: # LABEL/ENV/ARG "name"="value"
for index, token in enumerate(words):
key, _ = token.split("=", 1)
if WordSplitter(key).dequote() == instr_key:
if instr_value is None:
# Delete this label
del words[index]
else:
# Adjust label/env value
words[index] = "{0}={1}".format(
key, quote(instr_value))
if len(words) == 0:
# We removed the last label/env, delete the whole line
content = None
else:
# Now reconstruct the line
content = " ".join([instruction] + words) + '\n'
startline = candidate['startline']
endline = candidate['endline']
break
# We know the label/env we're looking for is there
assert startline and endline
# Re-write the Dockerfile
lines = self.lines
del lines[startline:endline + 1]
if content:
lines.insert(startline, content)
self.lines = lines
def deploy_sandbox_shared_setup(log,
verbose=True,
app=None,
exp_config=None,
prelaunch_actions=None):
"""Set up Git, push to Heroku, and launch the app."""
if verbose:
out = None
else:
out = open(os.devnull, "w")
config = get_config()
if not config.ready:
config.load()
heroku.sanity_check(config)
(heroku_app_id, tmp) = setup_experiment(log,
debug=False,
app=app,
exp_config=exp_config)
# Register the experiment using all configured registration services.
if config.get("mode") == "live":
log("Registering the experiment on configured services...")
registration.register(heroku_app_id, snapshot=None)
# Log in to Heroku if we aren't already.
log("Making sure that you are logged in to Heroku.")
heroku.log_in()
config.set("heroku_auth_token", heroku.auth_token())
log("", chevrons=False)
# Change to temporary directory.
cwd = os.getcwd()
os.chdir(tmp)
# Commit Heroku-specific files to tmp folder's git repo.
git = GitClient(output=out)
git.init()
git.add("--all")
git.commit('"Experiment {}"'.format(heroku_app_id))
# Initialize the app on Heroku.
log("Initializing app on Heroku...")
team = config.get("heroku_team", None)
heroku_app = HerokuApp(dallinger_uid=heroku_app_id, output=out, team=team)
heroku_app.bootstrap()
heroku_app.buildpack(
"https://github.com/stomita/heroku-buildpack-phantomjs")
heroku_app.set("PYTHON_NO_SQLITE3", "true")
# Set up add-ons and AWS environment variables.
database_size = config.get("database_size")
redis_size = config.get("redis_size")
addons = [
"heroku-postgresql:{}".format(quote(database_size)),
"heroku-redis:{}".format(quote(redis_size)),
"papertrail",
]
if config.get("sentry"):
addons.append("sentry")
for name in addons:
heroku_app.addon(name)
heroku_config = {
"aws_access_key_id": config["aws_access_key_id"],
"aws_secret_access_key": config["aws_secret_access_key"],
"aws_region": config["aws_region"],
"auto_recruit": config["auto_recruit"],
"smtp_username": config["smtp_username"],
"smtp_password": config["smtp_password"],
"whimsical": config["whimsical"],
"FLASK_SECRET_KEY": codecs.encode(os.urandom(16),
"hex").decode("ascii"),
}
# Set up the preferred class as an environment variable, if one is set
# This is needed before the config is parsed, but we also store it in the
# config to make things easier for recording into bundles.
preferred_class = config.get("EXPERIMENT_CLASS_NAME", None)
if preferred_class:
heroku_config["EXPERIMENT_CLASS_NAME"] = preferred_class
heroku_app.set_multiple(**heroku_config)
# Wait for Redis database to be ready.
log("Waiting for Redis...", nl=False)
ready = False
while not ready:
try:
r = connect_to_redis(url=heroku_app.redis_url)
r.set("foo", "bar")
ready = True
log("\n✓ connected at {}".format(heroku_app.redis_url),
chevrons=False)
except (ValueError, redis.exceptions.ConnectionError):
time.sleep(2)
log(".", chevrons=False, nl=False)
log("Saving the URL of the postgres database...")
config.extend({"database_url": heroku_app.db_url})
config.write()
git.add("config.txt")
git.commit("Save URL for database")
log("Generating dashboard links...")
heroku_addons = heroku_app.addon_parameters()
heroku_addons = json.dumps(heroku_addons)
if six.PY2:
heroku_addons = heroku_addons.decode("utf-8")
config.extend({"infrastructure_debug_details": heroku_addons})
config.write()
git.add("config.txt")
git.commit("Save URLs for heroku addon management")
# Launch the Heroku app.
log("Pushing code to Heroku...")
git.push(remote="heroku", branch="HEAD:master")
log("Scaling up the dynos...")
default_size = config.get("dyno_type")
for process in ["web", "worker"]:
size = config.get("dyno_type_" + process, default_size)
qty = config.get("num_dynos_" + process)
heroku_app.scale_up_dyno(process, qty, size)
if config.get("clock_on"):
heroku_app.scale_up_dyno("clock", 1, size)
if prelaunch_actions is not None:
for task in prelaunch_actions:
task(heroku_app, config)
# Launch the experiment.
log("Launching the experiment on the remote server and starting recruitment..."
)
launch_url = "{}/launch".format(heroku_app.url)
log("Calling {}".format(launch_url), chevrons=False)
launch_data = _handle_launch_data(launch_url, error=log)
result = {
"app_name": heroku_app.name,
"app_home": heroku_app.url,
"dashboard_url": "{}/dashboard/".format(heroku_app.url),
"recruitment_msg": launch_data.get("recruitment_msg", None),
}
log("Experiment details:")
log("App home: {}".format(result["app_home"]), chevrons=False)
log("Dashboard URL: {}".format(result["dashboard_url"]), chevrons=False)
log("Dashboard user: {}".format(config.get("dashboard_user")),
chevrons=False)
log(
"Dashboard password: {}".format(config.get("dashboard_password")),
chevrons=False,
)
log("Recruiter info:")
log(result["recruitment_msg"], chevrons=False)
# Return to the branch whence we came.
os.chdir(cwd)
log("Completed Heroku deployment of experiment ID {} using app ID {}.".
format(config.get("id"), heroku_app_id))
return result
def deploy_sandbox_shared_setup(verbose=True, app=None, exp_config=None):
"""Set up Git, push to Heroku, and launch the app."""
if verbose:
out = None
else:
out = open(os.devnull, 'w')
(id, tmp) = setup_experiment(debug=False,
verbose=verbose,
app=app,
exp_config=exp_config)
config = get_config() # We know it's ready; setup_experiment() does this.
# Register the experiment using all configured registration services.
if config.get("mode") == "live":
log("Registering the experiment on configured services...")
registration.register(id, snapshot=None)
# Log in to Heroku if we aren't already.
log("Making sure that you are logged in to Heroku.")
heroku.log_in()
config.set("heroku_auth_token", heroku.auth_token())
click.echo("")
# Change to temporary directory.
cwd = os.getcwd()
os.chdir(tmp)
# Commit Heroku-specific files to tmp folder's git repo.
git = GitClient(output=out)
git.init()
git.add("--all")
git.commit('"Experiment {}"'.format(id))
# Initialize the app on Heroku.
log("Initializing app on Heroku...")
team = config.get("heroku_team", '').strip() or None
heroku_app = HerokuApp(dallinger_uid=id, output=out, team=team)
heroku_app.bootstrap()
heroku_app.buildpack(
"https://github.com/stomita/heroku-buildpack-phantomjs")
# Set up add-ons and AWS environment variables.
database_size = config.get('database_size')
redis_size = config.get('redis_size', 'premium-0')
addons = [
"heroku-postgresql:{}".format(quote(database_size)),
"heroku-redis:{}".format(quote(redis_size)), "papertrail"
]
if config.get("sentry", False):
addons.append("sentry")
for name in addons:
heroku_app.addon(name)
heroku_config = {
"aws_access_key_id": config["aws_access_key_id"],
"aws_secret_access_key": config["aws_secret_access_key"],
"aws_region": config["aws_region"],
"auto_recruit": config["auto_recruit"],
"dallinger_email_username": config["dallinger_email_address"],
"dallinger_email_key": config["dallinger_email_password"],
"whimsical": config["whimsical"],
}
for k, v in sorted(heroku_config.items()): # sorted for testablility
heroku_app.set(k, v)
# Wait for Redis database to be ready.
log("Waiting for Redis...")
ready = False
while not ready:
r = redis.from_url(heroku_app.redis_url)
try:
r.set("foo", "bar")
ready = True
except redis.exceptions.ConnectionError:
time.sleep(2)
log("Saving the URL of the postgres database...")
# Set the notification URL and database URL in the config file.
config.extend({
"notification_url": heroku_app.url + "/notifications",
"database_url": heroku_app.db_url,
})
config.write()
git.add("config.txt")
time.sleep(0.25)
git.commit("Save URLs for database and notifications")
time.sleep(0.25)
# Launch the Heroku app.
log("Pushing code to Heroku...")
git.push(remote="heroku", branch="HEAD:master")
log("Scaling up the dynos...")
size = config.get("dyno_type")
for process in ["web", "worker"]:
qty = config.get("num_dynos_" + process)
heroku_app.scale_up_dyno(process, qty, size)
if config.get("clock_on"):
heroku_app.scale_up_dyno("clock", 1, size)
time.sleep(8)
# Launch the experiment.
log("Launching the experiment on the remote server and starting recruitment..."
)
launch_data = _handle_launch_data('{}/launch'.format(heroku_app.url))
result = {
'app_name': heroku_app.name,
'app_home': heroku_app.url,
'recruitment_msg': launch_data.get('recruitment_msg', None),
}
log("Experiment details:")
log("App home: {}".format(result['app_home']), chevrons=False)
log("Recruiter info:")
log(result['recruitment_msg'], chevrons=False)
# Return to the branch whence we came.
os.chdir(cwd)
log("Completed deployment of experiment " + id + ".")
return result
