def config(label):
"""
Reads deployment parameters for the given setup.
label: Label identifying the desired setup (e.g., prod, test, etc.)
"""
env.cfg_label = label
print "Deployment label is:", env.cfg_label
filename = ".codalabconfig"
print "Loading configuration from:", env.cfg_path
configuration = DeploymentConfig(label, env.cfg_path)
print "Configuring logger..."
logging.config.dictConfig(configuration.getLoggerDictConfig())
logger.info("Loaded configuration from file: %s", configuration.getFilename())
env.roledefs = {'web' : configuration.getWebHostnames()}
# Credentials
env.user = configuration.getVirtualMachineLogonUsername()
env.password = configuration.getVirtualMachineLogonPassword()
env.key_filename = configuration.getServiceCertificateKeyFilename()
# Repository
env.git_codalab_tag = configuration.getGitTag()
env.git_codalab_cli_tag = configuration.getBundleServiceGitTag()
env.deploy_codalab_dir = 'codalab'
env.deploy_codalab_cli_dir = 'codalab-cli'
env.django_settings_module = 'codalab.settings'
env.django_configuration = configuration.getDjangoConfiguration() # Prod or Dev
env.config_http_port = '80'
env.config_server_name = "{0}.cloudapp.net".format(configuration.getServiceName())
env.configuration = True
env.SHELL_ENV = {}
def config(label):
"""
Reads deployment parameters for the given setup.
label: Label identifying the desired setup (e.g., prod, test, etc.)
"""
env.cfg_label = label
print "Deployment label is:", env.cfg_label
print "Loading configuration from:", env.cfg_path
configuration = DeploymentConfig(label, env.cfg_path)
print "Configuring logger..."
logging.config.dictConfig(configuration.getLoggerDictConfig())
logger.info("Loaded configuration from file: %s",
configuration.getFilename())
env.roledefs = {'web': configuration.getWebHostnames()}
# Credentials
env.user = configuration.getVirtualMachineLogonUsername()
# Repository
env.git_codalab_tag = configuration.getGitTag()
env.deploy_codalab_dir = 'codalab-competitions' # Directory for codalab competitions
env.django_settings_module = 'codalab.settings'
env.django_configuration = configuration.getDjangoConfiguration(
) # Prod or Test
env.config_http_port = '80'
env.config_server_name = "{0}.cloudapp.net".format(
configuration.getServiceName())
print "Deployment configuration is for:", env.config_server_name
env.configuration = True
env.SHELL_ENV = {}
def config(label=None):
"""
Reads deployment parameters for the given setup.
label: Label identifying the desired setup.
"""
env.cfg_label = label
print "Deployment label is: ", env.cfg_label
if 'cfg_path' not in env:
env.cfg_path = os.path.join(os.environ['HOME'], '.codalabconfig')
print "Loading configuration from: ", env.cfg_path
configuration = DeploymentConfig(label, env.cfg_path)
print "Configuring logger..."
logging.config.dictConfig(configuration.getLoggerDictConfig())
logger.info("Loaded configuration from file: %s", configuration.getFilename())
env.user = configuration.getVirtualMachineLogonUsername()
env.password = configuration.getVirtualMachineLogonPassword()
env.key_filename = configuration.getServiceCertificateKeyFilename()
env.roledefs = { 'build' : [ configuration.getBuildHostname() ] }
if label is not None:
env.roledefs.update({ 'web' : configuration.getWebHostnames() })
env.git_user = configuration.getGitUser()
env.git_repo = configuration.getGitRepo()
env.git_tag = configuration.getGitTag()
env.git_repo_url = 'https://github.com/{0}/{1}.git'.format(env.git_user, env.git_repo)
env.deploy_dir = 'deploy'
env.build_archive = '{0}.tar.gz'.format(env.git_tag)
env.django_settings_module = 'codalab.settings'
env.django_configuration = configuration.getDjangoConfiguration()
env.config_http_port = '80'
env.config_server_name = "{0}.cloudapp.net".format(configuration.getServiceName())
env.configuration = True
def config(label=None):
"""
Reads deployment parameters for the given setup.
label: Label identifying the desired setup.
"""
env.cfg_label = label
print "Deployment label is: ", env.cfg_label
filename = ".codalabconfig"
if 'cfg_path' not in env:
env.cfg_path = os.path.join(os.getcwd(), filename)
if os.path.exists(env.cfg_path) == False:
env.cfg_path = os.path.join(os.path.expanduser("~"), filename)
print "Loading configuration from: ", env.cfg_path
configuration = DeploymentConfig(label, env.cfg_path)
print "Configuring logger..."
logging.config.dictConfig(configuration.getLoggerDictConfig())
logger.info("Loaded configuration from file: %s",
configuration.getFilename())
env.user = configuration.getVirtualMachineLogonUsername()
env.password = configuration.getVirtualMachineLogonPassword()
env.key_filename = configuration.getServiceCertificateKeyFilename()
env.roledefs = {'build': [configuration.getBuildHostname()]}
if label is not None:
env.roledefs.update({'web': configuration.getWebHostnames()})
# Information about main CodaLab repo
env.git_user = configuration.getGitUser()
env.git_repo = configuration.getGitRepo()
env.git_tag = configuration.getGitTag()
env.git_repo_url = 'https://github.com/{0}/{1}.git'.format(
env.git_user, env.git_repo)
# Information about Bundles repo
env.git_bundles_user = configuration.getBundleServiceGitUser()
env.git_bundles_repo = configuration.getBundleServiceGitRepo()
env.git_bundles_tag = configuration.getBundleServiceGitTag()
if len(configuration.getBundleServiceUrl()) > 0:
env.git_bundles_repo_url = 'https://github.com/{0}/{1}.git'.format(
env.git_bundles_user, env.git_bundles_repo)
else:
env.git_bundles_repo_url = ''
env.deploy_dir = 'deploy'
env.build_archive = '{0}.tar.gz'.format(env.git_tag)
env.django_settings_module = 'codalab.settings'
env.django_configuration = configuration.getDjangoConfiguration()
env.config_http_port = '80'
env.config_server_name = "{0}.cloudapp.net".format(
configuration.getServiceName())
env.configuration = True
def config(label=None):
"""
Reads deployment parameters for the given setup.
label: Label identifying the desired setup.
"""
env.cfg_label = label
print "Deployment label is: ", env.cfg_label
filename = ".codalabconfig"
if "cfg_path" not in env:
env.cfg_path = os.path.join(os.getcwd(), filename)
if os.path.exists(env.cfg_path) == False:
env.cfg_path = os.path.join(os.path.expanduser("~"), filename)
print "Loading configuration from: ", env.cfg_path
configuration = DeploymentConfig(label, env.cfg_path)
print "Configuring logger..."
logging.config.dictConfig(configuration.getLoggerDictConfig())
logger.info("Loaded configuration from file: %s", configuration.getFilename())
env.user = configuration.getVirtualMachineLogonUsername()
env.password = configuration.getVirtualMachineLogonPassword()
env.key_filename = configuration.getServiceCertificateKeyFilename()
env.roledefs = {"build": [configuration.getBuildHostname()]}
if label is not None:
env.roledefs.update({"web": configuration.getWebHostnames()})
# Information about main CodaLab repo
env.git_user = configuration.getGitUser()
env.git_repo = configuration.getGitRepo()
env.git_tag = configuration.getGitTag()
env.git_repo_url = "https://github.com/{0}/{1}.git".format(env.git_user, env.git_repo)
# Information about Bundles repo
env.git_bundles_user = configuration.getBundleServiceGitUser()
env.git_bundles_repo = configuration.getBundleServiceGitRepo()
env.git_bundles_tag = configuration.getBundleServiceGitTag()
if len(configuration.getBundleServiceUrl()) > 0:
env.git_bundles_repo_url = "https://github.com/{0}/{1}.git".format(
env.git_bundles_user, env.git_bundles_repo
)
else:
env.git_bundles_repo_url = ""
env.deploy_dir = "deploy"
env.build_archive = "{0}.tar.gz".format(env.git_tag)
env.django_settings_module = "codalab.settings"
env.django_configuration = configuration.getDjangoConfiguration()
env.config_http_port = "80"
env.config_server_name = "{0}.cloudapp.net".format(configuration.getServiceName())
env.configuration = True
def deploy_compute_worker(label):
'''
Deploy/update compute workers.
For monitoring make sure the azure instance has the port 8000 forwarded
:param label: Either test or prod
'''
env.deploy_codalab_dir = 'codalab-competitions'
# Create .codalabconfig within home directory
env.label = label
cfg = DeploymentConfig(
env.label,
env.cfg_path if hasattr(env, 'cfg_path') else '.codalabconfig')
dep = Deployment(cfg)
buf = StringIO()
buf.write(dep.get_compute_workers_file_content())
settings_file = os.path.join('~', '.codalabconfig')
put(buf, settings_file)
env.git_codalab_tag = cfg.getGitTag()
env.logs_password = cfg.get_compute_worker_logs_password()
# Initial setup
with cd(env.deploy_codalab_dir):
run('git checkout %s' % env.git_codalab_tag)
run('git pull')
run('source /home/azureuser/codalab-competitions/venv/bin/activate && pip install -r /home/azureuser/codalab-competitions/codalab/requirements/dev_azure.txt'
)
# run('./dev_setup.sh')
# run("source /home/azureuser/codalab-competitions/venv/bin/activate && pip install bottle==0.12.8")
current_directory = os.path.dirname(os.path.realpath(__file__))
put(local_path='{}/configs/upstart/codalab-compute-worker.conf'.format(
current_directory),
remote_path='/etc/init/codalab-compute-worker.conf',
use_sudo=True)
put(local_path='{}/configs/upstart/codalab-monitor.conf'.format(
current_directory),
remote_path='/etc/init/codalab-monitor.conf',
use_sudo=True)
# run("echo %s > /home/azureuser/codalab-competitions/codalab/codalabtools/compute/password.txt" % env.logs_password)
with settings(warn_only=True):
sudo("stop codalab-compute-worker")
sudo("stop codalab-monitor")
sudo("start codalab-compute-worker")
sudo("start codalab-monitor")
def deploy_compute_worker(label):
'''
Deploy/update compute workers.
For monitoring make sure the azure instance has the port 8000 forwarded
:param label: Either test or prod
'''
env.deploy_codalab_dir = 'codalab-competitions'
# Create .codalabconfig within home directory
env.label = label
cfg = DeploymentConfig(env.label, env.cfg_path if hasattr(env, 'cfg_path') else '.codalabconfig')
dep = Deployment(cfg)
buf = StringIO()
buf.write(dep.get_compute_workers_file_content())
settings_file = os.path.join('~', '.codalabconfig')
put(buf, settings_file)
env.git_codalab_tag = cfg.getGitTag()
env.logs_password = cfg.get_compute_worker_logs_password()
# Initial setup
with cd(env.deploy_codalab_dir):
run('git checkout %s' % env.git_codalab_tag)
run('git pull')
run('source /home/azureuser/codalab-competitions/venv/bin/activate && pip install -r /home/azureuser/codalab-competitions/codalab/requirements/dev_azure.txt')
# run('./dev_setup.sh')
# run("source /home/azureuser/codalab-competitions/venv/bin/activate && pip install bottle==0.12.8")
current_directory = os.path.dirname(os.path.realpath(__file__))
put(
local_path='{}/configs/upstart/codalab-compute-worker.conf'.format(current_directory),
remote_path='/etc/init/codalab-compute-worker.conf',
use_sudo=True
)
put(
local_path='{}/configs/upstart/codalab-monitor.conf'.format(current_directory),
remote_path='/etc/init/codalab-monitor.conf',
use_sudo=True
)
# run("echo %s > /home/azureuser/codalab-competitions/codalab/codalabtools/compute/password.txt" % env.logs_password)
with settings(warn_only=True):
sudo("stop codalab-compute-worker")
sudo("stop codalab-monitor")
sudo("start codalab-compute-worker")
sudo("start codalab-monitor")
def deploy_compute_worker(label):
'''
Deploy/update compute workers.
For monitoring make sure the azure instance has the port 8000 forwarded
:param label: Either test or prod
'''
env.deploy_codalab_dir = 'codalab-competitions'
# Create .codalabconfig within home directory
env.label = label
cfg = DeploymentConfig(
env.label,
env.cfg_path if hasattr(env, 'cfg_path') else '.codalabconfig')
dep = Deployment(cfg)
buf = StringIO()
buf.write(dep.get_compute_workers_file_content())
settings_file = os.path.join('~', '.codalabconfig')
put(buf, settings_file)
env.git_codalab_tag = cfg.getGitTag()
env.logs_password = cfg.get_compute_worker_logs_password()
# Initial setup
with cd(env.deploy_codalab_dir):
run('git checkout %s' % env.git_codalab_tag)
run('git pull')
# make sure we remove old version, added 2/20/17 can remove a while after that
run('source /home/azureuser/codalab-competitions/venv/bin/activate && pip uninstall django-storages && pip uninstall django-storages-redux',
warn_only=True)
run('source /home/azureuser/codalab-competitions/venv/bin/activate && pip install --upgrade pip && pip install -r /home/azureuser/codalab-competitions/codalab/requirements/dev_azure.txt'
)
# run('./dev_setup.sh')
# run("source /home/azureuser/codalab-competitions/venv/bin/activate && pip install bottle==0.12.8")
current_directory = os.path.dirname(os.path.realpath(__file__))
# Adding compute worker upstart config
broker_url = cfg.get_broker_url()
compute_worker_conf_template = open(
'{}/configs/upstart/codalab-compute-worker.conf'.format(
current_directory)).read()
compute_worker_conf = compute_worker_conf_template.format(
broker_url=broker_url)
compute_worker_conf_buf = StringIO()
compute_worker_conf_buf.write(compute_worker_conf)
put(
local_path=
compute_worker_conf_buf, # Path can be a file-like object, in this case our processed template
remote_path='/etc/init/codalab-compute-worker.conf',
use_sudo=True)
# Adding codalab monitor upstart config ### No longer needed!
# put(
# local_path='{}/configs/upstart/codalab-monitor.conf'.format(current_directory),
# remote_path='/etc/init/codalab-monitor.conf',
# use_sudo=True
# )
# run("echo %s > /home/azureuser/codalab-competitions/codalab/codalabtools/compute/password.txt" % env.logs_password)
with settings(warn_only=True):
sudo("stop codalab-compute-worker")
sudo("stop codalab-monitor"
) # just in case it's left from a previous install
sudo("start codalab-compute-worker")
