def config(label): """ Reads deployment parameters for the given setup. label: Label identifying the desired setup (e.g., prod, test, etc.) """ env.cfg_label = label print "Deployment label is:", env.cfg_label filename = ".codalabconfig" print "Loading configuration from:", env.cfg_path configuration = DeploymentConfig(label, env.cfg_path) print "Configuring logger..." logging.config.dictConfig(configuration.getLoggerDictConfig()) logger.info("Loaded configuration from file: %s", configuration.getFilename()) env.roledefs = {'web' : configuration.getWebHostnames()} # Credentials env.user = configuration.getVirtualMachineLogonUsername() env.password = configuration.getVirtualMachineLogonPassword() env.key_filename = configuration.getServiceCertificateKeyFilename() # Repository env.git_codalab_tag = configuration.getGitTag() env.git_codalab_cli_tag = configuration.getBundleServiceGitTag() env.deploy_codalab_dir = 'codalab' env.deploy_codalab_cli_dir = 'codalab-cli' env.django_settings_module = 'codalab.settings' env.django_configuration = configuration.getDjangoConfiguration() # Prod or Dev env.config_http_port = '80' env.config_server_name = "{0}.cloudapp.net".format(configuration.getServiceName()) env.configuration = True env.SHELL_ENV = {}
def config(label): """ Reads deployment parameters for the given setup. label: Label identifying the desired setup (e.g., prod, test, etc.) """ env.cfg_label = label print "Deployment label is:", env.cfg_label print "Loading configuration from:", env.cfg_path configuration = DeploymentConfig(label, env.cfg_path) print "Configuring logger..." logging.config.dictConfig(configuration.getLoggerDictConfig()) logger.info("Loaded configuration from file: %s", configuration.getFilename()) env.roledefs = {'web': configuration.getWebHostnames()} # Credentials env.user = configuration.getVirtualMachineLogonUsername() # Repository env.git_codalab_tag = configuration.getGitTag() env.deploy_codalab_dir = 'codalab-competitions' # Directory for codalab competitions env.django_settings_module = 'codalab.settings' env.django_configuration = configuration.getDjangoConfiguration( ) # Prod or Test env.config_http_port = '80' env.config_server_name = "{0}.cloudapp.net".format( configuration.getServiceName()) print "Deployment configuration is for:", env.config_server_name env.configuration = True env.SHELL_ENV = {}
def config(label=None): """ Reads deployment parameters for the given setup. label: Label identifying the desired setup. """ env.cfg_label = label print "Deployment label is: ", env.cfg_label if 'cfg_path' not in env: env.cfg_path = os.path.join(os.environ['HOME'], '.codalabconfig') print "Loading configuration from: ", env.cfg_path configuration = DeploymentConfig(label, env.cfg_path) print "Configuring logger..." logging.config.dictConfig(configuration.getLoggerDictConfig()) logger.info("Loaded configuration from file: %s", configuration.getFilename()) env.user = configuration.getVirtualMachineLogonUsername() env.password = configuration.getVirtualMachineLogonPassword() env.key_filename = configuration.getServiceCertificateKeyFilename() env.roledefs = { 'build' : [ configuration.getBuildHostname() ] } if label is not None: env.roledefs.update({ 'web' : configuration.getWebHostnames() }) env.git_user = configuration.getGitUser() env.git_repo = configuration.getGitRepo() env.git_tag = configuration.getGitTag() env.git_repo_url = 'https://github.com/{0}/{1}.git'.format(env.git_user, env.git_repo) env.deploy_dir = 'deploy' env.build_archive = '{0}.tar.gz'.format(env.git_tag) env.django_settings_module = 'codalab.settings' env.django_configuration = configuration.getDjangoConfiguration() env.config_http_port = '80' env.config_server_name = "{0}.cloudapp.net".format(configuration.getServiceName()) env.configuration = True
def config(label=None): """ Reads deployment parameters for the given setup. label: Label identifying the desired setup. """ env.cfg_label = label print "Deployment label is: ", env.cfg_label filename = ".codalabconfig" if 'cfg_path' not in env: env.cfg_path = os.path.join(os.getcwd(), filename) if os.path.exists(env.cfg_path) == False: env.cfg_path = os.path.join(os.path.expanduser("~"), filename) print "Loading configuration from: ", env.cfg_path configuration = DeploymentConfig(label, env.cfg_path) print "Configuring logger..." logging.config.dictConfig(configuration.getLoggerDictConfig()) logger.info("Loaded configuration from file: %s", configuration.getFilename()) env.user = configuration.getVirtualMachineLogonUsername() env.password = configuration.getVirtualMachineLogonPassword() env.key_filename = configuration.getServiceCertificateKeyFilename() env.roledefs = {'build': [configuration.getBuildHostname()]} if label is not None: env.roledefs.update({'web': configuration.getWebHostnames()}) # Information about main CodaLab repo env.git_user = configuration.getGitUser() env.git_repo = configuration.getGitRepo() env.git_tag = configuration.getGitTag() env.git_repo_url = 'https://github.com/{0}/{1}.git'.format( env.git_user, env.git_repo) # Information about Bundles repo env.git_bundles_user = configuration.getBundleServiceGitUser() env.git_bundles_repo = configuration.getBundleServiceGitRepo() env.git_bundles_tag = configuration.getBundleServiceGitTag() if len(configuration.getBundleServiceUrl()) > 0: env.git_bundles_repo_url = 'https://github.com/{0}/{1}.git'.format( env.git_bundles_user, env.git_bundles_repo) else: env.git_bundles_repo_url = '' env.deploy_dir = 'deploy' env.build_archive = '{0}.tar.gz'.format(env.git_tag) env.django_settings_module = 'codalab.settings' env.django_configuration = configuration.getDjangoConfiguration() env.config_http_port = '80' env.config_server_name = "{0}.cloudapp.net".format( configuration.getServiceName()) env.configuration = True
def config(label=None): """ Reads deployment parameters for the given setup. label: Label identifying the desired setup. """ env.cfg_label = label print "Deployment label is: ", env.cfg_label filename = ".codalabconfig" if "cfg_path" not in env: env.cfg_path = os.path.join(os.getcwd(), filename) if os.path.exists(env.cfg_path) == False: env.cfg_path = os.path.join(os.path.expanduser("~"), filename) print "Loading configuration from: ", env.cfg_path configuration = DeploymentConfig(label, env.cfg_path) print "Configuring logger..." logging.config.dictConfig(configuration.getLoggerDictConfig()) logger.info("Loaded configuration from file: %s", configuration.getFilename()) env.user = configuration.getVirtualMachineLogonUsername() env.password = configuration.getVirtualMachineLogonPassword() env.key_filename = configuration.getServiceCertificateKeyFilename() env.roledefs = {"build": [configuration.getBuildHostname()]} if label is not None: env.roledefs.update({"web": configuration.getWebHostnames()}) # Information about main CodaLab repo env.git_user = configuration.getGitUser() env.git_repo = configuration.getGitRepo() env.git_tag = configuration.getGitTag() env.git_repo_url = "https://github.com/{0}/{1}.git".format(env.git_user, env.git_repo) # Information about Bundles repo env.git_bundles_user = configuration.getBundleServiceGitUser() env.git_bundles_repo = configuration.getBundleServiceGitRepo() env.git_bundles_tag = configuration.getBundleServiceGitTag() if len(configuration.getBundleServiceUrl()) > 0: env.git_bundles_repo_url = "https://github.com/{0}/{1}.git".format( env.git_bundles_user, env.git_bundles_repo ) else: env.git_bundles_repo_url = "" env.deploy_dir = "deploy" env.build_archive = "{0}.tar.gz".format(env.git_tag) env.django_settings_module = "codalab.settings" env.django_configuration = configuration.getDjangoConfiguration() env.config_http_port = "80" env.config_server_name = "{0}.cloudapp.net".format(configuration.getServiceName()) env.configuration = True
def deploy_compute_worker(label): ''' Deploy/update compute workers. For monitoring make sure the azure instance has the port 8000 forwarded :param label: Either test or prod ''' env.deploy_codalab_dir = 'codalab-competitions' # Create .codalabconfig within home directory env.label = label cfg = DeploymentConfig( env.label, env.cfg_path if hasattr(env, 'cfg_path') else '.codalabconfig') dep = Deployment(cfg) buf = StringIO() buf.write(dep.get_compute_workers_file_content()) settings_file = os.path.join('~', '.codalabconfig') put(buf, settings_file) env.git_codalab_tag = cfg.getGitTag() env.logs_password = cfg.get_compute_worker_logs_password() # Initial setup with cd(env.deploy_codalab_dir): run('git checkout %s' % env.git_codalab_tag) run('git pull') run('source /home/azureuser/codalab-competitions/venv/bin/activate && pip install -r /home/azureuser/codalab-competitions/codalab/requirements/dev_azure.txt' ) # run('./dev_setup.sh') # run("source /home/azureuser/codalab-competitions/venv/bin/activate && pip install bottle==0.12.8") current_directory = os.path.dirname(os.path.realpath(__file__)) put(local_path='{}/configs/upstart/codalab-compute-worker.conf'.format( current_directory), remote_path='/etc/init/codalab-compute-worker.conf', use_sudo=True) put(local_path='{}/configs/upstart/codalab-monitor.conf'.format( current_directory), remote_path='/etc/init/codalab-monitor.conf', use_sudo=True) # run("echo %s > /home/azureuser/codalab-competitions/codalab/codalabtools/compute/password.txt" % env.logs_password) with settings(warn_only=True): sudo("stop codalab-compute-worker") sudo("stop codalab-monitor") sudo("start codalab-compute-worker") sudo("start codalab-monitor")
def deploy_compute_worker(label): ''' Deploy/update compute workers. For monitoring make sure the azure instance has the port 8000 forwarded :param label: Either test or prod ''' env.deploy_codalab_dir = 'codalab-competitions' # Create .codalabconfig within home directory env.label = label cfg = DeploymentConfig(env.label, env.cfg_path if hasattr(env, 'cfg_path') else '.codalabconfig') dep = Deployment(cfg) buf = StringIO() buf.write(dep.get_compute_workers_file_content()) settings_file = os.path.join('~', '.codalabconfig') put(buf, settings_file) env.git_codalab_tag = cfg.getGitTag() env.logs_password = cfg.get_compute_worker_logs_password() # Initial setup with cd(env.deploy_codalab_dir): run('git checkout %s' % env.git_codalab_tag) run('git pull') run('source /home/azureuser/codalab-competitions/venv/bin/activate && pip install -r /home/azureuser/codalab-competitions/codalab/requirements/dev_azure.txt') # run('./dev_setup.sh') # run("source /home/azureuser/codalab-competitions/venv/bin/activate && pip install bottle==0.12.8") current_directory = os.path.dirname(os.path.realpath(__file__)) put( local_path='{}/configs/upstart/codalab-compute-worker.conf'.format(current_directory), remote_path='/etc/init/codalab-compute-worker.conf', use_sudo=True ) put( local_path='{}/configs/upstart/codalab-monitor.conf'.format(current_directory), remote_path='/etc/init/codalab-monitor.conf', use_sudo=True ) # run("echo %s > /home/azureuser/codalab-competitions/codalab/codalabtools/compute/password.txt" % env.logs_password) with settings(warn_only=True): sudo("stop codalab-compute-worker") sudo("stop codalab-monitor") sudo("start codalab-compute-worker") sudo("start codalab-monitor")
def deploy_compute_worker(label): ''' Deploy/update compute workers. For monitoring make sure the azure instance has the port 8000 forwarded :param label: Either test or prod ''' env.deploy_codalab_dir = 'codalab-competitions' # Create .codalabconfig within home directory env.label = label cfg = DeploymentConfig( env.label, env.cfg_path if hasattr(env, 'cfg_path') else '.codalabconfig') dep = Deployment(cfg) buf = StringIO() buf.write(dep.get_compute_workers_file_content()) settings_file = os.path.join('~', '.codalabconfig') put(buf, settings_file) env.git_codalab_tag = cfg.getGitTag() env.logs_password = cfg.get_compute_worker_logs_password() # Initial setup with cd(env.deploy_codalab_dir): run('git checkout %s' % env.git_codalab_tag) run('git pull') # make sure we remove old version, added 2/20/17 can remove a while after that run('source /home/azureuser/codalab-competitions/venv/bin/activate && pip uninstall django-storages && pip uninstall django-storages-redux', warn_only=True) run('source /home/azureuser/codalab-competitions/venv/bin/activate && pip install --upgrade pip && pip install -r /home/azureuser/codalab-competitions/codalab/requirements/dev_azure.txt' ) # run('./dev_setup.sh') # run("source /home/azureuser/codalab-competitions/venv/bin/activate && pip install bottle==0.12.8") current_directory = os.path.dirname(os.path.realpath(__file__)) # Adding compute worker upstart config broker_url = cfg.get_broker_url() compute_worker_conf_template = open( '{}/configs/upstart/codalab-compute-worker.conf'.format( current_directory)).read() compute_worker_conf = compute_worker_conf_template.format( broker_url=broker_url) compute_worker_conf_buf = StringIO() compute_worker_conf_buf.write(compute_worker_conf) put( local_path= compute_worker_conf_buf, # Path can be a file-like object, in this case our processed template remote_path='/etc/init/codalab-compute-worker.conf', use_sudo=True) # Adding codalab monitor upstart config ### No longer needed! # put( # local_path='{}/configs/upstart/codalab-monitor.conf'.format(current_directory), # remote_path='/etc/init/codalab-monitor.conf', # use_sudo=True # ) # run("echo %s > /home/azureuser/codalab-competitions/codalab/codalabtools/compute/password.txt" % env.logs_password) with settings(warn_only=True): sudo("stop codalab-compute-worker") sudo("stop codalab-monitor" ) # just in case it's left from a previous install sudo("start codalab-compute-worker")