def _update_user_data(self): if 'filesystems' in self.app.ud: old_fs_list = self.app.ud.get('filesystems') or [] new_fs_list = [] # clear 'services' and replace with the new format for fs in old_fs_list: svc_roles = ServiceRole.from_string_array(fs['roles']) if ServiceRole.GALAXY_TOOLS in svc_roles and ServiceRole.GALAXY_DATA in svc_roles: # This condition should only occur in new configs, but check # added so that things work properly even if run against a new config # Only works for default configs though... new_fs_list.append(fs) elif ServiceRole.GALAXY_TOOLS in svc_roles: pass # skip adding the galaxy tools file system, no longer needed. else: if ServiceRole.GALAXY_DATA in ServiceRole.from_string_array(fs['roles']): fs['roles'] = ServiceRole.to_string_array([ServiceRole.GALAXY_TOOLS, ServiceRole.GALAXY_DATA]) new_fs_list.append(fs) else: new_fs_list.append(fs) self.app.ud['filesystems'] = new_fs_list self.app.ud['deployment_version'] = 2 self.app.ud.pop('galaxy_home', None) # TODO: Galaxy home is always reset # to default. Discuss implications return True
def _update_user_data(self): if 'filesystems' in self.app.config.user_data: old_fs_list = self.app.config.user_data.get('filesystems') or [] new_fs_list = [] # clear 'services' and replace with the new format for fs in old_fs_list: svc_roles = ServiceRole.from_string_array(fs['roles']) if ServiceRole.GALAXY_TOOLS in svc_roles and ServiceRole.GALAXY_DATA in svc_roles: # This condition should only occur in new configs, but check # added so that things work properly even if run against a new config # Only works for default configs though... new_fs_list.append(fs) elif ServiceRole.GALAXY_TOOLS in svc_roles: pass # skip adding the galaxy tools file system, no longer needed. else: if ServiceRole.GALAXY_DATA in ServiceRole.from_string_array(fs['roles']): fs['roles'] = ServiceRole.to_string_array([ServiceRole.GALAXY_TOOLS, ServiceRole.GALAXY_DATA]) new_fs_list.append(fs) else: new_fs_list.append(fs) self.app.config.user_data['filesystems'] = new_fs_list self.app.config.user_data['deployment_version'] = 2 self.app.config.user_data.pop('galaxy_home', None) # TODO: Galaxy home is always reset to default. Discuss implications return True
def __init__(self, app, instance_role='master'): super(PSSService, self).__init__(app) self.svc_roles = [ServiceRole.PSS] self.name = ServiceRole.to_string(ServiceRole.PSS) self.svc_type = ServiceType.CM_SERVICE self.instance_role = instance_role log.debug("Configured PSS as {0}".format(self.instance_role))
def __init__(self, app): super(ClouderaManagerService, self).__init__(app) self.svc_roles = [ServiceRole.CLOUDERA_MANAGER] self.name = ServiceRole.to_string(ServiceRole.CLOUDERA_MANAGER) self.dependencies = [] self.remaining_start_attempts = NUM_START_ATTEMPTS self.db_pwd = misc.random_string_generator() # Indicate if the web server has been configured and started self.started = False self.cm_port = 7180 # Default cluster configuration # TODO - read local cloud host name! # self.cm_host = socket.gethostname() self.cm_host = self.app.cloud_interface.get_local_hostname() self.host_list = [] self.cluster_name = "Cluster 1" self.cdh_version = "CDH5" self.cdh_version_number = "5" self.cm_username = "******" self.cm_password = "******" self.mgmt_service_name = "ManagementService" self.host_username = "******" self.host_password = self.app.config.get('password') self.cm_repo_url = None self.service_types_and_names = { "HDFS": "HDFS", "YARN": "YARN" }
def __init__(self, app): super(ClouderaManagerService, self).__init__(app) self.svc_roles = [ServiceRole.CLOUDERA_MANAGER] self.name = ServiceRole.to_string(ServiceRole.CLOUDERA_MANAGER) self.dependencies = [] self.db_pwd = misc.random_string_generator() # Indicate if the web server has been configured and started self.started = False self.cm_port = 7180 # Default cluster configuration self.cm_host = socket.gethostname() self.host_list = [self.cm_host] self.cluster_name = "Cluster 1" self.cdh_version = "CDH5" self.cdh_version_number = "5" self.cm_username = "******" self.cm_password = "******" self.cm_service_name = "ManagementService" self.host_username = "******" # Read the password from the system! self.host_password = self.app.config.get('password') self.cm_repo_url = None self.service_types_and_names = { "HDFS": "HDFS", "YARN": "YARN", "ZOOKEEPER": "ZooKeeper" }
def __init__(self, app): super(PostgresService, self).__init__(app) self.name = ServiceRole.to_string(ServiceRole.GALAXY_POSTGRES) self.svc_roles = [ServiceRole.GALAXY_POSTGRES] self.psql_port = app.path_resolver.psql_db_port self.dependencies = [ServiceDependency(self, ServiceRole.GALAXY_DATA), ServiceDependency(self, ServiceRole.MIGRATION)]
def __init__(self, app): super(HadoopService, self).__init__(app) self.svc_roles = [ServiceRole.HADOOP] self.name = ServiceRole.to_string(ServiceRole.HADOOP) self.dependencies = [ServiceDependency(self, ServiceRole.SGE)] self.id_rsa_path = os.path.join(paths.P_HADOOP_HOME, "id_rsa") self.id_rsa_pub_key_path = os.path.join(paths.P_HADOOP_HOME, "id_rsa.pub")
def __init__(self, app): super(HadoopService, self).__init__(app) self.svc_roles = [ServiceRole.HADOOP] self.name = ServiceRole.to_string(ServiceRole.HADOOP) self.dependencies = [ServiceDependency(self, ServiceRole.SGE)] self.id_rsa_path = os.path.join(paths.P_HADOOP_HOME, "id_rsa") self.id_rsa_pub_key_path = os.path.join( paths.P_HADOOP_HOME, "id_rsa.pub")
def __init__(self, app): super(MigrationService, self).__init__(app) self.svc_roles = [ServiceRole.MIGRATION] self.name = ServiceRole.to_string(ServiceRole.MIGRATION) self.dependencies = [] if 'filesystems' in self.app.ud: for fs in self.app.ud.get('filesystems') or []: # Wait for galaxy data, indices and tools to come up before attempting migration if ServiceRole.GALAXY_DATA in ServiceRole.from_string_array(fs['roles']): self.dependencies.append(ServiceDependency(self, ServiceRole.GALAXY_DATA)) if ServiceRole.GALAXY_TOOLS in ServiceRole.from_string_array(fs['roles']): self.dependencies.append(ServiceDependency(self, ServiceRole.GALAXY_TOOLS)) if ServiceRole.GALAXY_INDICES in ServiceRole.from_string_array(fs['roles']): self.dependencies.append(ServiceDependency(self, ServiceRole.GALAXY_INDICES))
def __init__(self, app): super(MigrationService, self).__init__(app) self.svc_roles = [ServiceRole.MIGRATION] self.name = ServiceRole.to_string(ServiceRole.MIGRATION) self.svc_type = ServiceType.CM_SERVICE self.dependencies = [] if 'filesystems' in self.app.config.user_data: for fs in self.app.config.user_data.get('filesystems') or []: # Wait for galaxy data, indices and tools to come up before attempting migration if ServiceRole.GALAXY_DATA in ServiceRole.from_string_array(fs['roles']): self.dependencies.append(ServiceDependency(self, ServiceRole.GALAXY_DATA)) if ServiceRole.GALAXY_TOOLS in ServiceRole.from_string_array(fs['roles']): self.dependencies.append(ServiceDependency(self, ServiceRole.GALAXY_TOOLS)) if ServiceRole.GALAXY_INDICES in ServiceRole.from_string_array(fs['roles']): self.dependencies.append(ServiceDependency(self, ServiceRole.GALAXY_INDICES))
def __init__(self, app): super(SlurmdService, self).__init__(app) self.svc_roles = [ServiceRole.SLURMD] self.name = ServiceRole.to_string(ServiceRole.SLURMD) self.dependencies = [ ServiceDependency(self, ServiceRole.SLURMCTLD), ] self.num_restarts = 0 self.max_restarts = 3
def __init__(self, app): super(PulsarService, self).__init__(app) self.pulsar_home = None self.pulsar_port = DEFAULT_PULSAR_PORT self.name = ServiceRole.to_string(ServiceRole.PULSAR) self.svc_roles = [ServiceRole.PULSAR] self.dependencies = [ ServiceDependency(self, ServiceRole.JOB_MANAGER) ]
def __init__(self, app): super(CloudgeneService, self).__init__(app) self.svc_roles = [ServiceRole.CLOUDGENE] self.name = ServiceRole.to_string(ServiceRole.CLOUDGENE) self.dependencies = [ServiceDependency(self, ServiceRole.CLOUDERA_MANAGER)] self.port = 8085 self.cg_url = "http://cloudgene.uibk.ac.at/downloads/cloudgene-cloudman.tar.gz" self.cg_base_dir = '/mnt/galaxy/cloudgene/' self.cg_home = os.path.join(self.cg_base_dir, 'cloudgene-cloudman')
def __init__(self, app): super(CloudgeneService, self).__init__(app) self.svc_roles = [ServiceRole.CLOUDGENE] self.name = ServiceRole.to_string(ServiceRole.CLOUDGENE) self.dependencies = [ServiceDependency(self, ServiceRole.CLOUDERA_MANAGER)] self.port = 8085 self.cg_url = "https://cloudman.s3.amazonaws.com/files/cloudgene-cloudman-daemon.tar.gz" self.cg_base_dir = '/mnt/galaxy/cloudgene/' self.cg_home = os.path.join(self.cg_base_dir, 'cloudgene-daemon')
def __init__(self, app): super(NodejsProxyService, self).__init__(app) self.svc_roles = [ServiceRole.NODEJSPROXY] self.name = ServiceRole.to_string(ServiceRole.NODEJSPROXY) self.dependencies = [ServiceDependency(self, ServiceRole.GALAXY_TOOLS), ServiceDependency(self, ServiceRole.SUPERVISOR)] self.np_port = 8800 self.supervisor_conf_dir = '/etc/supervisor/conf.d' self.supervisor_prog_name = 'galaxy_nodejs_proxy'
def __init__(self, app): """ Initialize the service class by setting role and indicating dependencies """ super(ProFTPdService, self).__init__(app) log.debug("Initializing ProFTPdService") self.svc_roles = [ServiceRole.PROFTPD] self.name = ServiceRole.to_string(ServiceRole.PROFTPD) self.dependencies = [ServiceDependency(self, ServiceRole.GALAXY_POSTGRES), ServiceDependency(self, ServiceRole.GALAXY_DATA)]
def __init__(self, app, as_min=-1, as_max=-1, instance_type=None): super(AutoscaleService, self).__init__(app) self.state = service_states.UNSTARTED self.svc_roles = [ServiceRole.AUTOSCALE] self.svc_type = ServiceType.CM_SERVICE self.name = ServiceRole.to_string(ServiceRole.AUTOSCALE) self.dependencies = [ServiceDependency(self, ServiceRole.MIGRATION)] self.as_max = as_max # Max number of nodes autoscale should maintain self.as_min = as_min # Min number of nodes autoscale should maintain self.instance_type = instance_type # Type of instances to start
def __init__(self, app): super(LwrService, self).__init__(app) self.lwr_home = self.app.path_resolver.lwr_home self.lwr_port = DEFAULT_LWR_PORT self.name = ServiceRole.to_string(ServiceRole.LWR) self.svc_roles = [ServiceRole.LWR] self.dependencies = [ ServiceDependency(self, ServiceRole.SGE), # Well someday anyway :) ServiceDependency(self, ServiceRole.GALAXY_TOOLS) # Anyway to make this depend on where LWR installed? ]
def __init__(self, app): super(GalaxyReportsService, self).__init__(app) self.galaxy_home = self.app.path_resolver.galaxy_home self.reports_port = DEFAULT_REPORTS_PORT self.name = ServiceRole.to_string(ServiceRole.GALAXY_REPORTS) self.svc_roles = [ServiceRole.GALAXY_REPORTS] self.dependencies = [ServiceDependency( self, ServiceRole.GALAXY)] # Hopefully Galaxy dependency alone enough to ensure database migrated, etc... self.conf_dir = os.path.join( self.app.path_resolver.galaxy_home, 'reports.conf.d')
def __init__(self, app, instance_role='master'): super(PSSService, self).__init__(app) self.svc_roles = [ServiceRole.PSS] self.name = ServiceRole.to_string(ServiceRole.PSS) self.instance_role = instance_role log.debug("Configured PSS as {0}".format(self.instance_role)) # Name of the default script to run self.pss_filename = 'post_start_script' if self.instance_role == 'master' \ else 'worker_post_start_script' self.pss_url = self.app.ud.get('post_start_script_url', None) if self.instance_role == 'master' \ else self.app.ud.get('worker_post_start_script_url', None)
def __init__(self, app): super(SupervisorService, self).__init__(app) self.svc_roles = [ServiceRole.SUPERVISOR] self.name = ServiceRole.to_string(ServiceRole.SUPERVISOR) self.dependencies = [] self.sv_port = 9002 self.pid_file = '/var/run/supervisord.pid' self.main_conf_file = '/etc/supervisord.conf' self.conf_dir = '/etc/supervisor/conf.d' self.server = None self.log_file = '/var/log/supervisor/supervisord.log'
def __init__(self, filesystem, from_archive=None): """ Instance's transient storage exposed over NFS. """ super(TransientStorage, self).__init__(filesystem.app) self.fs = filesystem self.app = self.fs.app self.device = None self.from_archive = from_archive self.svc_roles = [ServiceRole.TRANSIENT_NFS] self.name = ServiceRole.to_string(ServiceRole.TRANSIENT_NFS)
def __init__(self, app): super(NodejsProxyService, self).__init__(app) self.svc_roles = [ServiceRole.NODEJSPROXY] self.name = ServiceRole.to_string(ServiceRole.NODEJSPROXY) self.dependencies = [ ServiceDependency(self, ServiceRole.GALAXY_TOOLS), ServiceDependency(self, ServiceRole.SUPERVISOR) ] self.np_port = 8800 self.supervisor_conf_dir = '/etc/supervisor/conf.d' self.supervisor_prog_name = 'galaxy_nodejs_proxy'
def __init__(self, app): super(PulsarService, self).__init__(app) self.name = ServiceRole.to_string(ServiceRole.PULSAR) self.svc_roles = [ServiceRole.PULSAR] self.dependencies = [ ServiceDependency(self, ServiceRole.JOB_MANAGER), ServiceDependency(self, ServiceRole.SUPERVISOR) ] self.pulsar_home = '/mnt/pulsar' self.pulsar_port = 8913 self.supervisor_conf_dir = '/etc/supervisor/conf.d' self.supervisor_prog_name = 'pulsar'
def __init__(self, app): """ Initialize the service class by setting role and indicating dependencies """ super(ProFTPdService, self).__init__(app) log.debug("Initializing ProFTPdService") self.svc_roles = [ServiceRole.PROFTPD] self.name = ServiceRole.to_string(ServiceRole.PROFTPD) self.dependencies = [ ServiceDependency(self, ServiceRole.GALAXY_POSTGRES), ServiceDependency(self, ServiceRole.GALAXY_DATA) ]
def __init__(self, app): super(GalaxyReportsService, self).__init__(app) self.galaxy_home = self.app.path_resolver.galaxy_home self.reports_port = DEFAULT_REPORTS_PORT self.name = ServiceRole.to_string(ServiceRole.GALAXY_REPORTS) self.svc_roles = [ServiceRole.GALAXY_REPORTS] # Hopefully Galaxy dependency alone enough to ensure database migrated, etc... self.dependencies = [ ServiceDependency(self, ServiceRole.GALAXY), ServiceDependency(self, ServiceRole.GALAXY_POSTGRES) ] self.conf_dir = os.path.join(self.app.path_resolver.galaxy_home, 'reports.conf.d')
def __init__(self, app): super(NginxService, self).__init__(app) self.svc_roles = [ServiceRole.NGINX] self.name = ServiceRole.to_string(ServiceRole.NGINX) self.dependencies = [] self.exe = self.app.path_resolver.nginx_executable self.conf_dir = self.app.path_resolver.nginx_conf_dir self.conf_file = self.app.path_resolver.nginx_conf_file # Main conf file self.ssl_is_on = False # The list of services that Nginx service proxies self.proxied_services = ['Galaxy', 'GalaxyReports', 'Pulsar', 'ClouderaManager', 'Cloudgene'] # A list of currently active CloudMan services being proxied self.active_proxied = []
def __init__(self, app): super(NginxService, self).__init__(app) self.svc_roles = [ServiceRole.NGINX] self.name = ServiceRole.to_string(ServiceRole.NGINX) self.dependencies = [] self.exe = self.app.path_resolver.nginx_executable self.conf_dir = self.app.path_resolver.nginx_conf_dir self.conf_file = self.app.path_resolver.nginx_conf_file # Main conf file self.ssl_is_on = self.app.config.user_data.get('use_ssl', False) self.proxied_services = ['Galaxy', 'GalaxyReports', 'Pulsar', 'ClouderaManager', 'Cloudgene'] # A list of currently active CloudMan services being proxied self.active_proxied = [] self.was_started = False
def __init__(self, app, srv_type="master", host=""): """ the srv_type defines whether we are running a master node or a worker node. If we have run a worker the host IP should be passed in the host entry. """ super(HTCondorService, self).__init__(app) log.debug("Condor is preparing") self.svc_roles = [ServiceRole.HTCONDOR] self.name = ServiceRole.to_string(ServiceRole.HTCONDOR) self.srv_type = srv_type if self.srv_type == "master": self.flock_to = "" else: self.host = host
def __init__(self, app): super(ClouderaManagerService, self).__init__(app) self.svc_roles = [ServiceRole.CLOUDERA_MANAGER] self.name = ServiceRole.to_string(ServiceRole.CLOUDERA_MANAGER) self.dependencies = [] self.remaining_start_attempts = NUM_START_ATTEMPTS self.db_pwd = misc.random_string_generator() # Indicate if the web server has been configured and started self.started = False self.cm_port = 7180 # Default cluster configuration # TODO - read local cloud host name! # self.cm_host = socket.gethostname() self.cm_host = self.app.cloud_interface.get_local_hostname() # The actual worker nodes (note: this is a list of Instance objects) # (because get_worker_instances currently depends on tags, which is only # supported by EC2, get the list of instances only for the case of EC2 cloud. # This initialization is applicable only when restarting a cluster. # self.host_list = get_worker_instances() if ( # self.app.cloud_type == 'ec2' or self.app.cloud_type == 'openstack') else [] # self.host_list = ["w1", "w2"] # self.instances = self.app.manager.worker_instances # self.host_list = [l.get_local_hostname() for l in self.app.manager.worker_instances] # self.host_list = [l.get_private_ip for l in self.app.manager.worker_instances] self.host_list = None self.cluster_name = "Cluster 1" self.cdh_version = "CDH5" self.cdh_version_number = "5" self.cm_username = "******" self.cm_password = "******" self.mgmt_service_name = "ManagementService" self.host_username = "******" self.host_password = self.app.config.get('password') self.cm_repo_url = None self.hdfs_service_name = "HDFS" self.hadoop_data_dir_prefix = "/mnt/dfs" self.yarn_service_name = "YARN" self.parcel_version = "5.4.1" self.cmd_timeout = 180 self.api = None self.manager = None self.cluster = None self.hdfs_service = None self.yarn_service = None self.service_types_and_names = {"HDFS": "HDFS", "YARN": "YARN"}
def __init__(self, app): super(SlurmctldService, self).__init__(app) self.svc_roles = [ServiceRole.SLURMCTLD, ServiceRole.JOB_MANAGER] self.name = ServiceRole.to_string(ServiceRole.SLURMCTLD) self.dependencies = [ ServiceDependency(self, ServiceRole.MIGRATION), ServiceDependency(self, ServiceRole.TRANSIENT_NFS), ] self.slurm_info = SlurmInfo() self.num_restarts = 0 self.max_restarts = 3 # This must be the same on the workers self.slurm_lock_file = os.path.join( self.app.path_resolver.slurm_root_nfs, 'slurm.lockfile') # Following a cluster reboot, this file may have been left over so # clean it up before starting the service if os.path.exists(self.slurm_lock_file): os.remove(self.slurm_lock_file)
def __init__(self, app): super(SlurmctldService, self).__init__(app) self.svc_roles = [ServiceRole.SLURMCTLD, ServiceRole.JOB_MANAGER] self.name = ServiceRole.to_string(ServiceRole.SLURMCTLD) self.dependencies = [ ServiceDependency(self, ServiceRole.MIGRATION), ServiceDependency(self, ServiceRole.TRANSIENT_NFS), ] self.slurm_info = SlurmInfo() self.num_restarts = 0 self.max_restarts = 3 # This must be the same on the workers self.slurm_lock_file = os.path.join(self.app.path_resolver.slurm_root_nfs, 'slurm.lockfile') # Following a cluster reboot, this file may have been left over so # clean it up before starting the service if os.path.exists(self.slurm_lock_file): os.remove(self.slurm_lock_file)
def __init__(self, app): super(GalaxyService, self).__init__(app) self.name = ServiceRole.to_string(ServiceRole.GALAXY) self.svc_roles = [ServiceRole.GALAXY] self.remaining_start_attempts = NUM_START_ATTEMPTS self.configured = False # Indicates if the environment for running Galaxy has been configured self.ssl_is_on = False # Environment variables to set before executing galaxy's run.sh self.env_vars = {} self.dependencies = [ ServiceDependency(self, ServiceRole.JOB_MANAGER), ServiceDependency(self, ServiceRole.GALAXY_POSTGRES), ServiceDependency(self, ServiceRole.GALAXY_DATA), ServiceDependency(self, ServiceRole.GALAXY_INDICES), ServiceDependency(self, ServiceRole.GALAXY_TOOLS), ServiceDependency(self, ServiceRole.PROFTPD) ] self.option_manager = galaxy_option_manager(app)
def __init__(self, app): super(GalaxyService, self).__init__(app) self.name = ServiceRole.to_string(ServiceRole.GALAXY) self.svc_roles = [ServiceRole.GALAXY] self.remaining_start_attempts = NUM_START_ATTEMPTS # Indicates if the environment for running Galaxy has been configured self.configured = False self.ssl_is_on = False # Environment variables to set before executing galaxy's run.sh self.env_vars = {} self.dependencies = [ ServiceDependency(self, ServiceRole.JOB_MANAGER), ServiceDependency(self, ServiceRole.GALAXY_POSTGRES), ServiceDependency(self, ServiceRole.GALAXY_DATA), ServiceDependency(self, ServiceRole.GALAXY_INDICES), # ServiceDependency(self, ServiceRole.PROFTPD), ServiceDependency(self, ServiceRole.GALAXY_TOOLS) ] self.option_manager = galaxy_option_manager(app)
def __init__(self, app): super(GalaxyService, self).__init__(app) self.name = ServiceRole.to_string(ServiceRole.GALAXY) self.svc_roles = [ServiceRole.GALAXY] self.remaining_start_attempts = NUM_START_ATTEMPTS self.configured = False # Indicates if the environment for running Galaxy has been configured # Environment variables to set before executing galaxy's run.sh self.env_vars = { "SGE_ROOT": self.app.path_resolver.sge_root, "DRMAA_LIBRARY_PATH": self.app.path_resolver.drmaa_library_path } self.dependencies = [ ServiceDependency(self, ServiceRole.SGE), ServiceDependency(self, ServiceRole.GALAXY_POSTGRES), ServiceDependency(self, ServiceRole.GALAXY_DATA), ServiceDependency(self, ServiceRole.GALAXY_INDICES), ServiceDependency(self, ServiceRole.GALAXY_TOOLS), ServiceDependency(self, ServiceRole.PROFTPD) ] self.option_manager = galaxy_option_manager(app)
def __init__(self, app, as_min=-1, as_max=-1, instance_type=None, num_queued_jobs=2, mean_runtime_threshold=60, num_instances_to_add=1): """ :type as_min: int :param as_min: The minimum number of worker nodes of maintain. :type as_max: int :param as_max: The maximum number of worker nodes to maintain. :type instance_type: str :param instance_type: The type of instance to use. :type num_queued_jobs: int :param num_queued_jobs: Minimum number of jobs that need to be queued before autoscaling will trigger. :type mean_runtime_threshold: int :param mean_runtime_threshold: Mean running job runtime before autoscaling will trigger. :type num_instances_to_add: int :param num_instances_to_add: Number of instances to add when scaling up. """ super(AutoscaleService, self).__init__(app) self.state = service_states.UNSTARTED self.svc_roles = [ServiceRole.AUTOSCALE] self.svc_type = ServiceType.CM_SERVICE self.name = ServiceRole.to_string(ServiceRole.AUTOSCALE) self.dependencies = [ServiceDependency(self, ServiceRole.MIGRATION)] self.as_max = as_max self.as_min = as_min self.instance_type = instance_type
def send_add_s3fs(self, bucket_name, svc_roles): msg = 'ADDS3FS | {0} | {1}'.format(bucket_name, ServiceRole.to_string(svc_roles)) self._send_msg(msg)
log.debug("Tried to create a volume but it is in state '%s' (volume ID: %s)" % (self.status, self.volume_id)) # Add tags to newly created volumes (do this outside the inital if/else # to ensure the tags get assigned even if using an existing volume vs. # creating a new one) try: self.app.cloud_interface.add_tag( self.volume, 'clusterName', self.app.config['cluster_name']) self.app.cloud_interface.add_tag( self.volume, 'bucketName', self.app.config['bucket_cluster']) if filesystem: self.app.cloud_interface.add_tag(self.volume, 'filesystem', filesystem) self.app.cloud_interface.add_tag(self.volume, 'Name', "{0}FS".format(filesystem)) self.app.cloud_interface.add_tag(self.volume, 'roles', ServiceRole.to_string(self.fs.svc_roles)) except EC2ResponseError, e: log.error("Error adding tags to volume: %s" % e) def delete(self): """ Delete this volume. """ try: volume_id = self.volume_id self.volume.delete() log.debug("Deleted volume '%s'" % volume_id) self.volume = None except EC2ResponseError, e: log.error("Error deleting volume '%s' - you should delete it manually " "after the cluster has shut down: %s" % (self.volume_id, e))
def __init__(self, app): super(SGEService, self).__init__(app) self.svc_roles = [ServiceRole.SGE, ServiceRole.JOB_MANAGER] self.name = ServiceRole.to_string(ServiceRole.SGE) self.dependencies = [ServiceDependency(self, ServiceRole.MIGRATION)] self.sge_info = SGEInfo()
def normalize_user_data(app, ud): """ Normalize user data format to a consistent representation used within CloudMan. This is useful as user data and also persistent data evolve over time and thus calling this method at app start enables any necessary translation to happen. """ if ud.get('persistent_data_version', 1) < app.PERSISTENT_DATA_VERSION: # First make a backup of the deprecated persistent data file s3_conn = app.cloud_interface.get_s3_connection() copy_file_in_bucket( s3_conn, ud['bucket_cluster'], ud['bucket_cluster'], 'persistent_data.yaml', 'persistent_data-deprecated.yaml', preserve_acl=False, validate=False) # Convert (i.e., normalize) v2 ud if 'filesystems' in ud: log.debug("Normalizing v2 user data") for fs in ud['filesystems']: if 'roles' not in fs: fs['roles'] = ServiceRole.legacy_convert(fs['name']) if 'delete_on_termination' not in fs: if fs['kind'] == 'snapshot': fs['delete_on_termination'] = True else: fs['delete_on_termination'] = False for svc in ud.get('services', []): if 'roles' not in svc: svc['roles'] = ServiceRole.legacy_convert( svc.get('name', 'NoName')) # Convert (i.e., normalize) v1 ud if "static_filesystems" in ud or "data_filesystems" in ud: log.debug("Normalizing v1 user data") if 'filesystems' not in ud: ud['filesystems'] = [] if 'static_filesystems' in ud: for vol in ud['static_filesystems']: # Create a mapping between the old and the new format styles # Some assumptions are made here; namely, all static file systems # in the original data are assumed delete_on_termination, their name # defines their role and they are mounted under /mnt/<name> roles = ServiceRole.legacy_convert(vol['filesystem']) fs = {'kind': 'snapshot', 'name': vol['filesystem'], 'roles': roles, 'delete_on_termination': True, 'mount_point': os.path.join('/mnt', vol['filesystem']), 'ids': [vol['snap_id']]} ud['filesystems'].append(fs) ud.pop('static_filesystems') ud['cluster_type'] = 'Galaxy' if 'data_filesystems' in ud: for fs_name, fs in ud['data_filesystems'].items(): fs = {'kind': 'volume', 'name': fs_name, 'roles': ServiceRole.legacy_convert(fs_name), 'delete_on_termination': False, 'mount_point': os.path.join('/mnt', fs_name), 'ids': [fs[0]['vol_id']]} ud['filesystems'].append(fs) ud.pop('data_filesystems') if 'cluster_type' not in ud: ud['cluster_type'] = 'Data' if 'galaxy_home' in ud: ud.pop('galaxy_home') if 'services' in ud and 'service' in ud['services'][0]: log.debug("Normalizing v1 service user data") old_svc_list = ud['services'] ud['services'] = [] # clear 'services' and replace with the new format for svc in old_svc_list: if 'roles' not in svc: normalized_svc = {'name': svc['service'], 'roles': ServiceRole.legacy_convert(svc['service'])} ud['services'].append(normalized_svc) return ud
def normalize_user_data(app, ud): """ Normalize user data format to a consistent representation used within CloudMan. This is useful as user data and also persistent data evolve over time and thus calling this method at app start enables any necessary translation to happen. """ if ud.get("persistent_data_version", 1) < app.PERSISTENT_DATA_VERSION: # First make a backup of the deprecated persistent data file s3_conn = app.cloud_interface.get_s3_connection() copy_file_in_bucket( s3_conn, ud["bucket_cluster"], ud["bucket_cluster"], "persistent_data.yaml", "persistent_data-deprecated.yaml", preserve_acl=False, validate=False, ) # Convert (i.e., normalize) v2 ud if "filesystems" in ud: log.debug("Normalizing v2 user data") for fs in ud["filesystems"]: if "roles" not in fs: fs["roles"] = ServiceRole.legacy_convert(fs["name"]) if "delete_on_termination" not in fs: if fs["kind"] == "snapshot": fs["delete_on_termination"] = True else: fs["delete_on_termination"] = False for svc in ud.get("services", []): if "roles" not in svc: svc["roles"] = ServiceRole.legacy_convert(svc.get("name", "NoName")) # Convert (i.e., normalize) v1 ud if "static_filesystems" in ud or "data_filesystems" in ud: log.debug("Normalizing v1 user data") if "filesystems" not in ud: ud["filesystems"] = [] if "static_filesystems" in ud: for vol in ud["static_filesystems"]: # Create a mapping between the old and the new format styles # Some assumptions are made here; namely, all static file systems # in the original data are assumed delete_on_termination, their name # defines their role and they are mounted under /mnt/<name> roles = ServiceRole.legacy_convert(vol["filesystem"]) fs = { "kind": "snapshot", "name": vol["filesystem"], "roles": roles, "delete_on_termination": True, "mount_point": os.path.join("/mnt", vol["filesystem"]), "ids": [vol["snap_id"]], } ud["filesystems"].append(fs) ud.pop("static_filesystems") ud["cluster_type"] = "Galaxy" if "data_filesystems" in ud: for fs_name, fs in ud["data_filesystems"].items(): fs = { "kind": "volume", "name": fs_name, "roles": ServiceRole.legacy_convert(fs_name), "delete_on_termination": False, "mount_point": os.path.join("/mnt", fs_name), "ids": [fs[0]["vol_id"]], } ud["filesystems"].append(fs) ud.pop("data_filesystems") if "cluster_type" not in ud: ud["cluster_type"] = "Data" if "galaxy_home" in ud: ud.pop("galaxy_home") if "services" in ud and "service" in ud["services"][0]: log.debug("Normalizing v1 service user data") old_svc_list = ud["services"] ud["services"] = [] # clear 'services' and replace with the new format for svc in old_svc_list: if "roles" not in svc: normalized_svc = {"name": svc["service"], "roles": ServiceRole.legacy_convert(svc["service"])} ud["services"].append(normalized_svc) return ud
def __init__(self, app): super(SGEService, self).__init__(app) self.svc_roles = [ServiceRole.SGE] self.name = ServiceRole.to_string(ServiceRole.SGE) self.dependencies = [ServiceDependency(self, ServiceRole.MIGRATION)] self.hosts = []
def create(self, filesystem=None): """ Create a new volume. This can be done either from a snapshot (i.e., ``self.from_snapshot_id``) or a blank one if the snapshot ID is not set. Note that ``self.size`` needs to be set if creating a blank volume before calling the method. If creating a volume from a snapshot and ``self.size`` is not set, the new volume will be of the same size as the snapshot. """ if not self.size and not self.from_snapshot_id and not self.from_archive: log.error('Cannot add a {0} volume without a size, snapshot ID or ' 'archive url; aborting.'.format(self.fs)) return False # If creating the volume from a snaphost, get the expected volume size if self.from_snapshot_id and not self.volume: self.snapshot = self.app.cloud_interface.get_snapshot( self.from_snapshot_id) if not self.snapshot: log.error( "Did not retrieve Snapshot object for {0}; aborting.". format(self.from_snapshot_id)) return False # We need a size to be able to create a volume, so if none # is specified, use snapshot size if not self.size: si = self.app.cloud_interface.get_snapshot_info( self.from_snapshot_id) self.size = si.get('volume_size') # If it does not already exist, create the volume if self.status == volume_status.NONE: log.debug("Creating a new volume of size '%s' in zone '%s' from " "snapshot '%s' for %s." % (self.size, self.app.cloud_interface.get_zone(), self.from_snapshot_id, self.fs)) self.volume = self.app.cloud_interface.create_volume( self.size, self.app.cloud_interface.get_zone(), snapshot=self.from_snapshot_id) if self.volume: # When creating from a snapshot in Euca, volume.size may be None self.size = int(self.volume.size or 0) log.debug( "Created a new volume of size '%s' from snapshot '%s' " "with ID '%s' in zone '%s' for %s." % (self.size, self.from_snapshot_id, self.volume_id, self.app.cloud_interface.get_zone(), self.fs)) else: log.warning("No volume object - did not create a volume?") return False else: log.debug( "Tried to create a volume for %s but it is in state '%s' " "(volume ID: %s)" % (self.fs, self.status, self.volume_id)) return False # Add tags to newly created volumes (do this outside the inital if/else # to ensure the tags get assigned even if using an existing volume vs. # creating a new one) self.app.cloud_interface.add_tag(self.volume, 'Name', self.app.config['cluster_name']) self.app.cloud_interface.add_tag(self.volume, 'bucketName', self.app.config['bucket_cluster']) if self.fs: self.app.cloud_interface.add_tag(self.volume, 'filesystem', self.fs.get_full_name()) self.app.cloud_interface.add_tag( self.volume, 'roles', ServiceRole.to_string(self.fs.svc_roles)) return True