def __init__(self, name, s3_log_folder_path, aws_access_key = None, aws_secret_key = None, num_hosts = 1, region = 'us-west-2', instance_type = _DEFAULT_INSTANCE_TYPE, security_group = None, tags = None, CIDR_rule = None): super(EC2, self).__init__(name, _ENVIRONMENT.EC2) if(aws_access_key is not None and aws_secret_key is not None): self.aws_access_key, self.aws_secret_key = aws_access_key, aws_secret_key else: if bool(aws_access_key) != bool(aws_secret_key): raise IOError('You must specify both aws_access_key and aws_secret_key.') try: _get_credentials() except: __LOGGER__.warning('No AWS credentials were passed in and none are set globally.' \ ' In order to use this environment, you will need to' \ ' set your credentials globally using' \ ' graphlab.aws.set_credentials(...).' ) try: self.num_hosts = int(num_hosts) if num_hosts <= 0: raise TypeError("num_hosts must be a positive integer") except ValueError: raise TypeError("num_hosts is not an integer") if not isinstance(region, str): raise TypeError('region must be a string') if not isinstance(instance_type, str): raise TypeError('instance type must be a string') if security_group and not isinstance(security_group, str): raise TypeError('instance type must be a string') if tags and not isinstance(tags, dict): raise TypeError('tags must be a dict') if CIDR_rule and not (isinstance(CIDR_rule, str) or isinstance(CIDR_rule, list)): raise TypeError('CIDR_rule must be a string or a list of strings') (self.s3_bucket, self.s3_log_folder_path) = EC2._parse_s3_path(s3_log_folder_path) self.region = region self.instance_type = instance_type self.security_group = security_group self.tags = tags if tags else {} self.CIDR_rule = CIDR_rule
def run_job(self, job, session=None): # Determine which AWS credentials to use. aws_access_key, aws_secret_key = None, None if(hasattr(job.environment, 'aws_access_key') and hasattr(job.environment, 'aws_secret_key')): aws_access_key = job.environment.aws_access_key aws_secret_key = job.environment.aws_secret_key else: aws_access_key, aws_secret_key = _get_credentials() credentials = {'aws_access_key_id': aws_access_key, 'aws_secret_access_key': aws_secret_key} s3_log_folder = job.environment.s3_log_folder_path + job.name + '-' + str(_time.time()) + '/' s3_state_key = s3_log_folder + 'status' s3_log_key = s3_log_folder + 'log.txt' s3_status_key = s3_log_folder + 'jobstatus.txt' # Make sure we can write to S3 try: conn = _connect_s3(**credentials) bucket = conn.get_bucket(job.environment.s3_bucket, validate=False) k = _s3_key(bucket) k.key = s3_state_key k.set_contents_from_string('Pending') except: raise RuntimeError("Unable to start job. Could not write to S3 bucket: '%s'" % job.environment.s3_bucket) serialized_job_folder = _tempfile.mkdtemp(prefix='graphlab-ec2-job') serialized_job_file_path = _os.path.join(serialized_job_folder, 'job') job._serialize(serialized_job_file_path) commander = Ec2ExecutionEnvironment._start_commander_host(job.environment, credentials) post_url = "http://%s:9004/submit" % commander.public_dns_name __LOGGER__.debug("Sending %s to %s" % (serialized_job_file_path, post_url)) # Swallow all logging from the 'requests' module. logging.getLogger('requests').setLevel(logging.CRITICAL) # Technically the host should be started up now, but more than likely it needs a bit longer # before it's really started up. MAX_TRIES, current_tries, response = 60, 0, None post_params = {'s3_bucket': job.environment.s3_bucket, 's3_state_key': s3_state_key, 's3_log_key': s3_log_key, 's3_status_key': s3_status_key, 'required_packages': job.get_required_packages() } while current_tries < MAX_TRIES: with open(serialized_job_file_path, 'rb') as job_file: files = {'file': job_file} try: # Try to submit the work. current_tries += 1 response = _requests.post(post_url, files=files, params=post_params) break # Success except _requests.exceptions.ConnectionError, e: if current_tries >= MAX_TRIES: commander.stop() raise RuntimeError("Unable to submit job to EC2 instance: '%s'. Please check AWS Console to make sure any EC2 instances launched have been terminated." % commander.instance_id) _time.sleep(2)
def create(name, environment, state_path, description = None, api_key = None, admin_key = None, ssl_credentials = None): ''' Launch a Predictive Services cluster. This cluster can currently be launched on EC2 by specifying an EC2 environment. Parameters ---------- name : str The name of the Predictive Service that will be launched. This string can only contain: a-z, A-Z and hyphens. environment : :class:`~graphlab.deploy.environment.EC2` or str Must be either an EC2 Environment object or the name of EC2 environment. This is the environment where the Predictive Service will be deployed. Caching will always be used for query results but a distributed cache will only be used if the number of hosts is greater than two. state_path : str S3 path used to manage state for the Predictive Service. This path can also be used to create the Predictive Service object on another computer. description : str, optional Description of this Predictive Service. api_key : str, optional An API key that client must included with requests. If an api_key is not specified, one will be auto generated. The API Key can be retrieved from the return PredictiveService object. admin_key : str, optional An API key used for control operations (i.e. anything other than client requests). If an admin_key is not specified, one will be auto generated. The API Key can be retrieved from the return PredictiveService object. ssl_credentials : tuple of len three, with types: str, str, bool. The first string is the path to the private key file. The second string is the path to public key certificate. The third denotes whether the certificates are self signed (and any client should not verify the certificate). These files must be in the precise format AWS expects. Such a private key and a self-signed certificate can be generated using openssl with following commands: >>> openssl genrsa 1024 > privatekey.pem >>> openssl req -new -key privatekey.pem -out csr.pem >>> openssl x509 -req -days 365 -in csr.pem -signkey privatekey.pem -out server.crt If a tuple is not given, requests will be served over HTTP rather than HTTPS (i.e. encryption will not be used). Returns ------- out : :py:class:`~graphlab.deploy._predictive_service._predictive_service.PredictiveService` A Predictive Service object which can be used to manage the deployment. Examples -------- >>> ec2 = graphlab.deploy.environments['ec2'] >>> ps = graphlab.deploy.create('name', ec2, 's3://mybucket/pred_services/name') >>> print ps >>> ps.get_status() See Also -------- load ''' # local test hook if (environment is None): # load locally ret = _gl.deploy.predictive_service.load(state_path, override=True) ret._environment = _LocalPredictiveServiceEnvironment.launch(predictive_service_path = ret._s3_state_path, log_path = ret._environment.log_path, num_hosts = 3, admin_key = ret._environment.admin_key) return ret # Validate inputs for current session if not _gl.deploy._default_session.is_name_unique(name): # found another predictive service or predictive service endpoint with the same name raise RuntimeError("Validation Error: Predictive Service already exists with the name '%s', please rename or delete the existing Predictive Service." % name) # Validate 'environment' value if not isinstance(environment, _gl.deploy.environment.EC2) and not isinstance(environment, str): raise TypeError('Unsupported type given for environment parameter. Must be either an EC2' \ ' environment or the name of an EC2 environment.') if isinstance(environment, str): # Get the EC2 config object environment_name = environment environment = _gl.deploy.environments[environment_name] if not environment: raise TypeError("No environment named '%s'." % environment_name) if not isinstance(environment, _gl.deploy.environment.EC2): raise TypeError("%s is not an EC2 environment." % environment_name) # Save AWS config if(hasattr(environment, 'aws_access_key') and hasattr(environment, 'aws_secret_key')): aws_access_key = environment.aws_access_key aws_secret_key = environment.aws_secret_key else: try: aws_access_key, aws_secret_key = _get_credentials() except: raise IOError('No AWS credentials set. Credentials must either be set in the ' \ 'environment parameter or set globally using ' \ 'graphlab.aws.set_credentials(...).') aws_credentials = { 'aws_access_key_id': aws_access_key, 'aws_secret_access_key': aws_secret_key } # Validate 'name' value if not _name_checker.match(name): raise IOError('Names can only can only contain: a-z, A-Z and hyphens.') conn = _lb_connection(environment.region, **aws_credentials) for lb in conn.get_all_load_balancers(): if lb.name == name: raise IOError('There is already a load balancer with that name. Load balancer names' \ ' must be unique in their region. Please choose a different name.') # clone the environment, so not mutating existing environment environment = environment.clone() tracker = _mt._get_metric_tracker() tracker.track('deploy.predictive_service.create', value=1, properties={'num_hosts':environment.num_hosts, 'instance_type':environment.instance_type}) _logger.info("Launching Predictive Service with %d hosts, as specified by num_hosts parameter" % (environment.num_hosts)) # Add a tag for all EC2 instances to indicate they are related to this PredictiveService if not environment.tags: environment.tags = {} environment.tags.update({'Name':name, 'predictive_service':name}) # Set defaults values, if needed if not api_key: api_key = str(_random_uuid()) if not admin_key: admin_key = str(_random_uuid()) result = None env = None try: starttime = _datetime.datetime.now() _logger.info("Launching Predictive Service, with name: %s" % name) _logger.info("[Step 0/5]: Initializing S3 locations.") # Create the predictive service object. It writes init state to S3. result = _PredictiveService(name, state_path, description, api_key, aws_credentials) # Launch the environment. env = _Ec2PredictiveServiceEnvironment.launch(name, environment, state_path, admin_key, ssl_credentials, aws_credentials, started=starttime) # Attach the launched environment and write all service state to S3. result._environment = env result._save_state_to_s3() _logger.info("[Step 4/5]: Waiting for Load Balancer to put all instances into service.") while ((_datetime.datetime.now() - starttime).total_seconds() < _MAX_CREATE_TIMEOUT_SECS): # query status, verify all InService nodes = env.get_status(_show_errors = False) statuses = [] for node in nodes: statuses.append(node['state'] == 'InService') if all(statuses): _logger.info("Cluster is fully operational, [%d/%d] instances currently in service." % (statuses.count(True), len(statuses))) break else: _logger.info("Cluster not fully operational yet, [%d/%d] instances currently in service." % (statuses.count(True), len(statuses))) _time.sleep(15) else: _logger.error("Instances failed to be ready within 10 minutes. Tearing down.") raise RuntimeError("Cluster configuration not successful in time, timing out.") _logger.info("[Step 5/5]: Finalizing Configuration.") _gl.deploy._default_session.register(result) result.save() return result except Exception as e: # any exceptions we should gracefully terminate / tear down what we've created _logger.warning("Tearing down Predictive Service due to error launching") # default behavior deletes the log files in tear down. # To preserve the logs set GRAPHLAB_DEBUG in environment, and the logs will be preserved delete_logs = True if 'GRAPHLAB_DEBUG' in _os.environ: _logger.info("Preserving Log Files for debugging purposes") delete_logs = False if env: env.terminate(delete_logs) if result and delete_logs: _logger.info('Deleting model data.') try: _s3_recursive_delete(state_path, aws_credentials) except: _logger.error("Could not delete model data. Please manually delete data under: %s" % state_path) if isinstance(e,LicenseValidationException): # catch exception and print license check hint message here instead of raise _logger.info(e) else: # re-raise exception so object not returned etc raise
def load(s3_path, aws_access_key_id = None, aws_secret_access_key = None, override = True): ''' Loads a Predictive Service object from the s3_path. This will have the effect of instantiating an object locally, which contains all the relevant metadata associated with a Predictive Service. This is useful for getting a handle on a Predictive Service created from another machine, allowing you to query it and report its status, for example. Parameters ----------- s3_path : str S3 path used to manage state for the Predictive Service. This corresponds to the state path specified when the Predictive Service was initially created. aws_access_key_id : str The AWS Access Key to use to load the Predictive Service. If this is not specified, your system environment variables will be used. You may also use :py:func:`~graphlab.aws.set_credentials` to set the credentials. aws_secret_access_key : str The AWS secret Key to use to load the Predictive Service. If this is not specified, your system environment variables will be used. You may also use :py:func:`~graphlab.aws.set_credentials` to set the credentials. override: boolean, optional If override is True, the created predictive service from s3_path would override any local predictive service endpoint that has same name. False otherwise. Returns ------- out : :py:class:`~graphlab.deploy._predictive_service._predictive_service.PredictiveService` The Predictive Service handle that can be used to access and manage the deployed Predictive Service See Also -------- create Examples -------- >>> ps = graphlab.deploy.predictive_service.load('s3://mybucket/pred_services/name') >>> ps.get_status() ''' # Save the credentials. if bool(aws_access_key_id) != bool(aws_secret_access_key): raise IOError('Either both aws_access_key_id and aws_secret_access_key should be' \ ' specified or neither should be specified.') if not aws_access_key_id and not aws_secret_access_key: try: aws_access_key, aws_secret_key = _get_credentials() except: raise IOError('No AWS credentials set. Credentials must either be passed in,' \ ' or set globally using graphlab.aws.set_credentials(...).') aws_credentials = { 'aws_access_key_id': aws_access_key_id, 'aws_secret_access_key': aws_secret_access_key } s3_bucket_name, s3_key_name = _parse_s3_path(s3_path) config = _PredictiveService._get_s3_state_config(s3_bucket_name, s3_key_name + '/state.ini', aws_credentials) name = config.get(_PredictiveService._SERVICE_INFO_SECTION_NAME, 'Name') description = config.get(_PredictiveService._SERVICE_INFO_SECTION_NAME, 'Description') api_key = config.get(_PredictiveService._SERVICE_INFO_SECTION_NAME, 'API Key') result = _PredictiveService(name, s3_path, description, api_key, aws_credentials, _new_service = False) if not _gl.deploy._default_session.is_name_unique(name, typename="PredictiveService"): if override is True: # delete the object in the session before overriding _gl.deploy.predictive_services.delete(name) else: # found another predictive service or predictive service endpoint with the same name raise RuntimeError("Validation Error: Predictive Service already exists with the name '%s', please rename or delete the existing Predictive Service." % name) _mt._get_metric_tracker().track('deploy.predictive_service.load', value = 1, properties={'path': s3_path}) # register and save to session _gl.deploy._default_session.register(result) result.save() return result
def create(name, ec2_config, state_path, num_hosts = 1, description = None, api_key = None, admin_key = None, ssl_credentials = None, cors_origin = '', port = _PORT_DEFAULT_NUM): ''' Launch a Predictive Services cluster. This cluster can currently be launched on EC2 by specifying an EC2 environment. Parameters ---------- name : str The name of the Predictive Service that will be launched. This string can only contain: a-z, A-Z and hyphens. ec2_config : :class:`~graphlab.deploy.Ec2Config` Must be an EC2 Configuration object used for starting up EC2 host(s). This configuration should contain the instance type, region, aws credientials, and other information used to launch EC2 host(s). state_path : str S3 path used to manage state for the Predictive Service. This path can also be used to create the Predictive Service object on another computer. num_hosts : int, optional The number of EC2 host(s) to use for this Predictive Service. The default number of EC2 host(s) is 1. description : str, optional Description of this Predictive Service. api_key : str, optional An API key that client must included with requests. If an api_key is not specified, one will be auto generated. The API Key can be retrieved from the return PredictiveService object. admin_key : str, optional An API key used for control operations (i.e. anything other than client requests). If an admin_key is not specified, one will be auto generated. The API Key can be retrieved from the return PredictiveService object. ssl_credentials : tuple of len three, with types: str, str, bool. The first string is the path to the private key file. The second string is the path to public key certificate. The third denotes whether the certificates are self signed (and any client should not verify the certificate). These files must be in the precise format AWS expects. Such a private key and a self-signed certificate can be generated using openssl with following commands: >>> openssl genrsa 1024 > privatekey.pem >>> openssl req -new -key privatekey.pem -out csr.pem >>> openssl x509 -req -days 365 -in csr.pem -signkey privatekey.pem -out server.crt If a tuple is not given, requests will be served over HTTP rather than HTTPS (i.e. encryption will not be used). cors_origin : str, optional The string value to use as HTTP header Access-Control-Allow-Origin, in order to support Cross-Origin Resource Sharing as described in https://developer.mozilla.org/en-US/docs/Web/HTTP/Access_control_CORS. The default value is ''. If '' (empty String) is specified, CORS support will be disabled. If the string value is '*', CORS support will be enabled for all URIs. If the string value is 'https://dato.com', CORS support will be enabled for 'https://dato.com' only. port : int, optional The port the server listens to . The default port is 9005. Please do not specify 9006 or 19006. These are reserved for cache. Returns ------- out : :py:class:`~graphlab.deploy.PredictiveService` A Predictive Service object which can be used to manage the deployment. See Also -------- graphlab.deploy.predictive_service.load, graphlab.deploy.PredictiveService Examples -------- Let us start out by creating a single node Predictive Serivce on EC2. .. sourcecode:: python # create an Ec2Config for Predictive Service, with region, instance_type, # and aws credientials. ec2 = graphlab.deploy.Ec2Config(aws_access_key_id = 'ACCESS_KEY_ID', aws_secret_access_key = 'SECRET') # create a Predictive Service using a name, ec2 config, and # a directory to store the Predictive Service metadata and logs ps = graphlab.deploy.predictive_service.create('name', ec2, 's3://mybucket/pred_services/name') # get status of this Predictive Service ps.get_status() Now, lets create a multi-node Predictive Service by specifying the number of nodes needed in the EC2 environment variable .. sourcecode:: python # create an Ec2Config for Predictive Service, with region, instance_type, # and aws credientials. ec2 = graphlab.deploy.Ec2Config(aws_access_key_id = 'ACCESS_KEY_ID', aws_secret_access_key = 'SECRET') # create the Predictive Service with 3 nodes ps = graphlab.deploy.predictive_service.create('name_cluster', ec2_cluster, 's3://mybucket/pred_services/name_cluster', num_hosts = 3) # get status of this Predictive Service ps.get_status() # shut down this Predictive Service ps.terminate_service() ''' if num_hosts > _NODE_LAUNCH_LIMIT: raise ValueError("You cannot launch more than %d nodes at one time. " \ "If this limit is problematic, please contact " \ "*****@*****.**" % _NODE_LAUNCH_LIMIT) # Validate inputs for current session if _gl.deploy._default_session.exists(name, _PredictiveService._typename): # found another predictive service or predictive service endpoint with the same name raise ValueError("Validation Error: Predictive Service already exists with the name '%s', please rename or delete the existing Predictive Service." % name) # Validate Ec2 Config if not isinstance(ec2_config, _gl.deploy.Ec2Config): raise TypeError('Unsupported type given for ec2_config parameter. Must be an Ec2Config object.') # Save AWS config if(hasattr(ec2_config, 'aws_access_key_id') and hasattr(ec2_config, 'aws_secret_access_key')): aws_access_key = ec2_config.aws_access_key_id aws_secret_key = ec2_config.aws_secret_access_key else: try: aws_access_key, aws_secret_key = _get_credentials() except: raise IOError('No AWS credentials set. Credentials must either be set in the ' \ 'ec2_config parameter or set globally using ' \ 'graphlab.aws.set_credentials(...).') aws_credentials = { 'aws_access_key_id': aws_access_key, 'aws_secret_access_key': aws_secret_key } # Warn if specified bucket is in different region than specified in env. s3_bucket_name, _ = _file_util.parse_s3_path(state_path) __default_config_path = _os.path.join(_os.path.expanduser("~"), ".graphlab", "config") try: _file_util.upload_to_s3(__default_config_path, state_path + "/license", aws_credentials = aws_credentials, silent = True) region = _file_util.get_s3_bucket_region(s3_bucket_name, aws_credentials) except _S3ResponseError as e: _logger.error("Unable to connect to state_path's bucket; check your AWS credentials") raise if region != ec2_config.region: _logger.warn("The bucket in your state path is in a different region " \ "(%s) than the one specified in your environment (%s). " \ "AWS data transfer rates apply. Additionally, upload and " \ "download speeds may be slower than expected. If this is " \ "not what you intended, abort this operation or " \ "terminate the service upon its completion, then be sure " \ "that your environment and S3 bucket are in the same " \ "region before relaunching." % (region, ec2_config.region)) # Validate 'name' value if not _name_checker.match(name): raise ValueError('Predictive Service Name can only contain: a-z, A-Z and hyphens.') if len(name) > 32: raise ValueError("Predictive Service name can not be longer than 32 characters.") conn = _lb_connection(ec2_config.region, **aws_credentials) for lb in conn.get_all_load_balancers(): if lb.name == name: raise IOError('There is already a load balancer with that name. Load balancer names' \ ' must be unique in their region. Please choose a different name.') tracker = _mt._get_metric_tracker() tracker.track('deploy.predictive_service.create', value=1, properties={'num_hosts':num_hosts, 'instance_type':ec2_config.instance_type}) _logger.info("Launching Predictive Service with %d hosts, as specified by num_hosts parameter" % (num_hosts)) # Set defaults values, if needed if not api_key: api_key = str(_random_uuid()) if not admin_key: admin_key = str(_random_uuid()) result = None env = None try: starttime = _datetime.datetime.now() _logger.info("Launching Predictive Service, with name: %s" % name) _logger.info("[Step 0/5]: Initializing S3 locations.") # Create the predictive service object. It writes init state to S3. result = _PredictiveService(name, state_path, description, api_key, admin_key, aws_credentials, cors_origin = cors_origin, port = port) # Launch the environment. env = _Ec2PredictiveServiceEnvironment.launch(name, ec2_config, state_path, num_hosts, admin_key, ssl_credentials, aws_credentials, started=starttime, port = port) # Attach the launched environment and write all service state to S3. result._environment = env result._save_state() _logger.info("[Step 4/5]: Waiting for Load Balancer to put all instances into service.") while ((_datetime.datetime.now() - starttime).total_seconds() < _MAX_CREATE_TIMEOUT_SECS): # query status, verify all InService nodes = env.get_status(_show_errors = False) statuses = [] for node in nodes: statuses.append(node['state'] == 'InService') if all(statuses): _logger.info("Cluster is fully operational, [%d/%d] instances currently in service." % (statuses.count(True), len(statuses))) break else: _logger.info("Cluster not fully operational yet, [%d/%d] instances currently in service." % (statuses.count(True), len(statuses))) _time.sleep(15) else: _logger.error("Instances failed to be ready within 10 minutes. Tearing down.") raise RuntimeError("Cluster configuration not successful in time, timing out.") _logger.info("[Step 5/5]: Finalizing Configuration.") result.cache_enable(None, True) _gl.deploy._default_session.register(result) result.save() return result except Exception as e: # any exceptions we should gracefully terminate / tear down what we've created _logger.warning("Tearing down Predictive Service due to error launching") # default behavior deletes the log files in tear down. # To preserve the logs set GRAPHLAB_DEBUG in environment, and the logs will be preserved delete_logs = True if 'GRAPHLAB_DEBUG' in _os.environ: _logger.info("Preserving Log Files for debugging purposes") delete_logs = False if env: env.terminate(delete_logs) if result and delete_logs: _logger.info('Deleting model data.') try: _file_util.s3_recursive_delete(state_path, aws_credentials) except: _logger.error("Could not delete model data. Please manually delete data under: %s" % state_path) raise
def _load_imp(state_path, aws_access_key_id, aws_secret_access_key): ''' Internal implmentation of the load, used by both external facing load and by internal facing load (gl.deploy.predictive_services[name]) ''' aws_credentials = None if _file_util.is_s3_path(state_path): # Save the credentials. if bool(aws_access_key_id) != bool(aws_secret_access_key): raise IOError('Either both aws_access_key_id and aws_secret_access_key ' \ 'should be specified or neither should be specified.') if not aws_access_key_id and not aws_secret_access_key: try: aws_access_key_id, aws_secret_access_key = _get_credentials() except: raise IOError('No AWS credentials set. Credentials must either be ' \ 'passed in, or set globally using ' \ 'graphlab.aws.set_credentials(...).') aws_credentials = { 'aws_access_key_id': aws_access_key_id, 'aws_secret_access_key': aws_secret_access_key } elif (not _file_util.is_hdfs_path(state_path)) and (not _file_util.is_local_path(state_path)): raise ValueError("Invalid state path. Predictive Service only supports loading \ state path from S3, HDFS or Local file path.") config = _PredictiveServiceEnvironment._get_state_from_file(state_path, aws_credentials) name = config.get(_PredictiveService._SERVICE_INFO_SECTION_NAME, 'Name') description = config.get(_PredictiveService._SERVICE_INFO_SECTION_NAME, 'Description') api_key = config.get(_PredictiveService._SERVICE_INFO_SECTION_NAME, 'API Key') admin_key = config.get(_PredictiveService._ENVIRONMENT_SECTION_NAME, 'admin_key') # For backwards compatibility. Port used to be hard-coded as 9005 and does not # exist in the config. if (config.has_option(_PredictiveService._ENVIRONMENT_SECTION_NAME, 'port')): port = int(config.get(_PredictiveService._ENVIRONMENT_SECTION_NAME, 'port')) else: port = _PORT_DEFAULT_NUM global_cache_state = 'enabled' if _CACHE_STATE_SECTION_NAME_ in config.options(_PredictiveService._SERVICE_INFO_SECTION_NAME): global_cache_state = config.get(_PredictiveService._SERVICE_INFO_SECTION_NAME, _CACHE_STATE_SECTION_NAME_) cors_origin = '' if _CORS_ORIGIN_SECTION_NAME_ in config.options(_PredictiveService._SERVICE_INFO_SECTION_NAME): cors_origin = config.get(_PredictiveService._SERVICE_INFO_SECTION_NAME, _CORS_ORIGIN_SECTION_NAME_) system_config = _SystemConfig.from_config_parser( config, _PredictiveService._SYSTEM_SECTION_NAME) result = _PredictiveService(name, state_path, description, api_key, admin_key, aws_credentials, _new_service=False, cors_origin=cors_origin, global_cache_state=global_cache_state, system_config=system_config, port = port) # create environment environment_info = dict(config.items(_PredictiveService._ENVIRONMENT_SECTION_NAME)) if aws_credentials: environment_info['aws_credentials'] = aws_credentials result._environment = _predictive_service_environment_factory(environment_info) # get latest state result._get_latest_state() return result
def __init__(self, region = 'us-west-2', instance_type = _DEFAULT_INSTANCE_TYPE, aws_access_key_id = None, aws_secret_access_key = None, security_group = None, tags = None, cidr_ip = None, subnet_id = None, security_group_id = None): #We do not allow both security_group_name and security_group_id to be specified. if security_group is not None and security_group_id is not None: raise Exception("Specify either security_group or security_group_id but not Both.") # region if isinstance(region, str): if region not in _VALID_REGIONS: raise TypeError("%s is not a valid AWS region." % region) self.region = region else: raise TypeError('\'region\' must be a string.') # instance_type if isinstance(instance_type, str): # TODO: Need better instance type checking, like we do for region self.instance_type = instance_type else: raise TypeError('\'instance_type\' must be a string.') # aws keys if bool(aws_access_key_id) != bool(aws_secret_access_key): raise RuntimeError('If you specify a \'aws_access_key_id\' or a \'aws_secret_access_key\',' ' you must specify both \'aws_access_key_id\' and' ' \'aws_secret_access_key\'.') if aws_access_key_id and aws_secret_access_key: # aws keys where passed in if isinstance(aws_access_key_id, str) and isinstance(aws_secret_access_key, str): self.aws_access_key_id = aws_access_key_id self.aws_secret_access_key = aws_secret_access_key else: raise TypeError('\'aws_access_key_id\' and \'aws_secret_access_key\' must be string') else: # Try to get aws keys for the environment variables try: self.aws_access_key_id, self.aws_secret_access_key = _get_credentials() except: raise RuntimeError('AWS credentials must either be passed in or set as' ' environment variables.') # security_group if not security_group or isinstance(security_group, str): self.security_group = security_group else: raise TypeError('\'security_group\' must be a string.') # tags if not tags: self.tags = {} elif isinstance(tags, dict): self.tags = tags else: raise TypeError('\'tags\' must be a dict.') # cidr_ip if cidr_ip and not (isinstance(cidr_ip, str) or isinstance(cidr_ip, list)): raise TypeError('\'cidr_ip\' must be a string or a list of strings') else: self.cidr_ip = cidr_ip # subnet_id if not subnet_id or isinstance(subnet_id, str): self.subnet_id = subnet_id else: raise TypeError('\'subnet_id\' must be a string.') # security_group_id if not security_group_id or isinstance(security_group_id, str): self.security_group_id = security_group_id else: raise TypeError('\'security_group_id\' must be a string.')