def start(self, num_tolerable_ping_failures=3): properties = dict(product_key=self.product_key) _get_metric_tracker().track('engine-started', value=1, properties=properties, send_sys_info=True) _get_metric_tracker().track('engine-started-remote', value=1) # try to establish a connection to the server. (client_public_key, client_secret_key) = ('', '') if self.public_key != '': (client_public_key, client_secret_key) = get_public_secret_key_pair() try: c = Client([], self.server_addr, num_tolerable_ping_failures, public_key=client_public_key, secret_key=client_secret_key, server_public_key=self.public_key) if self.auth_token: c.add_auth_method_token(self.auth_token) c.start() finally: c.stop()
def launch(server_addr=None, server_bin=None, server_log=None, auth_token=None, server_public_key=''): """ Launch a connection to the graphlab server. The connection can be stopped by the `stop` function. Automatically spawns a local server, if no arguments provided or "server_bin" is specified. Notes ----- Only a single connection can exist at anytime. Prints warning if trying to launch while there is an active connection. Parameters ---------- server_addr : string The address of the server. server_bin : string The path to the server binary (local server only). server_log : string The path to the server log (local server only). server_public_key : string The server's libsodium public key, used for encryption. Default is no encryption. """ if is_connected(): __LOGGER__.warning( "Attempt to connect to a new server while still connected to a server." " Please stop the connection first by running 'graphlab.stop()' and try again.") return try: server_type = _get_server_type(server_addr) __LOGGER__.debug("Server type: %s" % server_type) except ValueError as e: __LOGGER__.error(e) _get_metric_tracker().track('server_launch.server_type_error', send_sys_info=True) return # Check that the unity_server binary exists if not hasattr(_DEFAULT_CONFIG, 'server_bin') and server_bin is None: __LOGGER__.error("Could not find a unity_server binary. Please try reinstalling.") raise AssertionError # Test that the unity_server binary works _verify_engine_binary(_DEFAULT_CONFIG.server_bin if server_bin is None else server_bin) # get and validate the product / registration key try: product_key = graphlab.product_key.get_product_key() except KeyError as k: __LOGGER__.error(k.message) # metric is tracked in product_key.py for different code paths return # check if there is product key if product_key is None: __LOGGER__.error(" ========================================\n" "GraphLab Create requires a license to use. To get a trial, non-commercial, " " or commercial license, visit https://dato.com/register.\n" "=================================================\n") _get_metric_tracker().track('server_launch.product_key_missing') return # get product key license info try: license_info = graphlab.product_key._get_license_info() except KeyError as k: __LOGGER__.error(k.message) # metric is tracked in product_key.py for different code paths return # product key check try: product_key_good = graphlab.product_key._is_product_key_valid(product_key, license_info) except RuntimeError as r: __LOGGER__.error("Fatal error. The unity_server process cannot be started. There may have been an " "issue during installation of graphlab-create. Please uninstall graphlab-create " "and reinstall it, looking for errors that may occur during installation. " "If the problem persists, please contact [email protected].") _get_metric_tracker().track('server_launch.unity_server_error', send_sys_info=True) return # verify product key is good if (not product_key_good): _get_metric_tracker().track('server_launch.product_key_invalid') return # construct a server server instance based on the server_type if (server_type == LOCAL_SERVER_TYPE): server = LocalServer(server_addr, server_bin, server_log, product_key) elif (server_type == REMOTE_SERVER_TYPE): server = RemoteServer(server_addr, auth_token, product_key, public_key=server_public_key) else: raise ValueError('Invalid server type: %s' % server_type) # start the server try: server.start() except Exception as e: __LOGGER__.error('Cannot start server: %s' % e) server.try_stop() return # start the client (public_key, secret_key) = ('', '') if server_public_key != '': (public_key, secret_key) = get_public_secret_key_pair() try: num_tolerable_ping_failures = 4294967295 client = Client([], server.get_server_addr(), num_tolerable_ping_failures, public_key=public_key, secret_key=secret_key, server_public_key=server_public_key) if hasattr(server, 'proc') and hasattr(server.proc, 'pid'): client.set_server_alive_watch_pid(server.proc.pid) if(auth_token is not None): client.add_auth_method_token(auth_token) client.start() except Exception as e: __LOGGER__.error("Cannot start client: %s" % e) if (client): client.stop() return _assign_server_and_client(server, client) assert is_connected()
def launch(server_addr=None, server_bin=None, server_log=None, auth_token=None, server_public_key=''): """ Launch a connection to the graphlab server. The connection can be stopped by the `stop` function. Automatically spawns a local server, if no arguments provided or "server_bin" is specified. Notes ----- Only a single connection can exist at anytime. Prints warning if trying to launch while there is an active connection. Parameters ---------- server_addr : string The address of the server. server_bin : string The path to the server binary (local server only). server_log : string The path to the server log (local server only). server_public_key : string The server's libsodium public key, used for encryption. Default is no encryption. """ if is_connected(): __LOGGER__.warning( "Attempt to connect to a new server while still connected to a server." " Please stop the connection first by running 'graphlab.stop()' and try again.") return try: server_type = _get_server_type(server_addr) __LOGGER__.debug("Server type: %s" % server_type) except ValueError as e: __LOGGER__.error(e) _get_metric_tracker().track('server_launch.server_type_error', send_sys_info=True) return # get and validate the product / registration key try: product_key = graphlab.product_key.get_product_key() except KeyError as k: __LOGGER__.error(k.message) # metric is tracked in product_key.py for different code paths return # product key check. This may fail if there are issues running # the unity_server binary try: product_key_good = graphlab.product_key.is_product_key_valid(product_key) except RuntimeError as r: __LOGGER__.error("Fatal error. The unity_server process cannot be started. There may have been an " "issue during installation of graphlab-create. Please uninstall graphlab-create " "and reinstall it, looking for errors that may occur during installation. " "If the problem persists, please contact [email protected].") _get_metric_tracker().track('server_launch.unity_server_error', send_sys_info=True) return # verify product key is good if (not product_key_good): if product_key is None: __LOGGER__.error("No product key found. Please configure your product key by setting the [%s] section with" " '%s' key in %s or by setting the environment variable GRAPHLAB_PRODUCT_KEY to the product key." " If you do not have a product key, please register for one at https://dato.com/register." % ( graphlab.product_key.__section, graphlab.product_key.__key, graphlab.product_key.__default_config_path)) _get_metric_tracker().track('server_launch.product_key_missing') else: __LOGGER__.error("Product Key validation failed, please confirm your product key is correct. " "If you believe this key to be valid, please contact [email protected]") _get_metric_tracker().track('server_launch.product_key_invalid') return # construct a server server instance based on the server_type if (server_type == LOCAL_SERVER_TYPE): server = LocalServer(server_addr, server_bin, server_log, product_key) elif (server_type == REMOTE_SERVER_TYPE): server = RemoteServer(server_addr, auth_token, product_key, public_key=server_public_key) else: raise ValueError('Invalid server type: %s' % server_type) # start the server try: server.start() except Exception as e: __LOGGER__.error('Cannot start server: %s' % e) server.try_stop() return # start the client (public_key, secret_key) = ('', '') if server_public_key != '': (public_key, secret_key) = get_public_secret_key_pair() try: num_tolerable_ping_failures = 3 client = Client([], server.get_server_addr(), num_tolerable_ping_failures, public_key=public_key, secret_key=secret_key, server_public_key=server_public_key) if(auth_token is not None): client.add_auth_method_token(auth_token) client.start() except Exception as e: __LOGGER__.error("Cannot start client: %s" % e) if (client): client.stop() return _assign_server_and_client(server, client) assert is_connected()
def start(self, num_tolerable_ping_failures=4294967295): properties = dict(product_key=self.product_key) _get_metric_tracker().track("engine-started", value=1, properties=properties, send_sys_info=True) _get_metric_tracker().track("engine-started-local", value=1) arglist = [self.server_bin, self.server_addr] if self.product_key: arglist.append("--product_key=%s" % self.product_key) if self.auth_token: arglist.append("--auth_token=%s" % self.auth_token) if self.secret_key != "": arglist.append("--secret_key=%s" % self.secret_key) arglist.append("--log_file=%s" % self.unity_log) arglist.append("--log_rotation_interval=%d" % default_local_conf.log_rotation_interval) arglist.append("--log_rotation_truncate=%d" % default_local_conf.log_rotation_truncate) self._validate_protocol_and_address() if sys.platform == "win32": self.unity_log += ".0" # Start a local server as a child process. try: FNULL = open(os.devnull, "w") if sys.platform == "win32": self.proc = subprocess.Popen( arglist, env=_sys_util.make_unity_server_env(), stdin=subprocess.PIPE, stdout=FNULL, stderr=None, bufsize=-1, ) # preexec_fn not supported on windows else: self.proc = subprocess.Popen( arglist, env=_sys_util.make_unity_server_env(), stdin=subprocess.PIPE, stdout=FNULL, stderr=None, bufsize=-1, preexec_fn=lambda: os.setpgrp(), ) # do not forward signal except OSError as e: raise RuntimeError('Invalid server binary "%s": %s' % (self.server_bin, str(e))) except KeyError as e: raise RuntimeError(e.message) # update the default server_addr if self.server_addr == "default": self.server_addr = "ipc:///tmp/graphlab_server-%s" % (self.proc.pid) self.logger.info( "Start server at: " + self.server_addr + " - " "Server binary: " + self.server_bin + " - " "Server log: " + self.unity_log ) # try to establish a connection to the server. (client_public_key, client_secret_key) = ("", "") if self.public_key != "" and self.secret_key != "": (client_public_key, client_secret_key) = get_public_secret_key_pair() max_retry = 5 retry = 0 server_alive = True while retry < max_retry: retry += 1 # Make sure the server process is still alive if self.proc.poll() is not None: server_alive = False # OK, server is alive, try create a client and connect if server_alive: try: c = Client( [], self.server_addr, num_tolerable_ping_failures, public_key=client_public_key, secret_key=client_secret_key, server_public_key=self.public_key, ) if self.auth_token: c.add_auth_method_token(self.auth_token) c.set_server_alive_watch_pid(self.proc.pid) c.start() # everything works, break out of the retry loop break except Exception as e: self.logger.error("Try connecting to server. Error: %s. Retry = %d" % (str(e), retry)) time.sleep(0.5) finally: c.stop() # Server process terminated, raise exception and get the return code else: retcode = self.proc.returncode self.proc = None self.logger.error("Cannot start server process. Return code: %d" % retcode) raise RuntimeError("Cannot start server process. Return code: %d" % retcode) if retry == max_retry: self.logger.error("Cannot connect to server. Exceeded max retry (%d)." % max_retry) raise RuntimeError("Cannot connect to server. Exceeded max retry (%d)." % max_retry) import threading def server_wait(): self.proc.wait() self.wait_thread = threading.Thread(target=server_wait) self.wait_thread.setDaemon(True) self.wait_thread.start()
def launch_EC2(instance_type=DEFAULT_INSTANCE_TYPE, region=None, CIDR_rule=None, auth_token=None, security_group=None, tags=None, use_secure_connection=True): """ Launches an EC2 instance. All subsequent GraphLab operations will be executed on that host. Prior to calling this function, AWS credentials must be set as environment variables (see :py:func:`~graphlab.aws.set_credentials`). Parameters ---------- instance_type : string, optional The EC2 instance type to launch. We support `all instance types <http://aws.amazon.com/ec2/instance-types/#Instance_Types>`_ that are not micros, smalls or mediums. region : string, optional The `AWS region <http://docs.aws.amazon.com/general/latest/gr/rande.html#ec2_region>`_ in which to launch your instance. Default is 'us-west-2'. CIDR_rule : string or list[string], optional The Classless Inter-Domain Routing rule(s) to use for the instance. Useful for restricting the IP Address Range for a client. Default is no restriction. If you specify CIDR_rule(s), you must also specify a security group to use. auth_token : string, optional The Authentication Token to be used by the instance. By default a randomly generated token is used. security_group : string, optional The name of the security group for the EC2 instance to use. tags : dict, optional A dictionary containing the name/value tag pairs to be assigned to the instance. If you want to create only a tag name, the value for that tag should be the empty string (i.e. ''). In addition to these specified tags, a 'GraphLab' tag will also be assigned. use_secure_connection : bool, optional Determine whether the communication, between your computer and the EC2 instance, should be encrypted. See Also -------- terminate_EC2 Examples -------- >>> # Launch a general purpose xlarge host in Oregon. >>> graphlab.aws.launch_EC2() >>> # Launch an xlarge compute optimized host in the US East Region. >>> graphlab.aws.launch_EC2(instance_type='c3.xlarge', region='us-east-1') """ # Check existing connection if glconnect.is_connected(): __LOGGER__.warning('You have GraphLab objects instantiated on the current server. If you want to \n' 'launch a new server, you must first stop the current server connection by running: \n' '\'graphlab.connect.main.stop()\'. Be warned: you will lose these instantiated objects.') return assert not glconnect.is_connected() (server_public_key, server_secret_key, client_public_key, client_secret_key) = ("", "", "", "") if use_secure_connection: (server_public_key, server_secret_key) = get_public_secret_key_pair() (client_public_key, client_secret_key) = get_public_secret_key_pair() __LOGGER__.debug('Launching server with public key: %s' % server_public_key) try: server = _Ec2GraphLabServer(instance_type, region, CIDR_rule, auth_token, security_group, tags, server_public_key, server_secret_key) try: # Once the EC2 instance starts up, the client still need to wait for graphlab engine to start up. # So allow for a larger than normal number of ping failures. This is especially important when launching # in other AWS regions (since longer latency and longer download from S3 for engine binary) server.start(num_tolerable_ping_failures=120) except Exception as e: __LOGGER__.error("Unable to successfully connect to GraphLab Server on EC2 instance: '%s'." " Please check AWS Console to make sure any EC2 instances launched have" " been terminated." % e) server.stop() raise e except LicenseValidationException as e: # catch exception and print license check hint message here instead of raise __LOGGER__.info(e) return # Create the client num_tolerable_ping_failures = 3 client = Client([], server.get_server_addr(), num_tolerable_ping_failures, public_key=client_public_key, secret_key=client_secret_key, server_public_key=server_public_key) auth_token = server.get_auth_token() if auth_token: client.add_auth_method_token(auth_token) client.start() glconnect._assign_server_and_client(server, client)
def start(self, num_tolerable_ping_failures=3): properties = dict(product_key=self.product_key) _get_metric_tracker().track('engine-started', value=1, properties=properties, send_sys_info=True) _get_metric_tracker().track('engine-started-local', value=1) product_key_arg = "--product_key=%s" % self.product_key arglist = [self.server_bin, self.server_addr, product_key_arg] if (self.auth_token): arglist.append("--auth_token=%s" % self.auth_token) if self.secret_key != '': arglist.append("--secret_key=%s" % self.secret_key) arglist.append("--log_file=%s" % self.unity_log) arglist.append("--log_rotation_interval=%d" % default_local_conf.log_rotation_interval) arglist.append("--log_rotation_truncate=%d" % default_local_conf.log_rotation_truncate) # Start a local server as a child process. if self.server_addr == 'default': protocol = 'ipc' else: protocol = _get_server_addr_protocol(self.server_addr) if (protocol is "ipc" and os.path.exists(self.server_addr[6:])): raise RuntimeError( 'Cannot start local server: local address %s already exists' % self.server_addr) try: FNULL = open(os.devnull, 'w') self.proc = subprocess.Popen( arglist, env=_sys_util.make_unity_server_env(), stdin=subprocess.PIPE, stdout=FNULL, stderr=subprocess.STDOUT, bufsize=-1, preexec_fn=lambda: os.setpgrp()) # do not forward signal except OSError as e: raise RuntimeError('Invalid server binary \"%s\": %s' % (self.server_bin, str(e))) except KeyError as e: raise RuntimeError(e.message) # update the default server_addr if (self.server_addr == 'default'): self.server_addr = 'ipc:///tmp/graphlab_server-%s' % ( self.proc.pid) # sleep one sec and make sure the server is running time.sleep(1) if (self.proc.poll() is not None): self.proc = None raise RuntimeError('Unable to start local server.') else: self.logger.info('Start server at: ' + self.server_addr + " - " 'Server binary: ' + self.server_bin + " - " 'Server log: ' + self.unity_log) # try to establish a connection to the server. (client_public_key, client_secret_key) = ('', '') if (self.public_key != '' and self.secret_key != ''): (client_public_key, client_secret_key) = get_public_secret_key_pair() max_retry = 3 retry = 0 while retry < max_retry: try: c = Client([], self.server_addr, num_tolerable_ping_failures, public_key=client_public_key, secret_key=client_secret_key, server_public_key=self.public_key) if self.auth_token: c.add_auth_method_token(self.auth_token) c.start() break except Exception as e: retry = retry + 1 time.sleep(0.5) if retry == max_retry: raise e finally: c.stop()
def launch_EC2(instance_type=DEFAULT_INSTANCE_TYPE, region=None, CIDR_rule=None, auth_token=None, security_group=None, tags=None, use_secure_connection=True): """ Launches an EC2 instance. All subsequent GraphLab operations will be executed on that host. Prior to calling this function, AWS credentials must be set as environment variables (see :py:func:`~graphlab.aws.set_credentials`). Parameters ---------- instance_type : string, optional The EC2 instance type to launch. We support `all instance types <http://aws.amazon.com/ec2/instance-types/#Instance_Types>`_ that are not micros, smalls or mediums. region : string, optional The `AWS region <http://docs.aws.amazon.com/general/latest/gr/rande.html#ec2_region>`_ in which to launch your instance. Default is 'us-west-2'. CIDR_rule : string or list[string], optional The Classless Inter-Domain Routing rule(s) to use for the instance. Useful for restricting the IP Address Range for a client. Default is no restriction. If you specify CIDR_rule(s), you must also specify a security group to use. auth_token : string, optional The Authentication Token to be used by the instance. By default a randomly generated token is used. security_group : string, optional The name of the security group for the EC2 instance to use. tags : dict, optional A dictionary containing the name/value tag pairs to be assigned to the instance. If you want to create only a tag name, the value for that tag should be the empty string (i.e. ''). In addition to these specified tags, a 'GraphLab' tag will also be assigned. use_secure_connection : bool, optional Determine whether the communication, between your computer and the EC2 instance, should be encrypted. See Also -------- terminate_EC2 Examples -------- >>> # Launch a general purpose xlarge host in Oregon. >>> graphlab.aws.launch_EC2() >>> # Launch an xlarge compute optimized host in the US East Region. >>> graphlab.aws.launch_EC2(instance_type='c3.xlarge', region='us-east-1') """ # Check existing connection if glconnect.is_connected(): __LOGGER__.error( 'You have GraphLab objects instantiated on the current server. If you want to \n' 'launch a new server, you must first stop the current server connection by running: \n' '\'graphlab.connect.main.stop()\'. Be warned: you will lose these instantiated objects.' ) return assert not glconnect.is_connected() (server_public_key, server_secret_key, client_public_key, client_secret_key) = ("", "", "", "") if use_secure_connection: (server_public_key, server_secret_key) = get_public_secret_key_pair() (client_public_key, client_secret_key) = get_public_secret_key_pair() __LOGGER__.debug('Launching server with public key: %s' % server_public_key) try: server = _Ec2GraphLabServer(instance_type, region, CIDR_rule, auth_token, security_group, tags, server_public_key, server_secret_key) try: # Once the EC2 instance starts up, the client still need to wait for graphlab engine to start up. # So allow for a larger than normal number of ping failures. This is especially important when launching # in other AWS regions (since longer latency and longer download from S3 for engine binary) server.start(num_tolerable_ping_failures=120) except Exception as e: __LOGGER__.error( "Unable to successfully connect to GraphLab Server on EC2 instance: '%s'." " Please check AWS Console to make sure any EC2 instances launched have" " been terminated." % e) server.stop() raise e except LicenseValidationException as e: # catch exception and print license check hint message here instead of raise __LOGGER__.info(e) return # Create the client num_tolerable_ping_failures = 3 client = Client([], server.get_server_addr(), num_tolerable_ping_failures, public_key=client_public_key, secret_key=client_secret_key, server_public_key=server_public_key) auth_token = server.get_auth_token() if auth_token: client.add_auth_method_token(auth_token) client.start() glconnect._assign_server_and_client(server, client)
def start(self, num_tolerable_ping_failures=3): properties = dict(product_key=self.product_key) _get_metric_tracker().track('engine-started', value=1, properties=properties, send_sys_info=True) _get_metric_tracker().track('engine-started-local', value=1) arglist = [self.server_bin, self.server_addr] if self.product_key: arglist.append("--product_key=%s" % self.product_key) if (self.auth_token): arglist.append("--auth_token=%s" % self.auth_token) if self.secret_key != '': arglist.append("--secret_key=%s" % self.secret_key) arglist.append("--log_file=%s" % self.unity_log) arglist.append("--log_rotation_interval=%d" % default_local_conf.log_rotation_interval) arglist.append("--log_rotation_truncate=%d" % default_local_conf.log_rotation_truncate) # Start a local server as a child process. if self.server_addr == 'default': protocol = 'ipc' else: protocol = _get_server_addr_protocol(self.server_addr) if (protocol is "ipc" and os.path.exists(self.server_addr[6:])): raise RuntimeError('Cannot start local server: local address %s already exists' % self.server_addr) try: FNULL = open(os.devnull, 'w') self.proc = subprocess.Popen(arglist, env=_sys_util.make_unity_server_env(), stdin=subprocess.PIPE, stdout=FNULL, stderr=subprocess.STDOUT, bufsize=-1, preexec_fn=lambda: os.setpgrp()) # do not forward signal except OSError as e: raise RuntimeError('Invalid server binary \"%s\": %s' % (self.server_bin, str(e))) except KeyError as e: raise RuntimeError(e.message) # update the default server_addr if (self.server_addr == 'default'): self.server_addr = 'ipc:///tmp/graphlab_server-%s' % (self.proc.pid) # sleep one sec and make sure the server is running time.sleep(1) if (self.proc.poll() is not None): self.proc = None raise RuntimeError('Unable to start local server.') else: self.logger.info('Start server at: ' + self.server_addr + " - " 'Server binary: ' + self.server_bin + " - " 'Server log: ' + self.unity_log) # try to establish a connection to the server. (client_public_key, client_secret_key) = ('', '') if (self.public_key != '' and self.secret_key != ''): (client_public_key, client_secret_key) = get_public_secret_key_pair() max_retry = 3 retry = 0 while retry < max_retry: try: c = Client([], self.server_addr, num_tolerable_ping_failures, public_key=client_public_key, secret_key=client_secret_key, server_public_key=self.public_key) if self.auth_token: c.add_auth_method_token(self.auth_token) c.start() break except Exception as e: retry = retry + 1 time.sleep(0.5) if retry == max_retry: raise e finally: c.stop()