def test_launch_to_ipc(self):
ipc_addr = 'ipc://' + tempfile.NamedTemporaryFile().name
auth_token = 'graphlab_awesome'
ipc_server = create_server(ipc_addr, auth_token)
ipc_server.start()
#default local launch
glconnect.launch()
self.assertTrue(glconnect.is_connected())
glconnect.stop()
self.assertFalse(glconnect.is_connected())
#launch with remote server ipc address
glconnect.launch(ipc_addr, auth_token=auth_token)
self.assertTrue(glconnect.is_connected())
self.assertTrue(isinstance(glconnect.get_server(),
server.RemoteServer))
glconnect.stop()
self.assertFalse(glconnect.is_connected())
#launch with remote server addr, and server_bin(ignored)
glconnect.launch(ipc_addr,
os.getenv("GRAPHLAB_UNITY"),
auth_token=auth_token)
self.assertTrue(glconnect.is_connected())
self.assertTrue(isinstance(glconnect.get_server(),
server.RemoteServer))
glconnect.stop()
self.assertFalse(glconnect.is_connected())
ipc_server.stop()
def test_launch_to_ipc(self):
ipc_addr = 'ipc://' + tempfile.NamedTemporaryFile().name
auth_token = 'graphlab_awesome'
ipc_server = create_server(ipc_addr, auth_token)
ipc_server.start()
#default local launch
glconnect.launch()
self.assertTrue(glconnect.is_connected())
glconnect.stop()
self.assertFalse(glconnect.is_connected())
#launch with remote server ipc address
glconnect.launch(ipc_addr, auth_token=auth_token)
self.assertTrue(glconnect.is_connected())
self.assertTrue(isinstance(glconnect.get_server(), server.RemoteServer))
glconnect.stop()
self.assertFalse(glconnect.is_connected())
#launch with remote server addr, and server_bin(ignored)
glconnect.launch(ipc_addr, os.getenv("GRAPHLAB_UNITY"), auth_token=auth_token)
self.assertTrue(glconnect.is_connected())
self.assertTrue(isinstance(glconnect.get_server(), server.RemoteServer))
glconnect.stop()
self.assertFalse(glconnect.is_connected())
ipc_server.stop()
def status():
"""
Returns the status of any running EC2 instance associated with the current
python session.
Examples
--------
>>> # With an EC2 instance running.
>>> graphlab.aws.status()
u'running'
>>> # Without an EC2 instance running.
>>> graphlab.aws.status()
>>> 'No instance running.'
"""
server = glconnect.get_server()
if server is None or not isinstance(server, _Ec2GraphLabServer):
return 'No instance running.'
aws_connection = boto.ec2.connect_to_region(server.region)
status = aws_connection.get_all_instance_status(instance_ids=[server.instance_id])
__LOGGER__.debug("EC2 status returned: %s" % status)
if(not(len(status) == 1)):
__LOGGER__.error('Invalid response from EC2. Unable to determine EC2 status. Please consult '
'the AWS EC2 Console.')
return status[0].state_name
def status():
"""
Returns the status of any running EC2 instance associated with the current
python session.
Examples
--------
>>> # With an EC2 instance running.
>>> graphlab.aws.status()
u'running'
>>> # Without an EC2 instance running.
>>> graphlab.aws.status()
>>> 'No instance running.'
"""
server = glconnect.get_server()
if server is None or not isinstance(server, _Ec2GraphLabServer):
return 'No instance running.'
aws_connection = boto.ec2.connect_to_region(server.region)
status = aws_connection.get_all_instance_status(
instance_ids=[server.instance_id])
__LOGGER__.debug("EC2 status returned: %s" % status)
if (not (len(status) == 1)):
__LOGGER__.error(
'Invalid response from EC2. Unable to determine EC2 status. Please consult '
'the AWS EC2 Console.')
return status[0].state_name
def test_launch_to_tcp(self):
auth_token = 'graphlab_awesome'
tcp_server = start_test_tcp_server(auth_token)
#launch with remote server tcp address
glconnect.launch(tcp_server.get_server_addr(), auth_token=auth_token)
self.assertTrue(glconnect.is_connected())
self.assertTrue(isinstance(glconnect.get_server(), server.RemoteServer))
glconnect.stop()
self.assertFalse(glconnect.is_connected())
tcp_server.stop()
def test_launch_to_tcp(self):
auth_token = 'graphlab_awesome'
tcp_server = start_test_tcp_server(auth_token)
#launch with remote server tcp address
glconnect.launch(tcp_server.get_server_addr(), auth_token=auth_token)
self.assertTrue(glconnect.is_connected())
self.assertTrue(isinstance(glconnect.get_server(),
server.RemoteServer))
glconnect.stop()
self.assertFalse(glconnect.is_connected())
tcp_server.stop()
def terminate_EC2():
"""
Terminates the EC2 instance associated with the current python session.
Examples
--------
>>> graphlab.aws.terminate_EC2()
"""
server = glconnect.get_server()
if server is not None and isinstance(server, _Ec2GraphLabServer):
__LOGGER__.debug("Stopping EC2 instance.")
glconnect.stop()
else:
__LOGGER__.info("No EC2 instance to stop.")
def terminate_EC2():
"""
Terminates the EC2 instance associated with the current python session.
Examples
--------
>>> graphlab.aws.terminate_EC2()
"""
server = glconnect.get_server()
if server is not None and isinstance(server, _Ec2GraphLabServer):
__LOGGER__.debug("Stopping EC2 instance.")
glconnect.stop()
else:
__LOGGER__.info("No EC2 instance to stop.")
def _make_internal_url(url):
"""
Takes a user input url string and translates into url relative to the server process.
- URL to a local location begins with "local://" or has no "*://" modifier.
If the server is local, returns the absolute path of the url.
For example: "local:///tmp/foo" -> "/tmp/foo" and "./foo" -> os.path.abspath("./foo").
If the server is not local, raise NotImplementedError.
- URL to a server location begins with "remote://".
Returns the absolute path after the "remote://" modifier.
For example: "remote:///tmp/foo" -> "/tmp/foo".
- URL to a s3 location begins with "s3://":
Returns the s3 URL with credentials filled in using graphlab.aws.get_aws_credential().
For example: "s3://mybucket/foo" -> "s3://$AWS_ACCESS_KEY_ID:$AWS_SECRET_ACCESS_KEY:mybucket/foo".
- URL to other remote locations, e.g. http://, will remain as is.
- Expands ~ to $HOME
Parameters
----------
string
A URL (as described above).
Raises
------
ValueError
If a bad url is provided.
"""
if not url:
raise ValueError('Invalid url: %s' % url)
# The final file path on server.
path_on_server = None
# Try to split the url into (protocol, path).
urlsplit = url.split("://")
if len(urlsplit) == 2:
protocol, path = urlsplit
if not path:
raise ValueError('Invalid url: %s' % url)
if protocol in ['http', 'https']:
# protocol is a remote url not on server, just return
return url
elif protocol == 'hdfs':
if isinstance(_glconnect.get_server(), _server.LocalServer
) and not _sys_util.get_hadoop_class_path():
raise ValueError(
"HDFS URL is not supported because Hadoop not found. Please make hadoop available from PATH or set the environment variable HADOOP_HOME and try again."
)
else:
return url
elif protocol == 's3':
return _try_inject_s3_credentials(url)
elif protocol == 'remote':
# url for files on the server
path_on_server = path
elif protocol == 'local':
# url for files on local client, check if we are connecting to local server
if (isinstance(_glconnect.get_server(), _server.LocalServer)):
path_on_server = path
else:
raise ValueError(
'Cannot use local URL when connecting to a remote server.')
else:
raise ValueError(
'Invalid url protocol %s. Supported url protocols are: remote://, local://, s3://, https:// and hdfs://'
% protocol)
elif len(urlsplit) == 1:
# expand ~ to $HOME
url = _os.path.expanduser(url)
# url for files on local client, check if we are connecting to local server
if (isinstance(_glconnect.get_server(), _server.LocalServer)):
path_on_server = url
else:
raise ValueError(
'Cannot use local URL when connecting to a remote server.')
else:
raise ValueError('Invalid url: %s' % url)
if path_on_server:
return _os.path.abspath(_os.path.expanduser(path_on_server))
else:
raise ValueError('Invalid url: %s' % url)
def run(toolkit_name, options, verbose=True, show_progress=False):
"""
Internal function to execute toolkit on the graphlab server.
Parameters
----------
toolkit_name : string
The name of the toolkit.
options : dict
A map containing the required input for the toolkit function,
for example: {'graph': g, 'reset_prob': 0.15}.
verbose : bool
If true, enable progress log from server.
show_progress : bool
If true, display progress plot.
Returns
-------
out : dict
The toolkit specific model parameters.
Raises
------
RuntimeError
Raises RuntimeError if the server fail executing the toolkit.
"""
unity = glconnect.get_unity()
if (not verbose):
glconnect.get_client().set_log_progress(False)
# spawn progress threads
server_addr = glconnect.get_server().get_server_addr()
splits = server_addr.split(':')
protocol = splits[0]
hostname = splits[1].split('//')[1]
try:
start_time = time.time()
(success, message, params) = unity.run_toolkit(toolkit_name, options)
end_time = time.time()
except:
raise
if (len(message) > 0):
logging.getLogger(__name__).error("Toolkit error: " + message)
track_props = {}
track_props['success'] = success
if success:
track_props['runtime'] = end_time - start_time
else:
if (len(message) > 0):
track_props['message'] = message
metric_name = 'toolkit.%s.executed' % (toolkit_name)
_get_metric_tracker().track(metric_name, value=1, properties=track_props, send_sys_info=False)
# set the verbose level back to default
glconnect.get_client().set_log_progress(True)
if success:
return params
else:
raise ToolkitError(str(message))
def _make_internal_url(url):
"""
Takes a user input url string and translates into url relative to the server process.
- URL to a local location begins with "local://" or has no "*://" modifier.
If the server is local, returns the absolute path of the url.
For example: "local:///tmp/foo" -> "/tmp/foo" and "./foo" -> os.path.abspath("./foo").
If the server is not local, raise NotImplementedError.
- URL to a server location begins with "remote://".
Returns the absolute path after the "remote://" modifier.
For example: "remote:///tmp/foo" -> "/tmp/foo".
- URL to a s3 location begins with "s3://":
Returns the s3 URL with credentials filled in using graphlab.aws.get_aws_credential().
For example: "s3://mybucket/foo" -> "s3://$AWS_ACCESS_KEY_ID:$AWS_SECRET_ACCESS_KEY:mybucket/foo".
- URL to other remote locations, e.g. http://, will remain as is.
- Expands ~ to $HOME
Parameters
----------
string
A URL (as described above).
Raises
------
ValueError
If a bad url is provided.
"""
if not url:
raise ValueError('Invalid url: %s' % url)
# The final file path on server.
path_on_server = None
# Try to split the url into (protocol, path).
urlsplit = url.split("://")
if len(urlsplit) == 2:
protocol, path = urlsplit
if not path:
raise ValueError('Invalid url: %s' % url)
if protocol in ['http', 'https']:
# protocol is a remote url not on server, just return
return url
elif protocol == 'hdfs':
if isinstance(_glconnect.get_server(), _server.LocalServer) and not _sys_util.get_hadoop_class_path():
raise ValueError("HDFS URL is not supported because Hadoop not found. Please make hadoop available from PATH or set the environment variable HADOOP_HOME and try again.")
else:
return url
elif protocol == 's3':
return _try_inject_s3_credentials(url)
elif protocol == 'remote':
# url for files on the server
path_on_server = path
elif protocol == 'local':
# url for files on local client, check if we are connecting to local server
if (isinstance(_glconnect.get_server(), _server.LocalServer)):
path_on_server = path
else:
raise ValueError('Cannot use local URL when connecting to a remote server.')
else:
raise ValueError('Invalid url protocol %s. Supported url protocols are: remote://, local://, s3://, https:// and hdfs://' % protocol)
elif len(urlsplit) == 1:
# expand ~ to $HOME
url = _os.path.expanduser(url)
# url for files on local client, check if we are connecting to local server
if (isinstance(_glconnect.get_server(), _server.LocalServer)):
path_on_server = url
else:
raise ValueError('Cannot use local URL when connecting to a remote server.')
else:
raise ValueError('Invalid url: %s' % url)
if path_on_server:
return _os.path.abspath(_os.path.expanduser(path_on_server))
else:
raise ValueError('Invalid url: %s' % url)