def _deserialize_output(self, task):
"""
Deserialize the output from a task.
Parameters
----------
Task definition of interest.
Returns
-------
The output of the run-time task associated with the task definition.
"""
filepath = self._task_output_paths[task]
non_hdfs_file_path = filepath
# Unpickler has no support for passing in additional HADOOP_CONF_DIR
# so we download HDFS folder first before calling to unpickler
if _file_util.is_hdfs_path(filepath):
non_hdfs_file_path = _make_temp_directory("job_output_")
_file_util.download_from_hdfs(filepath, non_hdfs_file_path,
hadoop_conf_dir=self.environment.hadoop_conf_dir, is_dir = True)
unpickler = gl_pickle.GLUnpickler(non_hdfs_file_path)
# We cannot delete this temporary file path becaue SFrame lazily load
# the content from disk. But the temporary folder will be removed
# eventually when the python session goes away
return unpickler.load()
def _deserialize_output(self, task):
"""
Deserialize the output from a task.
Parameters
----------
Task definition of interest.
Returns
-------
The output of the run-time task associated with the task definition.
"""
filepath = self._task_output_paths[task]
non_hdfs_file_path = filepath
# Unpickler has no support for passing in additional HADOOP_CONF_DIR
# so we download HDFS folder first before calling to unpickler
if _file_util.is_hdfs_path(filepath):
non_hdfs_file_path = _make_temp_directory("job_output_")
_file_util.download_from_hdfs(filepath, non_hdfs_file_path,
hadoop_conf_dir=self.environment.hadoop_conf_dir, is_dir = True)
unpickler = gl_pickle.GLUnpickler(non_hdfs_file_path)
# We cannot delete this temporary file path becaue SFrame lazily load
# the content from disk. But the temporary folder will be removed
# eventually when the python session goes away
return unpickler.load()
def _read_commander_init_status_file(self):
commander_file_path = self._get_commander_file_path()
local_file_name = _tempfile.mktemp(prefix='dml_file_')
try:
if _file_util.is_hdfs_path(commander_file_path):
_file_util.download_from_hdfs(
commander_file_path,
local_file_name,
hadoop_conf_dir = self.environment.hadoop_conf_dir)
elif _file_util.is_s3_path(commander_file_path):
_file_util.download_from_s3(
commander_file_path,
local_file_name,
aws_credentials = self.environment.get_credentials(),
silent = True)
with open(local_file_name,'r') as f:
status_json = _json.load(f)
port = status_json['port']
host_name = status_json['host_name']
if port > 0:
return 'http://%s:%s' % (host_name, port)
else:
return None
except:
# Ignore exception, we will fail after a few retry
pass
finally:
if _os.path.exists(local_file_name):
_os.remove(local_file_name)
def _load_remote(cls, path, schema_version, aws_credentials={}):
temp_dir = _gl.util._make_temp_filename(prefix='predictive_policy_')
if _file_util.is_s3_path(path):
_file_util.download_from_s3(path, temp_dir, is_dir=True,
aws_credentials=aws_credentials, silent=True)
elif _file_util.is_hdfs_path(path):
_file_util.download_from_hdfs(path, temp_dir, is_dir=True)
else:
assert 'Only support S3 and HDFS path for Predictive Object saving location!'
return cls._load_local(temp_dir)
def _load_remote(cls, path, schema_version, aws_credentials={}):
temp_dir = _tempfie.mkdtemp(prefix='predictive_object_')
if fu.is_s3_path(path):
fu.download_from_s3(path, temp_dir, is_dir=(schema_version > 2),
aws_credentials=aws_credentials)
elif fu.is_hdfs_path(path):
fu.download_from_hdfs(path, temp_dir, is_dir=(schema_version > 2))
else:
assert 'Only support S3 and HDFS path for Predictive Object saving location!'
return cls._load_local(temp_dir)
def _load_file_and_parse(self, file_name, parser_func, silent=False, test_url=True):
'''
Read remote file to a local temporary file, and use parser_func
to parse the content, returns the parsed result.
This function is used for parsing state and progress files from
either local, S3 or HDFS.
If there is any exception happened, returns None
'''
file_is_local = _file_util.is_local_path(file_name)
local_file_name = file_name if file_is_local else _tempfile.mktemp(prefix='job-status-')
try:
try:
if test_url and not self._test_url(file_name):
if not silent:
__LOGGER__.info("File %s is not available yet." % file_name)
return None
if _file_util.is_hdfs_path(file_name):
_file_util.download_from_hdfs(
hdfs_path = file_name,
local_path = local_file_name,
hadoop_conf_dir=self.environment.hadoop_conf_dir)
elif _file_util.is_s3_path(file_name):
_file_util.download_from_s3(
s3_path = file_name,
local_path = local_file_name,
is_dir = False,
aws_credentials = self.environment.ec2_config.get_credentials(),
silent = silent)
except Exception as e:
# It is ok the status file is not ready yet as the job is getting prepared
if not silent:
__LOGGER__.warning("Exception encountered when trying to download file from %s, error: %s" % (file_name, e))
return None
try:
# parse the local file
return parser_func(local_file_name)
except Exception as e:
__LOGGER__.info("Exception when parsing file %s. Error: %s" % (file_name, e))
return None
finally:
if (not file_is_local) and _os.path.exists(local_file_name):
_os.remove(local_file_name)
def _load_file_and_parse(self, file_name, parser_func, silent=False, test_url=True):
'''
Read remote file to a local temporary file, and use parser_func
to parse the content, returns the parsed result.
This function is used for parsing state and progress files from
either local, S3 or HDFS.
If there is any exception happened, returns None
'''
file_is_local = _file_util.is_local_path(file_name)
local_file_name = file_name if file_is_local else _tempfile.mktemp(prefix='job-status-')
try:
try:
if test_url and not self._test_url(file_name):
if not silent:
__LOGGER__.info("File %s is not available yet." % file_name)
return None
if _file_util.is_hdfs_path(file_name):
_file_util.download_from_hdfs(
hdfs_path = file_name,
local_path = local_file_name,
hadoop_conf_dir=self.environment.hadoop_conf_dir)
elif _file_util.is_s3_path(file_name):
_file_util.download_from_s3(
s3_path = file_name,
local_path = local_file_name,
is_dir = False,
aws_credentials = self.environment.ec2_config.get_credentials(),
silent = silent)
except Exception as e:
# It is ok the status file is not ready yet as the job is getting prepared
if not silent:
__LOGGER__.warning("Exception encountered when trying to download file from %s, error: %s" % (file_name, e))
return None
try:
# parse the local file
return parser_func(local_file_name)
except Exception as e:
__LOGGER__.info("Exception when parsing file %s. Error: %s" % (file_name, e))
return None
finally:
if (not file_is_local) and _os.path.exists(local_file_name):
_os.remove(local_file_name)
def _copy_predictive_object_files(source_path, target_path, is_dir, src_credentials, tgt_credentials):
'''
Copy either file or folder from source location to target location
'''
# Cleanup existing file path if exists
if _file_util.is_local_path(target_path) and _os.path.exists(target_path):
_shutil.rmtree(target_path)
if _file_util.is_s3_path(source_path) and _file_util.is_s3_path(target_path):
# compare credentials
_check_aws_credentials(src_credentials, tgt_credentials, source_path)
# intra s3 copy model
_file_util.intra_s3_copy_model(source_path, target_path, is_dir, tgt_credentials)
elif _file_util.is_local_path(source_path):
_file_util.copy_from_local(source_path, target_path, is_dir = is_dir)
else:
tmp_dir = _tempfile.mkdtemp(prefix = 'copy_predictive_object')
try:
# download to local first
local_path = _os.path.join(tmp_dir, 'temp_po_file')
if _file_util.is_s3_path(source_path):
_file_util.download_from_s3(
source_path,
local_path,
is_dir=is_dir,
aws_credentials=src_credentials,
silent=False)
elif _file_util.is_hdfs_path(source_path):
_file_util.download_from_hdfs(source_path, local_path, is_dir = False)
else:
raise RuntimeError('Unsupported file system type: %s' % source_path)
# upload from local to remote
if _file_util.is_s3_path(target_path):
_file_util.upload_to_s3(local_path, target_path, is_dir=is_dir,
aws_credentials=tgt_credentials, silent=False)
elif _file_util.is_hdfs_path(target_path):
_file_util.hdfs_mkdir(target_path)
_file_util.upload_to_hdfs(local_path, target_path, force=True, silent=False)
else:
_file_util.upload_to_local(local_path, target_path, is_dir=is_dir, silent=False)
finally:
_shutil.rmtree(tmp_dir)
def _read_cluster_state(self):
local_cluster_config_file = _tempfile.mktemp(prefix='hadoop-conf-')
try:
remote_cluster_config_file = "%s%s" % (self.dato_dist_path, HadoopCluster._DIST_INI)
if not _file_util.hdfs_test_url(remote_cluster_config_file, \
hadoop_conf_dir = self.hadoop_conf_dir):
raise ValueError('Path "%s" does not seem like a valid Dato Distributed '
'installation.' % self.dato_dist_path)
_file_util.download_from_hdfs(
hdfs_path = remote_cluster_config_file,
local_path = local_cluster_config_file,
hadoop_conf_dir=self.hadoop_conf_dir)
config = _ConfigParser.ConfigParser()
config.read(local_cluster_config_file)
return config
finally:
if _os.path.exists(local_cluster_config_file):
_os.remove(local_cluster_config_file)
def _download_remote_folder_to_local(self, remote_path, silent=False):
'''
Download all files from remote path to local. Caller is responsible for
cleaning up the local folder after finishing usage
Returns the local temporary folder
'''
local_path = _tempfile.mkdtemp(prefix='job-results')
try:
if _file_util.is_hdfs_path(remote_path):
_file_util.download_from_hdfs(
hdfs_path = remote_path,
local_path = local_path,
is_dir = True,
hadoop_conf_dir=self.environment.hadoop_conf_dir)
elif _file_util.is_s3_path(remote_path):
_file_util.download_from_s3(
s3_path = remote_path,
local_path = local_path,
is_dir = True,
aws_credentials = self.environment.ec2_config.get_credentials(),
silent = silent)
else:
raise RuntimeError("'%s' is not a supported remote path. Only S3 and HDFS"
" remote path are supported" % remote_path)
except:
# Make sure we cleanup local files if we cannot successfully
# download files
if _os.path.isdir(local_path):
_shutil.rmtree(local_path)
raise
return local_path
def _download_remote_folder_to_local(self, remote_path, silent=False):
'''
Download all files from remote path to local. Caller is responsible for
cleaning up the local folder after finishing usage
Returns the local temporary folder
'''
local_path = _tempfile.mkdtemp(prefix='job-results')
try:
if _file_util.is_hdfs_path(remote_path):
_file_util.download_from_hdfs(
hdfs_path = remote_path,
local_path = local_path,
is_dir = True,
hadoop_conf_dir=self.environment.hadoop_conf_dir)
elif _file_util.is_s3_path(remote_path):
_file_util.download_from_s3(
s3_path = remote_path,
local_path = local_path,
is_dir = True,
aws_credentials = self.environment.ec2_config.get_credentials(),
silent = silent)
else:
raise RuntimeError("'%s' is not a supported remote path. Only S3 and HDFS"
" remote path are supported" % remote_path)
except:
# Make sure we cleanup local files if we cannot successfully
# download files
if _os.path.isdir(local_path):
_shutil.rmtree(local_path)
raise
return local_path
def _read_cluster_state(self):
local_cluster_config_file = _tempfile.mktemp(prefix='hadoop-conf-')
try:
remote_cluster_config_file = "%s%s" % (self.turi_dist_path,
HadoopCluster._DIST_INI)
if not _file_util.hdfs_test_url(remote_cluster_config_file, \
hadoop_conf_dir = self.hadoop_conf_dir):
raise ValueError(
'Path "%s" does not seem like a valid Turi Distributed '
'installation.' % self.turi_dist_path)
_file_util.download_from_hdfs(hdfs_path=remote_cluster_config_file,
local_path=local_cluster_config_file,
hadoop_conf_dir=self.hadoop_conf_dir)
config = _ConfigParser.ConfigParser()
config.read(local_cluster_config_file)
return config
finally:
if _os.path.exists(local_cluster_config_file):
_os.remove(local_cluster_config_file)
