def _test_url(self,file_path):
if _file_util.is_hdfs_path(file_path):
return _file_util.hdfs_test_url(file_path,'e',self.environment.hadoop_conf_dir)
if _file_util.is_s3_path(file_path):
return _file_util.s3_test_url(file_path,self.environment.ec2_config.get_credentials())
else:
return _os.path.exists(file_path)
def _test_url(self,file_path):
if _file_util.is_hdfs_path(file_path):
return _file_util.hdfs_test_url(file_path,'e',self.environment.hadoop_conf_dir)
if _file_util.is_s3_path(file_path):
return _file_util.s3_test_url(file_path,self.environment.ec2_config.get_credentials())
else:
return _os.path.exists(file_path)
def _validate_dato_distr_param(dato_dist_path, hadoop_conf_dir = None):
if not type(dato_dist_path) in [str, unicode]:
raise TypeError('"dato_dist_path" needs to be a string')
if hadoop_conf_dir is not None and not _os.path.isdir(hadoop_conf_dir):
raise ValueError('"%s" has to be a local folder.' % hadoop_conf_dir)
if not _file_util.hdfs_test_url(dato_dist_path, \
hadoop_conf_dir = hadoop_conf_dir):
raise ValueError('Hadoop path "%s" does not exist.' % dato_dist_path)
def _validate_turi_distr_param(turi_dist_path, hadoop_conf_dir=None):
if not type(turi_dist_path) in [str, unicode]:
raise TypeError('"turi_dist_path" needs to be a string')
if hadoop_conf_dir is not None and not _os.path.isdir(hadoop_conf_dir):
raise ValueError('"%s" has to be a local folder.' % hadoop_conf_dir)
if not _file_util.hdfs_test_url(turi_dist_path, \
hadoop_conf_dir = hadoop_conf_dir):
raise ValueError('Hadoop path "%s" does not exist.' % turi_dist_path)
def _read_cluster_state(self):
local_cluster_config_file = _tempfile.mktemp(prefix='hadoop-conf-')
try:
remote_cluster_config_file = "%s%s" % (self.dato_dist_path, HadoopCluster._DIST_INI)
if not _file_util.hdfs_test_url(remote_cluster_config_file, \
hadoop_conf_dir = self.hadoop_conf_dir):
raise ValueError('Path "%s" does not seem like a valid Dato Distributed '
'installation.' % self.dato_dist_path)
_file_util.download_from_hdfs(
hdfs_path = remote_cluster_config_file,
local_path = local_cluster_config_file,
hadoop_conf_dir=self.hadoop_conf_dir)
config = _ConfigParser.ConfigParser()
config.read(local_cluster_config_file)
return config
finally:
if _os.path.exists(local_cluster_config_file):
_os.remove(local_cluster_config_file)
def _read_cluster_state(self):
local_cluster_config_file = _tempfile.mktemp(prefix='hadoop-conf-')
try:
remote_cluster_config_file = "%s%s" % (self.turi_dist_path,
HadoopCluster._DIST_INI)
if not _file_util.hdfs_test_url(remote_cluster_config_file, \
hadoop_conf_dir = self.hadoop_conf_dir):
raise ValueError(
'Path "%s" does not seem like a valid Turi Distributed '
'installation.' % self.turi_dist_path)
_file_util.download_from_hdfs(hdfs_path=remote_cluster_config_file,
local_path=local_cluster_config_file,
hadoop_conf_dir=self.hadoop_conf_dir)
config = _ConfigParser.ConfigParser()
config.read(local_cluster_config_file)
return config
finally:
if _os.path.exists(local_cluster_config_file):
_os.remove(local_cluster_config_file)
