def _test_url(self,file_path): if _file_util.is_hdfs_path(file_path): return _file_util.hdfs_test_url(file_path,'e',self.environment.hadoop_conf_dir) if _file_util.is_s3_path(file_path): return _file_util.s3_test_url(file_path,self.environment.ec2_config.get_credentials()) else: return _os.path.exists(file_path)
def _validate_dato_distr_param(dato_dist_path, hadoop_conf_dir = None): if not type(dato_dist_path) in [str, unicode]: raise TypeError('"dato_dist_path" needs to be a string') if hadoop_conf_dir is not None and not _os.path.isdir(hadoop_conf_dir): raise ValueError('"%s" has to be a local folder.' % hadoop_conf_dir) if not _file_util.hdfs_test_url(dato_dist_path, \ hadoop_conf_dir = hadoop_conf_dir): raise ValueError('Hadoop path "%s" does not exist.' % dato_dist_path)
def _validate_turi_distr_param(turi_dist_path, hadoop_conf_dir=None): if not type(turi_dist_path) in [str, unicode]: raise TypeError('"turi_dist_path" needs to be a string') if hadoop_conf_dir is not None and not _os.path.isdir(hadoop_conf_dir): raise ValueError('"%s" has to be a local folder.' % hadoop_conf_dir) if not _file_util.hdfs_test_url(turi_dist_path, \ hadoop_conf_dir = hadoop_conf_dir): raise ValueError('Hadoop path "%s" does not exist.' % turi_dist_path)
def _read_cluster_state(self): local_cluster_config_file = _tempfile.mktemp(prefix='hadoop-conf-') try: remote_cluster_config_file = "%s%s" % (self.dato_dist_path, HadoopCluster._DIST_INI) if not _file_util.hdfs_test_url(remote_cluster_config_file, \ hadoop_conf_dir = self.hadoop_conf_dir): raise ValueError('Path "%s" does not seem like a valid Dato Distributed ' 'installation.' % self.dato_dist_path) _file_util.download_from_hdfs( hdfs_path = remote_cluster_config_file, local_path = local_cluster_config_file, hadoop_conf_dir=self.hadoop_conf_dir) config = _ConfigParser.ConfigParser() config.read(local_cluster_config_file) return config finally: if _os.path.exists(local_cluster_config_file): _os.remove(local_cluster_config_file)
def _read_cluster_state(self): local_cluster_config_file = _tempfile.mktemp(prefix='hadoop-conf-') try: remote_cluster_config_file = "%s%s" % (self.turi_dist_path, HadoopCluster._DIST_INI) if not _file_util.hdfs_test_url(remote_cluster_config_file, \ hadoop_conf_dir = self.hadoop_conf_dir): raise ValueError( 'Path "%s" does not seem like a valid Turi Distributed ' 'installation.' % self.turi_dist_path) _file_util.download_from_hdfs(hdfs_path=remote_cluster_config_file, local_path=local_cluster_config_file, hadoop_conf_dir=self.hadoop_conf_dir) config = _ConfigParser.ConfigParser() config.read(local_cluster_config_file) return config finally: if _os.path.exists(local_cluster_config_file): _os.remove(local_cluster_config_file)