def test_hdfs_rmr(self):
p = "/test/text"
# 如果不存在, 不存在，新建
if not FabricHdfsUtils().hdfs_exits(p):
FabricHdfsUtils().hdfs_mkdir(p)
a = FabricHdfsUtils().hdfs_rmr(p).return_code
# 删除
TestUtils.assert_equal(0,
FabricHdfsUtils().hdfs_rmr(p).return_code,
"删除目录: %s" % p)
# 是否删除成功
TestUtils.assert_false(FabricHdfsUtils().hdfs_exits(p),
"判断目录:%s 是否存在" % p)
class LoadRawData(object):
fabric_utils = FabricHdfsUtils()
raw_file_folder = RawFiles()
def put_utils(self, file_name):
raw = os.path.join(RAW_DATA_ROOT_WITH_DATE, file_name)
hdfs = os.path.join(HDFS_RISK_MODEL_AUTO_RAW, file_name)
self.fabric_utils.hdfs_put(raw, hdfs)
def put_all_daily(self):
map(lambda file_name: self.put_utils(file_name), self.raw_file_folder.DAILY_FILES)
def test_hdfs_exits(self):
p = "/test/text"
TestUtils.assert_false(FabricHdfsUtils().hdfs_exits(p),
"判断目录:%s 是否存在" % p)
def test_spark_submit(self):
FabricHdfsUtils().spark_submit("./aagsgsfhsg.py")
def test_hdfs_mkdir(self):
p = "/test/text"
rst = FabricHdfsUtils().hdfs_mkdir(p)
print("rst", rst)
TestUtils.assert_equal(0, rst.return_code, "创建目录: %s" % p)
def reload_all_daily_hdfs():
fabric_hdfs_utils = FabricHdfsUtils()
if fabric_hdfs_utils.hdfs_exits(HDFS_RISK_MODEL_AUTO_RAW):
fabric_hdfs_utils.hdfs_rmr(HDFS_RISK_MODEL_AUTO_RAW)
fabric_hdfs_utils.hdfs_mkdir(HDFS_RISK_MODEL_AUTO_RAW)
LoadRawData().put_all_daily()
def __init__(self):
self.task_dag = TaskDag()
self.cls_map = {}
self._task_run_serial = []
self._task_run_serial_edg = []
self.fabric_hdfs_utils = FabricHdfsUtils()