def run(self):
spark_app = SparkBaseApp()
spark = spark_app.spark
hdfs = spark_app.hdfs
hdfs_save_path = Path(config.HDFS_FEATURE_DIR)
if hdfs.exists(hdfs_save_path / "_SUCCESS"):
logging.info("Exists data found. {}".format(hdfs_save_path))
logging.info("Data is ready! Skip prepare...")
return
train_data_path, test_data_path = get_data()
if config.IRIS_DATASET_FOR == "train":
local_data_path = train_data_path
else:
local_data_path = test_data_path
pdf = pd.read_csv(local_data_path, header=None)
pdf.columns = data_header
pdf["idx"] = [i for i in range(1, len(pdf) + 1)]
df = spark.createDataFrame(pdf)
logging.info("Saving Dataset to {}".format(hdfs_save_path))
df.repartition(1).write.save(hdfs_save_path.as_posix())
def __init__(self):
super(SparkNormalizer, self).__init__()
self._normalizer_map = None
self.default_method = "minmax"
self._methods = {
"minmax": MinMaxScaler,
"zscore": StandardScaler,
"pnorm": Normalizer
}
self._models = {
"minmax": MinMaxScalerModel,
"zscore": StandardScalerModel,
"pnorm": Normalizer
}
self._params = {
"minmax": {"min": 0.0, "max": 1.0},
"zscore": {"withMean": True, "withStd": True},
"pnorm": {"p": 2.0}
}
self._spark_app = SparkBaseApp()
self._sc = self._spark_app.sc
def run(self):
def solve_local_path(raw_path):
path = Path(raw_path).resolve()
if not path.is_dir():
path.mkdir(parents=True)
return path.as_posix()
config.LOCAL_WORKSPACE = solve_local_path(config.LOCAL_WORKSPACE)
config.LOCAL_MODEL_DIR = solve_local_path(config.LOCAL_MODEL_DIR)
config.LOCAL_FEMODEL_DIR = solve_local_path(config.LOCAL_FEMODEL_DIR)
config.LOCAL_TMP_DIR = solve_local_path(config.LOCAL_TMP_DIR)
def seq_conf_parser(v, sign=","):
if isinstance(v, str):
iter_v = v.split(sign)
elif isinstance(v, list):
iter_v = v
else:
iter_v = []
return [i for i in map(lambda x: x.strip(), iter_v) if i]
config.PRIMARY_KEYS = seq_conf_parser(config.PRIMARY_KEYS)
config.LABELS = seq_conf_parser(config.LABELS)
config.DROP_COLUMNS = seq_conf_parser(config.DROP_COLUMNS)
if "SPARK" in config:
app_name = ".".join(["DLFlow", config.uuid])
spark_conf = config.SPARK.dense_dict if config.SPARK else {}
spark_app = SparkBaseApp()
spark_app.initialize_spark(app_name, spark_conf)
else:
hdfs = HDFS()
hdfs.initialize_hdfs()
def __init__(self):
super(SparkFeParser, self).__init__()
self._spark_app = SparkBaseApp()
self._spark = self._spark_app.spark
def run(self):
spark_app = SparkBaseApp()
spark = spark_app.spark
sc = spark_app.sc
hdfs = spark_app.hdfs
dirs = config._build_dirs
tmp_fmap_dir = dirs["tmp_fmap_dir"]
hdfs_ckpt_dir = dirs["hdfs_ckpt_dir"]
hdfs_static_dir = dirs["hdfs_static_dir"]
sc.addFile(hdfs.hdfs_whole_path(hdfs_static_dir.as_posix()),
recursive=True)
sc.addFile(hdfs.hdfs_whole_path(hdfs_ckpt_dir.as_posix()),
recursive=True)
fmap = Fmap.load(tmp_fmap_dir)
bc_static_model_dir = sc.broadcast("static")
bc_fmap = sc.broadcast(fmap)
bc_config = sc.broadcast(config)
def predict_map(rdd):
from pyspark.files import SparkFiles
config = bc_config.value
fmap = bc_fmap.value
static_dir = SparkFiles.get(bc_static_model_dir.value)
ckpt_dir = SparkFiles.get("ckpt")
from dlflow.mgr import Collector, model
collect = Collector()
collect(static_dir, "Static models")
input_cls = model[config.MODEL.input_name]
dataset = input_cls(fmap).rdd_inputs(rdd, config.MODEL.batch_size)
model_cls = model[config.MODEL.model_name]
model_ins = model_cls(fmap)
model_ins.load_model(ckpt_dir)
return model_ins.predict_act(dataset)
local_model = model[config.MODEL.model_name](fmap)
df_title = local_model.pkey_names
df_title.extend(local_model.output_names)
df = spark.read.parquet(config.HDFS_ENCODE_DIR)
parallelism = spark.conf.get("spark.default.parallelism", None)
if parallelism:
num_partitions = int(parallelism)
else:
num_partitions = df.rdd.getNumPartitions()
pred_df = df.repartition(num_partitions) \
.rdd \
.mapPartitions(predict_map) \
.toDF(df_title)
hdfs_predict_dir = config.HDFS_PREDICT_DIR
spark_app.save_compress(pred_df, hdfs_predict_dir)
logging.info(
i18n("Predicting result save to {}").format(hdfs_predict_dir))
logging.info(i18n("Predicting Done."))
def close(self):
from dlflow.utils.sparkapp import SparkBaseApp, HDFS
SparkBaseApp().close()
HDFS().close()
def run(self):
spark_app = SparkBaseApp()
spark = spark_app.spark
hdfs = spark_app.hdfs
if "HDFS_TFRECORD_DIR" in config:
hdsf_tfrecord_dir = Path(config.HDFS_TFRECORD_DIR)
if hdfs.exists(hdsf_tfrecord_dir / "_SUCCESS"):
logging.info(i18n("TFRecords already exists, skip encoding."))
return
elif "HDFS_ENCODE_DIR" in config:
hdfs_encode_dir = Path(config.HDFS_ENCODE_DIR)
if hdfs.exists(hdfs_encode_dir / "_SUCCESS"):
logging.info(
i18n("Encode result already exists, encoding done."))
return
fmap_dir = "fmap_{}".format(config.uuid)
tmp_fmap_dir = Path(config.LOCAL_TMP_DIR).joinpath(fmap_dir)
local_fmap_dir = Path(config.LOCAL_FEMODEL_DIR).joinpath("fmap")
local_norm_dir = Path(config.LOCAL_FEMODEL_DIR).joinpath("norm")
hdfs_fmap_dir = Path(config.HDFS_FEMODEL_DIR).joinpath("fmap")
hdfs_norm_dir = Path(config.HDFS_FEMODEL_DIR).joinpath("norm")
if not hdfs.exists(config.HDFS_FEMODEL_DIR):
hdfs.mkdirs(hdfs_fmap_dir.parent)
spark_parser = Parser("spark")
parser_cls = spark_parser.get_parser()
normalizer_cls = spark_parser.get_normalizer()
if "SPARK.spark.default.parallelism" in config:
parallelism = int(config.SPARK.spark.default.parallelism)
else:
parallelism = _DEFAULT_PARALLELISM
df = spark.read \
.parquet(config.HDFS_FEATURE_DIR) \
.repartition(parallelism)
parser = parser_cls()
if hdfs.exists(hdfs_fmap_dir.joinpath("fmap.meta")):
logging.info(i18n("Using HDFS fmap: {}").format(hdfs_fmap_dir))
hdfs.get(hdfs_fmap_dir, tmp_fmap_dir)
else:
logging.info(
i18n("There is no fmap available, start to "
"generate fmap by parsing features."))
primary_keys = config.PRIMARY_KEYS
labels = config.LABELS
drop_columns = config.DROP_COLUMNS
buckets = None if config.BUCKET is None else config.BUCKET.dict
parser.fit(df,
buckets=buckets,
drop_columns=drop_columns,
primary_keys=primary_keys,
labels=labels)
parser.save(tmp_fmap_dir)
logging.info(i18n("Put fmap to HDFS: {}").format(hdfs_fmap_dir))
hdfs.delete(hdfs_fmap_dir)
hdfs.put(tmp_fmap_dir, hdfs_fmap_dir)
if local_fmap_dir.exists():
logging.warning(
i18n("Local directory {} already exists, "
"it will be overwritten: {}").format(
"fmap", local_fmap_dir))
shutil.rmtree(local_fmap_dir)
shutil.copytree(tmp_fmap_dir, local_fmap_dir)
parser.load(tmp_fmap_dir)
encode_df = parser.transform(df)
normalizer = normalizer_cls()
if hdfs.exists(
hdfs_norm_dir.joinpath("normalizers_metadata", "_SUCCESS")):
normalizer.load(hdfs_norm_dir)
else:
hdfs.mkdirs(hdfs_norm_dir)
try:
bucket_conf = config.BUCKET.dict
except AttributeError:
bucket_conf = None
if config.BUCKET is not None:
logging.error(i18n("Get wrong type bucket configuration."))
normalizer.fit(encode_df, parser.fmap, bucket_conf=bucket_conf)
normalizer.save(hdfs_norm_dir)
if local_norm_dir.exists():
logging.warning(
i18n("Local directory {} already exists, "
"it will be overwritten: {}").format(
"norm", local_norm_dir))
shutil.rmtree(local_norm_dir)
hdfs.get(hdfs_norm_dir, local_norm_dir)
norm_df = normalizer.transform(encode_df)
spark_app.save_compress(norm_df, config.HDFS_ENCODE_DIR, use_tfr=False)
if "HDFS_TFRECORD_DIR" in config:
spark_app.save_compress(norm_df,
config.HDFS_TFRECORD_DIR,
use_tfr=True)
logging.info(i18n("Encoding done."))