def write(self, msg: MSG):
# logging.debug("FILE::{0}::WRITE msg:{1}".format(self.conn, msg))
if self.url_is_dir:
if not os.path.exists(self.conn.path):
os.mkdir(self.conn.path)
# -------------------------------
rm_idx = []
for i in range(0, msg.dt_count, 1):
dt = msg.read_datum(i)
if self.url_is_dir:
path = os.path.join(self.conn.path, dt["path"])
else:
path = self.conn.path
suffix = os.path.splitext(path)[-1]
# -------------------------------
if os.path.exists(path):
logging.warning("FILE::{0}::WRITE failed.".format(path))
elif os.path.isfile(os.path.split(path)[0]):
logging.warning("FILE::{0}::WRITE failed.".format(path))
else:
if not os.path.isdir(os.path.split(path)[0]):
mkdir(os.path.split(path)[0])
# -------------------------------
if suffix in [".xls", ".xlsx"]:
if isinstance(dt["stream"], pd.DataFrame):
dt["stream"].to_excel(path, index=False)
logging.info(
"FILE::EXCEL::{0}::WRITE successfully.".format(
path))
else:
logging.warning(
"FILE::EXCEL::{0}::WRITE failed.".format(path))
elif suffix in [".npy"]:
if isinstance(dt["stream"], np.ndarray):
np.save(path, dt["stream"])
logging.info(
"FILE::NUMPY::{0}::WRITE successfully.".format(
path))
else:
logging.warning(
"FILE::NUMPY::{0}::WRITE failed.".format(path))
elif suffix in [""]:
logging.warning("FILE::{0}::WRITE None.".format(path))
else:
with open(path, "wb") as file:
file.write(dt["stream"])
logging.info("FILE::{0}::WRITE successfully.".format(path))
rm_idx = [i] + rm_idx
# -------------------------------
if CONFIG.IS_DATA_WRITE_END:
for i in rm_idx:
msg.remove_datum(i)
def write(self, msg: MSG):
# logging.debug("FTP::{0}::WRITE msg:{1}".format(self.conn, msg))
ftp = self.open()
# -------------------------------------------------
rm_idx = []
for i in range(0, msg.dt_count, 1):
dt = msg.read_datum(i)
if self.url_is_dir:
path = os.path.join(self.conn.path, dt["path"])
else:
path = self.conn.path
# ----------------------------
suffix = os.path.splitext(path)[-1]
temp_path = "temp/temp" + suffix
mkdir(os.path.split(temp_path)[0])
# ----------------------------
if suffix in [".xls", ".xlsx"]:
if isinstance(dt["stream"], pd.DataFrame):
dt["stream"].to_excel(temp_path, index=False)
with open(temp_path, "rb") as temp_file:
stream = temp_file.read()
logging_info = "::EXCEL"
else:
stream = None
logging_info = "::EXCEL"
elif suffix in [".npy"]:
if isinstance(dt["stream"], np.ndarray):
np.save(temp_path, dt["stream"])
with open(temp_path, "rb") as temp_file:
stream = temp_file.read()
logging_info = "::NUMPY"
else:
stream = None
logging_info = "::NUMPY"
else:
stream = dt["stream"]
logging_info = ""
# ----------------------------
ftp.storbinary('STOR ' + path, stream, CONFIG.FTP_BUFFER)
rm_idx = [i] + rm_idx
logging.info("FTP{0}::{1}::WRITE successfully.".format(
logging_info, self.conn))
# -------------------------------------------------
if CONFIG.IS_DATA_WRITE_END:
for i in rm_idx:
msg.remove_datum(i)
ftp.close()
def write(self, msg: MSG):
# logging.debug("SQL::{0} write:{1}".format(self.conn, msg))
# ----------------------------------
rm_idx = []
for i in range(0, msg.dt_count, 1):
rt = msg.read_datum(i)
df = rt["stream"]
path = os.path.splitext(rt["path"])[0]
if isinstance(df, pd.DataFrame):
df.to_sql(path,
con=self.engine,
if_exists='replace',
index=False,
index_label=False)
logging.info("SQL::{0}::WRITE successfully.".format(self.conn))
else:
logging.warning("SQL::{0}::WRITE failed.".format(self.conn))
rm_idx = [i] + rm_idx
# ----------------------------------
if CONFIG.IS_DATA_WRITE_END:
for i in rm_idx:
msg.remove_datum(i)
