def start_transformation(selected_file_name, working_file_path, info,
source_file_path, backup_file_path):
import sys, time
parent_path = str(Path().resolve().parent)
if parent_path not in sys.path:
sys.path.insert(1, str(Path().resolve().parent))
from sales.configManager import ConfigManager
from Common.db import set_db
configManager = ConfigManager.createInstance()
set_db(configManager) # configure database connection object
from sales.dataProviderManager import get_data_provider_manager
from Common.logging.loggingManager import get_applogger
_log = get_applogger() # Initialize log Manager
FileInfo = collections.namedtuple('FileInfo', 'name path size')
try:
start_time = time.time()
_log.info(
f"[ sales_data_fetch_intern -> start_transformation ] start file Processing Start : {selected_file_name} at {time.ctime()}"
)
data_provider_manager = get_data_provider_manager(configManager)
data_transformer = data_provider_manager.create_data_transformer(
_log, configManager)
move(working_file_path,
working_file_path + ".old") # Rename File with .old Extension
data_transformer.transform_file(working_file_path + ".old",
working_file_path)
transformed_data_file = FileInfo(name=selected_file_name,
path=working_file_path,
size=info.st_size)
_log.info(
f"[ sales_data_fetch_intern -> start_transformation ] Downloaded Transformed data File {transformed_data_file}"
)
destination_file_path = configManager.get_destination_file_path(
transformed_data_file.name)
_log.debug(
f"[ sales_data_fetch_intern -> start_transformation ] Copying Transformed data file:{transformed_data_file.path} to Destination Location: {destination_file_path}"
)
copyfile(transformed_data_file.path, destination_file_path)
_log.info(
f'[ sales_data_fetch_intern -> start_transformation ] Sales Data Files has been fetched and stored in {destination_file_path}'
)
_log.info(
f"[ sales_data_fetch_intern -> start_transformation ] file {selected_file_name} Processing time : {time.time() - start_time} seconds"
)
return {
"source_file_path": source_file_path,
"backup_file_path": backup_file_path,
"is_transformed": True
}
except Exception as e:
_log.error("[ sales_data_fetch_intern -> get_remote_data_file ] " +
str(e))
return {"is_transformed": False}
def transform_row(df):
import sys
from pathlib import Path
parent_path = str(Path().resolve().parent)
if parent_path not in sys.path:
sys.path.insert(1, str(Path().resolve().parent))
from sales.configManager import ConfigManager
from sales.etl.settings import Settings
configManager = ConfigManager.createInstance()
from Common.db import set_db
set_db(configManager)
from Common.db.models import Product
from Common.logging.loggingManager import get_applogger
_log = get_applogger() # Initialize log Manager
data_provider_options = Settings(
configManager.get_data_provider_settings())
try:
for index, row in df.iterrows():
try:
a = Product.get(Product.room_id == row[
data_provider_options.source_room_id_field]).name
df.loc[index, data_provider_options.product_name_field] = a
except:
df.loc[index, data_provider_options.product_name_field] = ''
return df
except Exception as e:
_log.error("[ data_transform -> transform_file ] " + str(e))
# Created By: Jeenal Suthar
# Created Date:
# Last Modified: 22/01/2020
# Description: This module get file form remote directory or from local directory.
import os
import shutil
from shutil import move
from Common.Utils import create_directory
from Common.sftp_file_manager import SFTPFileManager
from Common.logging.loggingManager import get_applogger
_log = get_applogger()
class FileFetchManager():
_sftpFileManager = None
def __init__(self, config):
self.config = config
self._abs_dir = None
self.sftp_enable = self.config.get_sftp_enabled()
if self.sftp_enable:
self._sftpFileManager = SFTPFileManager(self.config.get_sftp_host(), self.config.get_sftp_username(),
self.config.get_sftp_password())
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
if self.sftp_enable: