def on_created(self, event):
logging.info(
f'event type: {event.event_type} path : {event.src_path}')
try:
copyfile(
event.src_path,
config.get('Directory', 'temp') +
os.path.basename(event.src_path))
connection = utils.get_oracle_connection_obj()
update_file_details(
True, connection, config.get('Database', 'tablespace'),
config.get('Database', 'file_details_tablename'),
os.path.basename(event.src_path))
except Exception as ex:
error.Errors(ex).errorrun()
err, = ex.args
copyfile(
event.src_path,
config.get('Directory', 'error') +
os.path.basename(event.src_path))
update_file_details(
False, connection, config.get('Database', 'tablespace'),
config.get('Database', 'file_details_tablename'),
os.path.basename(event.src_path), f"{err.code}:{err.message}")
def on_created(self, event):
logging.info(
f'event type: {event.event_type} path : {event.src_path}')
try:
copyfile(
event.src_path,
config.get('Directory', 'outbound') +
os.path.basename(event.src_path))
except Exception as ex:
error.Errors(ex).errorrun()
copyfile(
event.src_path,
config.get('Directory', 'error') +
os.path.basename(event.src_path))
def extract_source_data(extraction_id, enc):
"""
Extacts source and source database connection details from
extraction_details and connection_detalis tables respectively
and extracts source data and store it in provided delimited file.
:param extraction_id: source extraction_id
:param enc : AESCipher object
"""
try:
logging.getLogger().setLevel(logging.INFO)
connection = utils.get_oracle_connection_obj()
config = utils.get_config_parser()
extraction_query = get_select_query(
config.get('Database', 'extraction_columnlist'),
config.get('Database', 'tablespace'),
config.get('Database', 'extraction_tablename'),
"EXTRACTION_ID=" + extraction_id.__str__())
logging.info("Extraction_details query = " + extraction_query)
extraction_details_df = pd.read_sql(extraction_query, con=connection)
# Get connection details for given connection_id corresponding to extraction id
connection_id = extraction_details_df.iloc[0]['CONNECTION_ID']
connection_query = get_select_query(
config.get('Database', 'connection_columnlist'),
config.get('Database', 'tablespace'),
config.get('Database', 'connection_tablename'),
"CONNECTION_ID=" + connection_id.__str__())
logging.info("Connection_details query = " + connection_query)
connection_details_df = pd.read_sql(connection_query, con=connection)
# create connection object for extraction table db
password_file = connection_details_df.iloc[0]['PASSWORD_FILE']
password = enc.decrypt_file(password_file)
connection_src_extraction = utils.get_oracle_connection_obj(
connection_details_df.iloc[0]['USERNAME'], password,
connection_details_df.iloc[0]['CONNECTION_STRING'])
# Get current incremental column's max value from source data
max_value = get_max(connection_src_extraction, extraction_details_df)
source_extraction_query = get_select_query(
extraction_details_df.iloc[0]['COLUMN_LIST'],
extraction_details_df.iloc[0]['DATABASE_NAME'],
extraction_details_df.iloc[0]['TABLE_NAME'],
extraction_details_df.iloc[0]['INCREMENTAL_COLUMN'] + " > " +
extraction_details_df.iloc[0]['INCREMENTAL_VALUE'].__str__() +
" AND " + extraction_details_df.iloc[0]['INCREMENTAL_COLUMN'] +
" <= " + max_value.__str__())
logging.info("Source extraction query = " + source_extraction_query)
source_extraction_df = pd.read_sql(source_extraction_query,
con=connection_src_extraction)
logging.info("copying " + extraction_details_df.iloc[0]['FILENAME'] +
" file to " + config.get('Directory', 'inbound') +
" path")
source_extraction_df.to_csv(
config.get('Directory', 'inbound') +
extraction_details_df.iloc[0]['FILENAME'],
index=False,
sep=extraction_details_df.iloc[0]['FILE_DELIMITER'])
logging.info("Updating File_details oracle table")
today = date.today().strftime("%Y-%m-%d")
insert_file_details(
connection, config.get('Database', 'tablespace'),
config.get('Database', 'file_details_tablename'),
f"0,'{connection_details_df.iloc[0]['SYSTEM']}', "
f"'{extraction_details_df.iloc[0]['DATABASE_NAME']}', "
f"'{extraction_details_df.iloc[0]['TABLE_NAME']}', "
f"'{extraction_details_df.iloc[0]['FILENAME']}', "
f"TO_DATE('{datetime.now().strftime('%Y%m%d%H%M%S')}','yyyymmddhh24miss'),"
f"{source_extraction_df.shape[0]}, 'False', 'False', 'None', date'{today}',"
f" '{getpass.getuser()}',"
f" date'{today}', '{getpass.getuser()}'")
logging.info("Successfully extracted data for " +
extraction_details_df.iloc[0]['TABLE_NAME'] + " table.")
logging.info(
"Updating incremental column's value in extraction_details")
# #####Uncomment below code to update incremental_value column in excraction_details table #######
# update_extraction_details(connection, config.get('Database', 'tablespace'),
# config.get('Database', 'extraction_tablename'),
# extraction_id, max_value)
logging.info(
"####################### JOB COMPLETED SUCCESSFULLY ######################"
)
except Exception as ex:
# err, = ex.args
error.Errors(ex).errorrun()
# logging.error("Error code = ", err.code)
# logging.error("Error Message = ", err.message)
connection.close()
connection_src_extraction.close()
os._exit(1)
connection.close()
connection_src_extraction.close()
return 0
def with_approximation(self):
self.figure.clear()
self.without_approximation()
self.ax4.set_title("Total approximation errors",
fontweight="bold",
size=13)
self.ax4.grid(True)
# x_locator = mplt.ticker.MultipleLocator(base=1)
# self.ax4.xaxis.set_major_locator(x_locator)
euler_max_errors = []
improved_euler_max_errors = []
runge_kutta_max_errors = []
nums = [k for k in range(self.N0, self.NF + 1)]
for k in range(self.N0, self.NF + 1):
num_methods = numericals.Numerical_methods(self.x0, self.y0,
self.X, k)
x, y_graph = graph.Graph(self.x0, self.y0, self.X, k).solve()
local_errors = errors.Errors(k)
x, y_euler = num_methods.euler()
y_local_euler = local_errors.local_error(y_graph, y_euler)
euler_max_errors.append(max(y_local_euler))
x, y_improved_euler = num_methods.improved_euler()
y_local_improved_euler = local_errors.local_error(
y_graph, y_improved_euler)
improved_euler_max_errors.append(max(y_local_improved_euler))
x, y_runge_kutta = num_methods.runge_kutta()
y_local_runge_kutta = local_errors.local_error(
y_graph, y_runge_kutta)
runge_kutta_max_errors.append(max(y_local_runge_kutta))
self.ax4.plot(nums,
euler_max_errors,
"red",
label='total approximation euler error')
self.ax4.plot(nums,
improved_euler_max_errors,
"blue",
label='total approximation improved euler error')
self.ax4.plot(nums,
runge_kutta_max_errors,
"green",
label='total approximation runge-kutta error')
if (self.mradio1.isChecked()):
self.ax4.plot(nums, euler_max_errors, 'ro', markersize=4)
self.ax4.plot(nums, improved_euler_max_errors, 'bo', markersize=4)
self.ax4.plot(nums, runge_kutta_max_errors, 'go', markersize=4)
for i, j in zip(nums, euler_max_errors):
self.ax4.annotate(str(i) + ";" + str(round(j, 3)),
xy=(i, j),
color='red')
for i, j in zip(nums, improved_euler_max_errors):
self.ax4.annotate(str(i) + ";" + str(round(j, 3)),
xy=(i, j),
color='blue')
for i, j in zip(nums, runge_kutta_max_errors):
self.ax4.annotate(str(i) + ";" + str(round(j, 3)),
xy=(i, j),
color='green')
self.ax4.set_ylabel("Error value")
self.ax4.set_xlabel("Number of points on each step")
self.ax4.legend(loc='best')
self.figure.tight_layout()
self.canvas.draw()
def without_approximation(self):
self.figure.clear()
num_methods = numericals.Numerical_methods(self.x0, self.y0, self.X,
self.NUMBERS)
graphic = graph.Graph(self.x0, self.y0, self.X, self.NUMBERS)
if self.radio2.isChecked():
self.ax1 = self.figure.add_subplot(311)
self.ax2 = self.figure.add_subplot(312)
self.ax3 = self.figure.add_subplot(313)
else:
self.ax1 = self.figure.add_subplot(411)
self.ax2 = self.figure.add_subplot(412)
self.ax3 = self.figure.add_subplot(413)
self.ax4 = self.figure.add_subplot(414)
self.ax1.set_title("Numerical methods", fontweight="bold", size=13)
self.ax2.set_title("Exact solution", fontweight="bold", size=13)
self.ax3.set_title("Local errors", fontweight="bold", size=13)
self.ax1.grid(True)
self.ax2.grid(True)
self.ax3.grid(True)
# x_locator = mplt.ticker.MultipleLocator(base=self.X_STEP)
# y_locator = mplt.ticker.MultipleLocator(base=self.Y_STEP)
x, y_graph = graphic.solve()
self.ax2.plot(x, y_graph, "purple", label="y'=sin(x)+y")
self.ax2.legend(loc='upper left')
# self.ax2.xaxis.set_major_locator(x_locator)
# self.ax2.yaxis.set_major_locator(y_locator)
x, y_euler = num_methods.euler()
self.ax1.plot(x, y_euler, "red", label="Euler")
self.ax1.legend(loc='upper left')
x, y_improved_euler = num_methods.improved_euler()
self.ax1.plot(x, y_improved_euler, "blue", label="Improved Euler")
self.ax1.legend(loc='upper left')
x, y_runge_kutta = num_methods.runge_kutta()
self.ax1.plot(x, y_runge_kutta, "green", label="Runge Kutta")
self.ax1.legend(loc='upper left')
# self.ax1.xaxis.set_major_locator(x_locator)
# self.ax1.yaxis.set_major_locator(y_locator)
local_errors = errors.Errors(self.NUMBERS)
y_local_euler = local_errors.local_error(y_graph, y_euler)
self.ax3.plot(x, y_local_euler, "red", label="local euler error")
self.ax3.legend(loc="upper left")
y_local_improved = local_errors.local_error(y_graph, y_improved_euler)
self.ax3.plot(x,
y_local_improved,
"blue",
label="local improved euler error")
self.ax3.legend(loc="upper left")
y_local_runge_kutta = local_errors.local_error(y_graph, y_runge_kutta)
self.ax3.plot(x,
y_local_runge_kutta,
"green",
label="local runge-kutta error")
self.ax3.legend(loc="upper left")
"""needed the second one because common locator for
3 graphs takes minimum y grid (what is too small
because of graph of approximation)
"""
# y_error_locator = mplt.ticker.MultipleLocator(base=self.Y_STEP)
# self.ax3.xaxis.set_major_locator(x_locator)
# self.ax3.yaxis.set_major_locator(y_error_locator)
hand, label = self.ax3.get_legend_handles_labels()
handout = []
lblout = []
for h, l in zip(hand, label):
if l not in lblout:
lblout.append(l)
handout.append(h)
self.ax3.legend(handout, lblout, loc='best')
self.figure.tight_layout()
self.canvas.draw()