def load_stk_ratio():
# Variables declaration
start = time.time()
file_path = os.path.join(commons.get_prop('base-path', 'ratio-input'))
files = [os.path.join(file_path, fn) for fn in next(os.walk(file_path))[2]]
all_pages = []
try:
for file in files:
read_lines = h.read_list_from_json_file(file)
all_pages.extend(read_lines)
# Total number of links to process
print("No of urls to process", len(all_pages))
page_bins = h.chunks(THREAD_COUNT, all_pages)
pool = ThreadPool(processes=THREAD_COUNT)
# use all available cores, otherwise specify the number you want as an argument
for link_array in page_bins:
pool.apply_async(process_pages,
args=(link_array, ),
callback=log_result)
pool.close()
pool.join()
for df_frames in result_list:
try:
result = pd.concat(df_frames, ignore_index=True)
if len(result) > 0:
df_columns = list(result)
table = "STK_PERF_HISTORY"
values = "to_date(%s, 'MONYY'), %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s"
constraint = ', '.join(['NAME', 'NSE_CODE', 'STK_YEAR'])
# create INSERT INTO table (columns) VALUES('%s',...)
insert_stmt = h.create_update_query(
table, df_columns, values, constraint)
curr, con = db.get_connection()
execute_batch(curr, insert_stmt, result.values)
con.commit()
db.close_connection(con, curr)
except Exception as err:
print("Exception while inserting data into table ", str(err))
except Exception as err:
print(str(err))
print("Execution time = {0:.5f}".format(time.time() - start))
def get_listed_ipo(stock_url):
jobs = []
spipe_list = []
# Variables declaration
start = time.time()
# Get the shares from chitt
urls = get_list_of_urls(stock_url)
# urls = ['http://www.chittorgarh.com/ipo/ipo_perf_tracker.asp', 'http://www.chittorgarh.com/ipo/ipo_perf_tracker.asp?FormIPOPT_Page=2', 'http://www.chittorgarh.com/ipo/ipo_perf_tracker.asp?FormIPOPT_Page=3', 'http://www.chittorgarh.com/ipo/ipo_perf_tracker.asp?FormIPOPT_Page=4']
print(urls)
cpdus = multi.cpu_count()
page_bins = h.chunks(cpdus, urls)
for cpdu in range(cpdus):
recv_end, send_end = multi.Pipe(False)
worker = multi.Process(target=process_page,
args=(page_bins[cpdu], send_end))
worker.daemon = True
jobs.append(worker)
spipe_list.append(recv_end)
worker.start()
for job in jobs:
job.join(timeout=10)
print("All jobs completed......")
try:
ipo_size = 0
result_list = [x.recv() for x in spipe_list]
curr, con = db.get_connection()
statement = create_update_query('IPO_STK_DETAILS')
for results in result_list:
for data in results:
values = [data[k] for k in data]
ipo_size += len(values)
df = get_data_frame(values)
records = df.to_dict(orient='records')
print(records)
execute_batch(curr, statement, df.values)
con.commit()
db.close_connection(con, curr)
print("IPOs listed so far = {}".format(ipo_size))
except Exception as err:
traceback.print_exc()
print(str(err))
print("Execution time = {0:.5f}".format(time.time() - start))
def get_stk_history():
jobs = []
# Variables declaration
start = time.time()
# Get the shares from yahoo
urls = get_yahoo_fin_urls()
# urls = ["https://in.finance.yahoo.com/quote/3IINFOTECH.NS/history?p=3IINFOTECH.NS", "https://in.finance.yahoo.com/quote/3IINFOTECH.NS/history?p=3IINFOTECH.NS", "https://in.finance.yahoo.com/quote/3IINFOTECH.NS/history?p=3IINFOTECH.NS", "https://in.finance.yahoo.com/quote/3IINFOTECH.NS/history?p=3IINFOTECH.NS"]
print("No of URLs = ", len(urls))
cpdus = multi.cpu_count()
page_bins = h.chunks(cpdus, urls)
for chunk in page_bins:
worker = multi.Process(target=process_page, args=(chunk,))
worker.daemon = True
jobs.append(worker)
worker.start()
for job in jobs:
job.join(timeout=10)
print("All jobs completed......")
print("Execution time = {0:.5f}".format(time.time() - start))