def resample_data(symbol, freqs, resample_method='default',
new_col_names=['Ticker','Open','High','Low','Close','Volume','TotalTrades']):
if 'default' == resample_method:
resample_rule = {'Ticker': lambda x: x.head(1),
'FirstTradePrice': lambda x: x.head(1),
'HighTradePrice': np.max,
'LowTradePrice': np.min,
'LastTradePrice': lambda x: x.tail(1),
'Volume': np.sum,
'TotalTrades': np.sum}
astype_dict = {'FirstTradePrice': 'str',
'HighTradePrice': 'str',
'LowTradePrice': 'str',
'LastTradePrice': 'str',
'Volume': 'str',
'TotalTrades': 'str'}
start_time = time.time()
print('Updating ' + symbol + '...')
for freq in freqs:
print(symbol + ' ' + freq)
# get latest date in stock_freq table
sm_f = StockMethodFreq(symbol, freq)
latest_date_freq = Utility().as_str(sm_f.get_latest_date())
print('latest_date_freq: ' + latest_date_freq)
# get latest date in algoseek table
sm_as = StockMethodAS(symbol)
latest_date_as = Utility().as_str(sm_as.get_latest_date())
print('latest_date_as: ' + latest_date_as)
if not latest_date_freq or latest_date_freq != latest_date_as:
sm = StockMethodAS(symbol)
if latest_date_freq: # if not None
stock_data = sm.query_by_symbol(start_date=latest_date_freq)
else:
stock_data = sm.query_by_symbol()
stock_data = Utility().DateTimeAsIndex(stock_data)
res = stock_data.resample(freq).apply(resample_rule).dropna()
res = res.astype(astype_dict)
res.columns = new_col_names
try:
db_con = DBConnection().db_sqlalchemy()
res.to_sql(name=(symbol + '_' + freq).lower(), con=db_con, if_exists='replace', index=True, index_label='DateTime')
except Exception as e:
print(e)
finally:
db_con.close()
else:
print(symbol + ' ' + freq + ' is skipped.')
end_time = time.time()
print('End of updating ' + symbol + '...')
print(end_time - start_time)
def generate_token():
if not request.json or not 'identifier' in request.json:
abort(400)
name = request.json['identifier']
room = request.json['roomName']
uti = Utility(name, room)
return jsonify({'accessToken': uti.generate_token().decode('utf-8')})
def insert_table(row):
"""
This function will create a row against a conversion, footfall, queue_wait_time.
"""
utility = Utility(log_file="big_data_process_log_file.txt", debug=1)
df_ = pd.DataFrame(columns=columns)
df_.loc[0] = row
# utility.loginfofile(str(df_))
# sql = "INSERT INTO BIG_DATA (date, time, image_key, face_id, type, gender, age, matched_grp_key, similarity, location, emotion, psngr_pnr, psngr_flight, flight_name, destination, time_grp, age_grp, valid_status) values {};".format((str(row[0]), str(row[1]), row[2], row[3], row[4], row[5], str(row[6]), row[7], str(row[8]), row[9], row[10], row[11], row[12], row[13],row[14], row[15], str(row[16]), row[17]))
try:
# print sql
# cursor.execute(sql)
df_.to_sql("BIG_DATA",
con=engine,
flavor=None,
schema=None,
if_exists='append',
index=False,
index_label=None,
chunksize=None,
dtype=None)
utility.loginfofile("row inserted in big_data")
except Exception, e:
utility.loginfofile("issue occurred while updating big data table " +
str(e))
# rollback of the transaction into the database
return False
def generate_report(unprocessed_data_list):
utility = Utility(log_file="big_data_process_log_file.txt", debug=1, db='Analytics')
for data in unprocessed_data_list:
image_key = data[0]
sql = "SELECT location, similarity, count(*) FROM BIG_DATA WHERE image_key = '{}' and device_type = '{}' GROUP BY location;".format(image_key,'footfall')
matched_location_list, matched_location_list_count = utility.query_database(sql)
if not matched_location_list_count:
print("passenger not found at any of the shops")
continue
for matched_location in matched_location_list_count:
similarity, location = matched_location[0], matched_location[1]
if int(similarity) < 85:
continue
location = "".join(location.split(" "))
try:
location_list_footfall[location] += 1
except:
print("appending new location in list")
location_list_footfall[location] = 1
#insert row into the database #cursor.execute('SELECT last_insert_id()')
print(location_list_footfall)
insert_data()
def get_psngr_flight_info(airline, airport):
"""
:param airline:
:param airport:
:return: passenger flight number
"""
sql_get_airport_code = "SELECT Airport_3LC FROM Airports_Info where City_Desc = '{}'".format(
airport)
utility = Utility(log_file="data_gen_process_log_file.txt", debug=1)
airport_code, airport_count = utility.query_database(sql_get_airport_code)
if airport_count == 0:
print("'{}': airport not served".format(airport))
return None
sql_get_airline_code = "SELECT Airline2LC FROM Airlines_Info where Description = '{}'".format(
airline)
airline_code, airline_count = utility.query_database(sql_get_airline_code)
if airline_count == 0:
print("'{};: airline service not available".format(airline))
return None
sql_get_flight_code = "SELECT FLNO3 FROM Flights_Info where Destination='{}' and FLNO3 LIKE '{}%'".format(
airport_code[0][0], airline_code[0][0])
psngr_flight, flight_count = utility.query_database(sql_get_flight_code)
if flight_count == 0:
print("{} not served to {}".format(airline, airport))
return None
return psngr_flight[0][0]
def check(indice,attrnames,attrtypes,attrvalues,conjunctions,operations):
utility = Utility()
error = []
if not set(conjunctions).issubset(set(['AND','OR'])):
error.append({'error':'Conjuntion must be AND/OR'})
if utility.CheckStockCode(indice) is False:
error.append({'error':'Wrong index code'})
else:
if not set(attrtypes).issubset(set(['string','date','number'])):
error.append({'error':'Type not in string/date/number'})
else:
if utility.CheckAttrNamesOfStock(indice,attrnames):
if utility.CheckAttrTypesOfStock(indice,attrnames,attrtypes,attrvalues):
#operation check
if set(operations).issubset(set(['equals','greater','lesser'])) and all(utility.CheckOperation(attrtypes[i],operations[i]) for i in range(len(attrtypes))):
pass
else:
error.append({'error':'Operation not in equals/greater/lesser.Note :String can have equals operation only'})
else:
error.append({'error':'Attribute type mismatch or value is null'})
else:
error.append({'error':'Attribute name doesnot exist'})
return error
def setUpClass(self):
self.utility = Utility()
# CHANGE THE LOG FILE NAME IN THE NEXT LINE******************************************************************************************
self.log = open(self.utility.logpath + "/WV-00.txt", "a+")
self.suite_start_time = time.time()
self.log.write("Suite started at {}\n".format(
str(time.ctime(int(self.suite_start_time)))))
self.url = URL()
self.loginPageStaticTexts = LoginPageStaticText()
self.loginPageTestData = LoginPageTestData()
self.configTestCase = configparser.RawConfigParser()
# CHANGE THE CONFIG PROPERTY FILE NAME IN THE NEXT LINE******************************************************************************************
self.configTestCase.read(
os.path.dirname(os.getcwd()) +
'/TestCases/WV_00_Config.properties')
self.configECG = configparser.RawConfigParser()
self.configECG.read(
os.path.dirname(os.getcwd()) +
'/Scripts/ECGRelatedData.properties')
self.configDevice = configparser.RawConfigParser()
self.configDevice.read(
os.path.dirname(os.getcwd()) +
'/Scripts/DeviceRelatedData.properties')
self.sendECG = SendECG()
yield
self.suite_end_time = time.time()
self.total_time_taken_suite = self.suite_end_time - self.suite_start_time
self.log.write("Suite ended at {}\n".format(
str(time.ctime(int(self.suite_end_time)))))
self.log.write(
"Total time taken by Test Suite to finish: {} seconds\n".format(
self.total_time_taken_suite))
self.log.close()
def process_queue_wait_time(log_id, date, time, image_key, face_id, image_type, gender, age, emotion, psngr_pnr, psngr_flight, flight_name, destination, time_grp, age_grp, processed_images):
utility = Utility(log_file="big_data_process_log_file.txt", debug=1)
utility.loginfofile("Initializing thread- " + threading.currentThread().getName() + ", type-queue_wait")
# queue_row_data = {"matched_grp_key":"", "similarity":"", "img_logdate":""}
for processed_image in processed_images:
if processed_image[5] != "queue_wait":
print("checking queue: {} found !!!!!!!!!".format(processed_image[5]))
continue
#check individual logdate must be greater group logdate
print("checking queue: {} found !!!!!!!!!".format(processed_image[5]))
utility.loginfofile("passenger found at queue")
device_type, ind_logdate, grp_logdate, similarity = processed_image[5], processed_image[6], processed_image[7], processed_image[2]
if (ind_logdate < grp_logdate):
print("Alert!!!!!!!!!!...............passenger found at queue after sha (sha:{}, queue:{})".format(str(ind_logdate), str(grp_logdate)))
continue
matched_grp_key, matched_devid, valid_status = processed_image[1], processed_image[3], "NA"
queue_wait_time = str((ind_logdate - grp_logdate).total_seconds())
try:
location = locations[matched_devid]
except:
location = ""
#now add row in the table
row = [str(date), str(time), image_key, face_id, image_type, gender, str(age), matched_grp_key,
str(similarity)[0:9], location, device_type, emotion, psngr_pnr, psngr_flight, flight_name, destination,
time_grp, str(age_grp), queue_wait_time, valid_status]
#insert_excel(row)
insert_table(row)
# update status with 5 in AP_ImageDate
check_analytic_true(log_id)
return None
def calc_happiness(H):
'Function to calculate happiness'
with open('./gifts_list.csv', 'r') as csvfile:
ffp = csv.reader(csvfile, delimiter=',')
gifts = []
for row in ffp:
if (row[3] == 'Luxury'):
gifts += [Luxury(row[0], int(row[1]), int(row[2]), row[3])]
elif (row[3] == 'Essential'):
gifts += [Essential(row[0], int(row[1]), int(row[2]), row[3])]
elif (row[3] == 'Utility'):
gifts += [Utility(row[0], int(row[1]), int(row[2]), row[3])]
csvfile.close()
gifts = sorted(gifts, key=lambda item: item.cost)
logging.warning('\n\nDetails of Gifts:\n')
for i in H:
if (i.boy.type_ == 'Miser'):
miser(gifts, i)
if (i.boy.type_ == 'Generous'):
generous(gifts, i)
if (i.boy.type_ == 'Geek'):
geek(gifts, i)
k = random.randint(1, len(H))
happy_couple(H, k)
def pax_flow_analyzer(img_date, img_time, image_key, matched_image_list):
utility = Utility(log_file="big_data_process_log_file.txt", debug=1)
utility.loginfo("Initializing ......... PaxFlowAnalyzer")
pax_location, pax_location_type, pax_key, pax_date, pax_sha_enter_time, pax_sha_exit_time = "SHA", "SHA", \
image_key, img_date, img_time, img_time
# passenger must be at sha
pax_sha = PaxFlow(pax_key, pax_date, str(pax_sha_enter_time),
str(pax_sha_exit_time), pax_location, pax_location_type)
# PaxFlow("F1XXCdf0000", "2018-01-23", "13:00:00", "21:00:00", "sha1", "sha")
pax_sha.add_pax_flow(utility)
del pax_sha
if len(matched_image_list) == 0:
return None
for pax in matched_image_list:
pax_similarity, pax_location, pax_location_type, pax_location_time = pax[
2], pax[3], pax[5], pax[7].time()
utility.loginfo("passenger({}) found at {}, type: {}".format(
pax_key, pax_location, pax_location_type, str(pax_location_time)))
# if pax_location_time == "footfall":
try:
print(
"passenger({}) found at {}, type: {}, capture time: {}".format(
pax_key, pax_location, pax_location_type,
str(pax_location_time)))
# passenger location changes , break passenger flow.
if pax_location != pax_flow.get("location"):
# add passenger flow in the database
pax_flow.add_pax_flow(utility)
# update passenger flow for new location
print(
"updating passenger flow for ({}) new location({})........., type: {}"
.format(pax_key, pax_location, pax_location_type))
pax_flow.set("location", pax_location)
pax_flow.set("location_type", pax_location_type)
pax_flow.set("enter_time", str(pax_location_time))
pax_flow.set("exit_time", str(pax_location_time))
continue
pax_exit_time = datetime.strptime(pax_flow.get("exit_time"),
'%H:%M:%S').time()
if pax_location_time > pax_exit_time:
pax_flow.set("exit_time", str(pax_location_time))
except NameError:
utility.loginfo(
"creating object for passenger({}) found at {}, type:{}".
format(pax_key, pax_location, pax_location_type))
pax_flow = PaxFlow(pax_key, str(pax_date), str(pax_location_time),
str(pax_location_time), pax_location,
pax_location_type)
# add current passenger flow in the db
pax_flow.add_pax_flow(utility)
del pax_flow
return None
def __init__(self, gridRes):
self.gridRes = gridRes
self.beamList = list()
self.gratingList = list()
self.result = list()
self.UTIL = Utility()
self.tempHologram = np.zeros((self.gridRes), dtype = complex)
self.numCols = 1
self.multiCols = False
def main():
if len(sys.argv)!=3:
print('python3 '+sys.argv[0]+' [MODEL FILE] [TEST FILE]', file = sys.stderr)
return
mod_path = sys.argv[1]
test_path = sys.argv[2]
util = Utility('', mod_path, test_path)
nbc = NBClassifier(False, True, False, util)
nbc.classify()
def Bundle_Adjustment_motion_refine_optimization(params):
Util = Utility()
Util.read_utility_parameters(params)
hmr_dict, data_dict = Util.load_hmr_data()
LR_cams = Util.load_camera_pkl()
LR_cams = LR_cams[0]
texture_img = cv2.imread(Util.texture_path +
"../../output_nonrigid/texture.png")
texture_vt = np.load(Util.texture_path + "vt.npy")
LR_path = Util.hmr_path + Util.refine_reference_path
LR_pkl_files = os.listdir(LR_path)
LR_pkl_files = sorted(
[filename for filename in LR_pkl_files if filename.endswith(".pkl")],
key=lambda d: int((d.split('_')[3]).split('.')[0]))
LR_length = len(LR_pkl_files)
LR_array = np.zeros((LR_length, 24 * 3))
LR_betas = np.zeros([LR_length, 10])
LR_trans = np.zeros([LR_length, 3])
HR_path = Util.HR_pose_path
HR_pkl_files = os.listdir(HR_path)
HR_pkl_files = sorted(
[filename for filename in HR_pkl_files if filename.endswith(".pkl")],
key=lambda d: int((d.split('_')[3]).split('.')[0]))
HR_length = len(HR_pkl_files)
HR_array = np.zeros((HR_length, 24 * 3))
for ind, LR_pkl_file in enumerate(LR_pkl_files):
LR_pkl_path = os.path.join(LR_path, LR_pkl_file)
with open(LR_pkl_path) as f:
param = pickle.load(f)
pose = param['pose'].squeeze()
beta = param['betas'].squeeze()
tran = param['trans'].squeeze()
for i in range(24 * 3):
LR_array[ind, i] = pose[i]
for i in range(10):
LR_betas[ind, i] = beta[i]
for i in range(3):
LR_trans[ind, i] = tran[i]
for ind, HR_pkl_file in enumerate(HR_pkl_files):
HR_pkl_path = os.path.join(HR_path, HR_pkl_file)
with open(HR_pkl_path) as f:
param = pickle.load(f)
pose = param['pose'].squeeze()
for i in range(24 * 3):
HR_array[ind, i] = pose[i]
hr_points = algorithms.get_hr_points(HR_array)
lr_points = algorithms.get_lr_points(LR_array, HR_array, Util.match_idx)
output = algorithms.periodicDecomp(LR_array, HR_array, lr_points,
hr_points)
#output = algorithms.refineLR(LR_array, HR_array, Util.match_idx)
refine_optimization(output, LR_betas, LR_trans, data_dict, hmr_dict,
LR_cams, texture_img, texture_vt, Util)
def __init__(self):
self._utility = Utility()
self._mongo_client = MongoClient(Configurations.MONGO_DB_HOST, int(Configurations.MONGO_DB_PORT))
print("Successfully connected to Mongo DB host: {0} and port: {1}".format(Configurations.MONGO_DB_HOST,
str(Configurations.MONGO_DB_PORT)))
self._create_db_and_collections_if_not_exist()
self._create_indexes()
self._get_all_manufacturers()
self._get_all_models()
self._time_format = "%Y-%m-%d"
def __init__(self, p, l, beamWidth):
self.p = p
self.l = l
self.beamWidth = beamWidth
self.SquareRoot2 = math.sqrt(2)
self.PI = math.pi
self.LAG = Laguerre()
self.UTIL = Utility()
self.w = 0.12077 ############################## still need to change this
def __init__(self):
self.newHolo = Hologram((1920, 1080))
self.UTIL = Utility()
print(
"Input the number of required columns\n input l, p and/or other variables in the following format"
)
print("pass the matrix via obj.passBeams()")
print("pass in the gratings with obj.passfGratings()")
print("generate beams with obj.generateHologram()")
#self.blaze = Blazed(0, 0)
self.display = list()
def __init__(self):
if len(sys.argv) < 2:
print(
"Please provide the testcase filepath as a command line argument"
)
return
input = Utility().read_file(sys.argv[1])
s = Solution_test(input)
s.output_vector()
print("Your solution")
print("==========================================")
print(s.output_vector())
def create_device_mapper():
"""
:return: dictionary of device mapping
"""
utility = Utility(log_file="data_gen_process_log_file.txt", debug=1)
sql = "SELECT DeviceName, DeviceType, DeviceLocation FROM DeviceMapper"
mapped_device_list, device_count = utility.query_database(sql)
if device_count == 0:
print("no device registered in the database")
return None
for mapped_device in mapped_device_list:
print mapped_device
return None
def __init__(self):
if len(sys.argv) < 2:
print(
"Please provide the testcase filepath as a command line argument"
)
return
input = Utility().read_file(sys.argv[1])
s = Solution(len(input.women_preferences), input.women_preferences,
input.men_preferences)
sol = s.output_stable_matchings()
print(len(sol))
for m in sol:
print(m)
def __init__(self):
if len(sys.argv) < 2:
print(
"Please provide the testcase filepath as a command line argument"
)
return
input = Utility().read_file(sys.argv[1])
#print(input)
s = Solution(len(input.women_preferences), input.women_preferences,
input.men_preferences)
sol = s.output_stable_matchings()
for match in sol: #Sort by women
match.sort()
sol.sort() #sorting across differnt matchings
def __init__(self):
if len(sys.argv) < 2:
print(
"Please provide the testcase filepath as a command line argument"
)
return
input = Utility().read_file(sys.argv[1])
s = Solution(len(input.hospital_preferences),
len(input.student_preferences),
input.hospital_preferences, input.student_preferences,
input.hospital_open_slots)
solution = s.get_matches()
#print("Your Solution")
#print("=============================================")
print(solution)
def main():
global applicants, cache
bed, parking_spot, spla_list, lahsa_list, applicants = ReadWriteFile.read_input_file(
"../grading_case/input22.txt")
SPLA = Utility(bed)
# Modify SPLA utility for already selected applicants by SPLA
for i in spla_list:
applicants[int(i) - 1].selected_by_spla = True
SPLA.modify(applicants[int(i) - 1])
LAHSA = Utility(parking_spot)
# Modify LAHSA utility for already selected applicants by LAHSA
for i in lahsa_list:
applicants[int(i) - 1].selected_by_lahsa = True
LAHSA.modify(applicants[int(i) - 1])
cache = {}
leftover_applicant = []
# creating separate lists depending on
LAHSA_applicant_list = []
SPLA_applicant_list = []
common_applicant_list = []
for a in applicants:
if a.selected_by_lahsa or a.selected_by_spla:
continue
if a.valid_lahsa and a.valid_spla:
common_applicant_list.append(a.id)
else:
if a.valid_spla:
SPLA_applicant_list.append(a.id)
elif a.valid_lahsa:
LAHSA_applicant_list.append(a.id)
SPLA_best_score, LAHSA_best_score, best_applicant = game_play_spla(
common_applicant_list, SPLA_applicant_list, LAHSA_applicant_list, SPLA,
LAHSA)
ReadWriteFile.create_output_file(best_applicant)
print best_applicant
print "Final Value %s %s" % (SPLA_best_score, LAHSA_best_score)
def k_means(self, df, n_clusters):
'''
Returns the label encoded clusters for each example in the df.
'''
df = Utility().normalize(df)
kmeans = KMeans(n_clusters = n_clusters)
kmeans.fit(df)
print("Reduced inertia:", kmeans.inertia_)
print("Clusters centers:")
display(pd.DataFrame(kmeans.cluster_centers_, columns = df.columns,
index = ["cluster %i" %i for i in np.arange(n_clusters)]))
centers = pd.DataFrame(kmeans.cluster_centers_, columns = df.columns, index =["cluster %i" %i for i in np.arange(n_clusters)])
self.plot_cluster_centers(centers)
# return Utility().label_encode(kmeans.labels_)
return kmeans.labels_
def check(indice,subscribeFieldlist,triggermessage):
utility = Utility()
error = []
if not triggermessage:
error.append({'error':'Message cannot be empty'})
if utility.CheckStockCode(indice) is False:
error.append({'error':'Wrong index code'})
else:
if not utility.CheckAttrNamesOfStock(indice,subscribeFieldlist):
error.append({'error':'Field name doesnot exist'})
return error
def __init__(self):
self.util = Utility()
self.screen = self.util.screen
self.tkinterRoot = tkinter.Tk()
self.tkinterRoot.withdraw() #Hides the tkinter window that pops up
self.qA = QualityAppraiser(clientId, clientSecret)
self.qualityAppraiser = self.qA.get_token()
self.url = 'https://api.everypixel.com/v1/quality'
self.filedialog = filedialog
self.eventDict = self.get_event_dict()
self.userPath = None
self.waitCursor = pygame.cursors.broken_x
self.arrowCursor = pygame.cursors.arrow
def get_resample_rt_data(self, resample_rule="default"):
data = Utility().DateTimeAsIndex(self.get_real_time_data())
if "default" == resample_rule:
resample_rule = {'Ticker': lambda x: x.head(1),
'Open': lambda x: x.head(1),
'High': np.max,
'Low': np.min,
'Close': lambda x: x.tail(1),
# 'Volume': np.sum}
'Volume': lambda x: np.sum(x)*100}
resampled_data = data.resample(self.freq).apply(resample_rule).dropna()
# index as column
resampled_data = resampled_data.reset_index(drop=False)
return resampled_data
def check_analytic_true(img_id):
while True:
try:
utility = Utility(log_file="big_data_process_log_file.txt", debug=1)
print("connected to mysql db")
break
except:
print("mysql server gone....... trying again")
sql = "Update AP_ImageData SET status = {} where ID = '{}';".format(5, img_id)
flag = utility.update_database(sql)
if flag:
utility.loginfofile("Quitting thread after updating status- " + str(threading.currentThread().getName()))
return True
else:
utility.loginfofile("Database query Failed while updating status for thread " + str(threading.currentThread().getName()) + str(e))
return False
def setUpClass(request):
print("setUpClass started")
utility = Utility()
utility.createLogFolder()
log = open(utility.logpath + "/WV-00.txt", "a+")
suite_start_time = time.time()
log.write("Suite started at {}\n".format(
str(time.ctime(int(suite_start_time)))))
loginPageStaticTexts = LoginPageStaticText()
loginPageTestData = LoginPageTestData()
configTestCase = configparser.RawConfigParser()
configTestCase.read(
os.path.dirname(os.getcwd()) + '/TestCases/WV_00_Config.properties')
configECG = configparser.RawConfigParser()
configECG.read(
os.path.dirname(os.getcwd()) + '/Scripts/ECGRelatedData.properties')
configDevice = configparser.RawConfigParser()
configDevice.read(
os.path.dirname(os.getcwd()) + '/Scripts/DeviceRelatedData.properties')
sendECG = SendECG()
request.cls.log = log
request.cls.utility = utility
request.cls.suite_start_time = suite_start_time
request.cls.loginPageStaticTexts = loginPageStaticTexts
request.cls.loginPageTestData = loginPageTestData
request.cls.configTestCase = configTestCase
request.cls.configECG = configECG
request.cls.configDevice = configDevice
request.cls.sendECG = sendECG
print("setUpClass END")
yield
print("setUpClass_yield")
suite_end_time = time.time()
total_time_taken_suite = suite_end_time - suite_start_time
log.write("Suite ended at {}\n".format(str(time.ctime(
int(suite_end_time)))))
log.write("Total time taken by Test Suite to finish: {} seconds\n".format(
total_time_taken_suite))
log.close()
def DB_Execute_Sql_Parames(self,
Sql,
Params,
host=_Host,
user=_User,
password=_Password):
u"""
Default Value:
user = '******' passwrod = 'derbysoft'
Examples:
When gizp is a File
| @{aaa}= | Test Data | maplist.xml |
| DB_Execute_Sql_Parames | INSERT INTO marriott_ari.`process` (check_in, create_time, hotel, los_rs,process_key,rate_plan_candidate,rate_plan_candidate_value) VALUES (%s, %s, %s, %s, %s, %s, %s) | '2017-08-05', '2017-07-07 06:23:56', 'HNLOW',r'gzip@{aaa}[0]', 'HNLOW:2017-08-05:CATEGORY:DN1', 'CATEGORY', 'DN1' |
When gizp is a String
| @{aaa}= | Test Data | maplist.xml |
| ${f} | get file | @{aaa}[0] |
| DB_Execute_Sql_Parames | INSERT INTO marriott_ari.`process` (check_in, create_time, hotel, los_rs,process_key,rate_plan_candidate,rate_plan_candidate_value) VALUES (%s, %s, %s, %s, %s, %s, %s) | '2017-08-05', '2017-07-07 06:23:56', 'HNLOW','''gzip${f}''', 'HNLOW:2017-08-05:CATEGORY:DN1', 'CATEGORY', 'DN1' |
"""
try:
conn = mysql.connector.connect(user=user,
password=password,
host=_Host)
cur = conn.cursor()
params = ast.literal_eval(Params)
res = []
for p in params:
if "gzip" in p:
p = p.replace('gzip', '')
u = Utility()
p = u.Compress_Gzip(p)
res.append(p)
cur.execute(Sql, res)
conn.commit()
dlog.write("DB_Execute_Sql_Parames is successfully")
except Exception, e:
dlog.fail(e)
def main():
utility = Utility(log_file='big_data_process_log_file.txt', debug=1, db='Analytics')
sql = "SELECT ReportID FROM CR_Reports WHERE {}".format(filter_cr_reports)
print sql
report_list, report_list_count = utility.query_database(sql)
print report_list, report_list_count
if report_list_count:
print("report already processed")
return None
sql_1 = "SELECT distinct(image_key) FROM BIG_DATA WHERE {}".format(filter_big_data)
print sql_1
unprocessed_data_list, unprocessed_data_list_count = utility.query_database(sql_1)
print unprocessed_data_list_count
if unprocessed_data_list_count:
generate_report(unprocessed_data_list)
