def calculate_match_score(target_productID, product_2, feature_matchList,
feature_scoreList):
final_score = 0
brand_score = 0
price_score = 0
feature_match_score = 0
product_info_1 = find_product_info_by_productID(target_productID)
product_info_2 = find_product_info_by_productID(product_2)
if product_info_1[0] == product_info_2[0]:
brand_score = 1
if float(product_info_1[1]) != 0 and float(product_info_2[1]) != 0:
if abs(float(product_info_1[1]) - float(product_info_2[1])) / float(
product_info_1[1]) < 0.2:
price_score = 1
product_2_featureList = product_info_2[2].split(';')
if len(MF.list_intersection(feature_matchList, product_2_featureList)) > 0:
for iNumber in range(len(feature_matchList)):
if feature_matchList[iNumber] in product_2_featureList:
feature_match_score = feature_match_score + feature_scoreList[
iNumber]
feature_match_score = feature_match_score / len(
MF.list_intersection(feature_matchList, product_2_featureList))
final_score = (brand_score + price_score) / 2 + feature_match_score * 2
return final_score
def codeToRun():
myPrint("Hello world!")
MyFunctions.add(4, 5)
i = 2
myPrint(i)
MyFunctions.subtract(4, 5)
myPrint("On the Internet nobody knows you're a dog.")
def codeToRun():
myPrint("Hello world!")
MyFunctions.add(4, 5)
i = 2
myPrint(i)
MyFunctions.subtract(4, 5)
myPrint("On the Internet nobody knows you're a dog.")
def run(folderpath, read_filepath, write_filepath):
'''Aggregate all functions and run this module
folderpath: VisitAllFile(folderpath)
'''
all_poi_normal_num = {}
all_poi_normal = getStations_Inorder()
for i in service:
all_poi_normal_num[i] = len(all_poi_normal[i])
# print('all_poi_normal_num: ', all_poi_normal_num)
df_row = pd.DataFrame()
all_record = {}
for filePath in VisitAllFile(folderpath)[0]:
dir_filePath = '/'.join([folderpath, filePath])
tmp_df = match_Service(dir_filePath, all_poi_normal, all_poi_normal_num)[0]
df_row = pd.concat([df_row, tmp_df], axis = 1)
date = '-'.join(filePath.split('-')[1:])
result_path = ''.join(['Final_Result/', 'test-dfrow-', date])
MyFunctions.checkPath(result_path)
df_row.to_csv(result_path)
service_time_pair = match_Service(dir_filePath, all_poi_normal, all_poi_normal_num)[1]
all_record[filePath.split('-')[0]] = service_time_pair
# pprint.pprint(all_record) #{2023:[BTC1, start_row, stop_row, BTC2, start_row, stop_row, ....], 2024:...}
#Last step to write final result on new_raw data
tagService_RawData(read_filepath, write_filepath, all_record)
def run_RTT(folderpath):
'''Run
Arguments:
filepath {[String]} -- [eg: '../output/2017-09-04/final_2017-09-04.csv']
'''
fileName = MyFunctions.visitAllFile(folderpath)
if '.DS_Store' in fileName[0]:
fileName[0].pop(fileName[0].index('.DS_Store'))
print('Modified file list is: ', fileName[0])
print('------ There are %d files ------' % len(fileName[0]))
count = 1
for item in fileName[0]:
filepath = folderpath + '/' + item
print('This is %d file out of %d' % (count, len(fileName[0])))
nodeid_duration = calculateDuration(filepath)
count += 1
# filepath_nor = 'Mobility/mobility_2017-09-05.csv'
# normalizeDuration(filepath_nor)
filename = 'mobility_' + filepath.split('/')[-1].split('_')[1]
newpath = '../mobility/' + filename
print(newpath)
MyFunctions.checkPath(newpath)
writeMobility(filepath, newpath, nodeid_duration)
def seperate_vehicle(filepath, nodeid_list):
'''
filepath: path of the table with marked POI after Geo Fence and calculation
'''
headers = [
'node_id', 'vehicle_serial', 'gps_time', 'latitude', 'longitude',
'altitude', 'speed', 'heading', 'POI'
]
for vehicleid in nodeid_list:
with open(filepath, 'r') as f_vehicle: #with打开不用在意close
r_vehicle = csv.reader(f_vehicle)
headings_v = next(r_vehicle)
# print(headings_v)
Vehicle = namedtuple('Vehicle',
headings_v) #用namedtuple 方便后面用headings来读取数据
newfilename = '-'.join([vehicleid,
filepath[14:]]) #根据不同的车辆号码输入创建不同的文件名
newpath = '/'.join(
['fenced_vehicle', filepath[14:24], newfilename])
print(newpath)
MyFunctions.checkPath(newpath) #checkPath 检查文件路径是否正确，不正确的话就进行创建
with open(
newpath, 'w', newline=''
) as fnew: #use 'w' for writing str and newline='' for deleting extra idle row
w_vehicle = csv.writer(fnew)
w_vehicle.writerow(headers)
for row_v in r_vehicle:
# vehicle = Vehicle._make(row_v)
if row_v[0] == vehicleid:
w_vehicle.writerow([
row_v[0], row_v[1], row_v[2], row_v[3], row_v[4],
row_v[5], row_v[6], row_v[7], row_v[8]
])
def handleOneFile(read_filepath, service):
all_poi_normal = getAllPOI(service)[0]
all_POI = getAllPOI(service)[1]
POI_services = getAllPOI(service)[2]
# pprint.pprint(POI_services)
# Get all info about POI_services_interval = {} # {YIH: [{A1:[T1, T2, T3]}, {A2:[T4, T5, T6]}, {B1:[T7, T8, T9]}]}
# POI_services_start = {} # {YIH: [{A1:[S1, S2, S3]}, {A2:[S4, S5, S6]}, {B1:[S7, S8, S9]}]}
for poi in all_POI:
print('POI------: ', poi)
POI_services_interval[poi] = [] # Initialize for different poi
POI_services_start[poi] = []
start_row_num[poi] = [
] # {YIH: [{A1:[row1, row2, row3]}, {A2:[row1, row2, row3]}, {B1:[row1, row2, row3]}]}
for service in POI_services[poi]:
# use 'start_list'(dict) to record the start of every service for a particular poi
start_list = {}
start_list[service] = []
service_ss = recordStartStop(read_filepath, poi, service)[0]
start_row_num[poi].append(
recordStartStop(read_filepath, poi, service)[1])
final_POI_services_interval = calculateDuration(
poi, service, service_ss, POI_services_interval)
# write the interval information into a file
newname = 'interval_' + read_filepath.split('/')[-1].split('_')[
1] # interval_2017-09-04.csv'
write_filepath = '../interval' + '/' + newname # '../interval/interval_2017-09-04.csv'
print(write_filepath)
MyFunctions.checkPath(write_filepath)
print('Writing. Please wait...')
writeIntervalInfo(read_filepath, write_filepath,
final_POI_services_interval, start_row_num)
def find_info_by_ID(productID):
outputlist = []
sql_string = 'select product_title, product_brand, product_gender, product_price, product_desc, product_details from testdb.product where idproduct = %s'
cursor.execute(sql_string, productID)
fetch_result = cursor.fetchone()
if fetch_result is not None:
title = fetch_result[0].strip().lower()
outputlist.append(title)#title
brand = fetch_result[1].strip().lower()
outputlist.append(brand)#brand
outputlist.append(fetch_result[2])#gender
price = re.findall(r'[\d,.]+', fetch_result[3])[0]
outputlist.append(price.strip())#price
descriptionList = MF.extract_nouns_from_string(fetch_result[4])#description
description = MF.list_to_string(descriptionList).lower()
outputlist.append(description)#description
colour = ''
if len(re.findall(r'Colour%[\w]+', fetch_result[5]))>0:
colour = re.findall(r'Colour%[\w ]+', fetch_result[5])[0].replace('Colour%','')
outputlist.append(colour.lower()) # description
return outputlist
def clean_data():
'''
Cleans the data frame to prepare it for analysis.
'''
#Trim data table
useful_columns = ['BusinessUnit','eventOccurredDate','companyInvolved','operationOrDevelopment'
,'jobTypeObserved','event','eventClassification','eventType','stopJob'
,'immediateActionsTaken','actionCompletedOnsiteDetail','furtherActionNecessaryComments','rigInvolved']
df_trimmed = df.copy()
df_trimmed = df_trimmed[useful_columns]
#Clean data table
df_trimmed[['rigInvolved']] = df_trimmed[['rigInvolved']].fillna(value='No')
df_trimmed[['stopJob']] = df_trimmed[['stopJob']].fillna(value='No')
df_trimmed.loc[(df_trimmed['immediateActionsTaken'] == 'Action Completed Onsite'),'furtherActionNecessaryComments'] = np.nan
df_trimmed.loc[(df_trimmed['immediateActionsTaken'] == 'Further Action Necessary'),'actionCompletedOnsiteDetail'] = np.nan
df_trimmed['comments'] = df_trimmed.actionCompletedOnsiteDetail.combine_first(df_trimmed.furtherActionNecessaryComments)
df_trimmed[['comments']] = df_trimmed[['comments']].fillna(value='None')
df_trimmed = df_trimmed.drop(['actionCompletedOnsiteDetail', 'furtherActionNecessaryComments'], axis=1)
df_trimmed = df_trimmed[df_trimmed.companyInvolved.isnull() == False]
df_trimmed = df_trimmed[df_trimmed.jobTypeObserved.isnull() == False]
df_trimmed = df_trimmed[df_trimmed.immediateActionsTaken.isnull() == False]
df_trimmed = df_trimmed[df_trimmed.BusinessUnit.isnull() == False]
df_trimmed.loc[(df_trimmed['companyInvolved'] != 'BP'),'companyInvolved'] = 'Other'
df_trimmed.event.unique()
##categories can be considered counts over time. Which could then become ratios.
categories = ['event','eventClassification']#,'companyInvolved','operationOrDevelopment','jobTypeObserved','stopJob','immediateActionsTaken','rigInvolved']
dfBUMatrix = pd.get_dummies(df_trimmed,columns = categories)
useful_columns = ['BusinessUnit','eventOccurredDate','event_Observation','event_Incident']
dfBUMatrix = dfBUMatrix[useful_columns]
dfBUMatrix = dfBUMatrix[dfBUMatrix['BusinessUnit'] != 'Central Support']
dfBUMatrix['eventOccurredDate'] = pd.to_datetime(df['eventOccurredDate']).dt.date
dfBUMatrix['eventOccurredDate'] = pd.to_datetime(dfBUMatrix['eventOccurredDate'])
dfBUMatrix = dfBUMatrix.groupby([dfBUMatrix.eventOccurredDate,dfBUMatrix.BusinessUnit]).sum()
dfBUMatrix = dfBUMatrix.reset_index()
dfBUMatrix.sort_values(by = ['eventOccurredDate'])
BUList = dfBUMatrix.BusinessUnit.unique()
#Remove Central Support due to only having a few entries.
BUList = BUList[BUList != 'Central Support']
#Create dfDates dataframe based on max and min values in main dataframe.
end_date = dfBUMatrix['eventOccurredDate'].max()
start_date = dfBUMatrix['eventOccurredDate'].min()
dfDates = MyFunctions.create_dates_dataframe(start_date,end_date)
#Grab the distinct list of BU's.
dfBUList = pd.DataFrame(BUList,columns = ['BusinessUnit'])
#Spread the dates across all BU's.
dfCounts = MyFunctions.dataframe_crossjoin(dfDates, dfBUList)
#Spread the counts across all dates even when you have zero for the count.
dfFinal = pd.merge(dfCounts, dfBUMatrix, left_on=['DateKey','BusinessUnit'],right_on=['eventOccurredDate','BusinessUnit'], how='left')
dfFinal = dfFinal.fillna(value = 0)
dfFinal.drop('eventOccurredDate',axis = 1,inplace = True)
return dfFinal
def doProcessing(self):
MyFunctions.processing_rule = ModelConfig.buy_or_sell # Yes - this is very messy...
for i in range(0, len(infile_array)):
infile_array[i] = fileprefix + infile_array[i]
print(infile_array[i])
while True:
# modelDict = MyFunctions.read_row(ModelConfig.datafile)
modelDict = MyFunctions.read_from_from_db(ModelConfig.db_read_sort)
if modelDict == None:
break
try:
model = MyLSTMModelV2b(modelDict)
model.myf.model_description = 'LSTMMModelV2b_ATR3_Beta98 ' + ModelConfig.buy_or_sell + model.myf.dict_to_description(
modelDict) + ModelConfig.opt
model.myf.default_optimizer = ModelConfig.opt
model.model.summary()
if ModelConfig.buy_or_sell[:3] == 'Buy':
rule_column = 'Buy'
else:
rule_column = 'Sell'
model.myf.parse_process_plot_multi_source(
infile_array,
rule_column + "WeightingRule",
model.model,
model.myf.model_description,
version=2)
# model.myf.finish_update_row(ModelConfig.datafile, modelDict)
model.myf.db_update_row(
modelDict
) # Use the myf from the model to make sure we get the relevant global values. This may explain some strange behaviour with updates not working...
if model.myf.model_best_loss < 1.5:
model.myf.save_model(
model.model,
str(modelDict['unique_id']) + '_' +
str(modelDict['Layers']) + '_Layers_' +
ModelConfig.buy_or_sell + '.h5')
except:
print("Oops!", sys.exc_info()[0], "occurred.")
print('Occurred with configDict:')
print(modelDict)
modelDict['ErrorDetails'] = sys.exc_info()[0]
# MyFunctions.finish_update_row(ModelConfig.datafile, modelDict, False)
MyFunctions.db_update_row(modelDict, False)
def plot_occ_all(self, dfs, n=5, scale=1.2, height=4):
for x, df in enumerate(dfs[0:n]):
L = np.floor(len(df)/24)*scale
ax = df.plot(y=name, title = self.name, figsize = (L,height), legend=False)
ax = self.highlight_weekend(self.days, df, ax)
save_dir = my.make_storage_directory(os.path.join(self.write_dir, 'Occ_Figs'))
plt.savefig(os.path.join(save_dir, f'{self.name}_{x}.png'))
def test_something(self):
# Let's load a known file, parse it, and then validate it is as expected
trainX, testX, trainY, testY, trainYRnd, testYRnd = MyFunctions.parse_data_to_trainXY_plusRandY ("DAX_Prices_WithMLPercentages.csv", "BuyWeightingRule")
print("TrainX:")
print(trainX)
print("TestX")
print(testX)
print("TrainY:")
print(trainY)
print("TestY:")
print(testY)
print("trainXRnd")
print(trainYRnd)
print("testYRnd")
print(testYRnd)
print("Shapes")
print(trainX.shape)
print(testX.shape)
print(trainY.shape)
print(testY.shape)
print(trainYRnd.shape)
print(testYRnd.shape)
self.assertEqual(len(trainY), len(trainYRnd), "TestX Length")
self.assertEqual(len(testY), len(testYRnd), "TestX Length")
def Prob_23(n): abundant = [] allnum = [] total = 395465626 for i in range(12,n): divisor = [] isPrime = MyFunctions.Prime(i) if isPrime == False: for j in range (1,i): z = i % j if z == 0: divisor.append(j) if sum(divisor) > i: abundant.append(i) #if i == 945: #print divisor, ' ',sum(divisor) #if sum(divisor)<i: #print i, ' is deficient, its sum is ',sum(divisor),' ',divisor #elif sum(divisor) == i: #print i, ' is perfect, its sum is ',sum(divisor),' ',divisor #else: #print i, ' is abundant, its sum is ',sum(divisor),' ',divisor #abundant.append(i) #print abundant #print len(abundant) for x in range(1,28124): allnum.append(x) #print allnum print sum(allnum) return total
def Prob_27(): for a in range(-3,4): for b in range(-3,4): sw = 1 n = 0 while (sw==1): #print(n,"^2 + ",a,n," + ",b) x = n*n + a*n + b if (x>0): isPrime = MyFunctions.prime(x) print(a,b,n,x,isPrime) n+=1 if(isPrime == 0): sw = 0 else: break #print (a,b,n,x) #if (x<0): #number = MyFunctions.prime(-x) #print(a,b,n,-x,number) #else: #number = MyFunctions.prime(x) #print(a,b,n,x,number) #if number == True: #n+=1 #else: #break return ""
def Prob_10():
ans = 17
for x in range(8, 2000000):
isPrime = MyFunctions.Prime(x)
if isPrime == 1:
ans = ans + x
return ans
def calculateDuration(poi, service, service_ss, POI_services_interval):
'''calculate the duration according to start_time and stop_time
Arguments:
start_time {[str]} -- [description]
stop_time {[str]} -- [description]
POI_services_interval = {} # {YIH: [{A1:[T1, T2, T3]}, {A2:[T4, T5, T6]}, {B1:[T7, T8, T9]}]}
service_interval = {} # {service: [interval1, interval2, ...]}
'''
service_interval = {}
service_interval[service] = []
index = 0 # index in the service_ss value list
while (index < len(service_ss[service]) - 1):
last_leave_time = service_ss[service][index][1]
# print(last_leave_time)
next_arrive_time = service_ss[service][index + 1][0]
# print(next_arrive_time)
interval = MyFunctions.calculateDuration(last_leave_time,
next_arrive_time)
service_interval[service].append(interval)
index += 1
POI_services_interval[poi].append(service_interval)
# print(POI_services_interval)
# print('Successfully generate POI_services_interval')
return (POI_services_interval)
def write_occ(self, occ_probs, fname, cols):
store_dir = my.make_storage_directory(os.path.join(self.write_dir, 'TransitionProbabilties'))
csv_file = os.path.join(store_dir, fname)
with open(csv_file, 'w') as f:
f.write(f'{cols[0]}, {cols[1]}, {cols[2]}'+'\n')
for hour in occ_probs:
f.write(f'{hour}:00, {occ_probs[hour][0]:.4f}, {occ_probs[hour][1]:.4f}\n')
print(csv_file + ': Write Successful!')
def calculate_match_score_for_each_product(target_productInfoList, productID,
feature_matchList,
feature_scoreList):
final_score = 0
brand_score = 0
price_score = 0
color_score = 0
feature_match_score = 0
match_product_info = find_product_info(productID)
# sequence is idproduct, breadcrumbs, title, brand, gender, price, description, colour, category, feature
if target_productInfoList[3] == match_product_info[3]:
brand_score = 1
if colour_similarity(target_productInfoList[7],
match_product_info[7]) == 1:
color_score = 1
if float(
match_product_info[5]
) > float(target_productInfoList[5]) * 0.8 and float(
match_product_info[5]) < float(target_productInfoList[5]) * 1.2:
price_score = 1 - abs(
float(match_product_info[5]) - float(target_productInfoList[5])
) / float(target_productInfoList[5])
if len(feature_matchList) > 0:
match_product_featureList = match_product_info[9].split(';')
if len(
MF.list_intersection(feature_matchList,
match_product_featureList)) > 0:
scorelist = []
for iNumber in range(len(feature_matchList)):
if feature_matchList[iNumber] in match_product_featureList:
scorelist.append(feature_scoreList[iNumber])
feature_match_score = sum(scorelist) / len(
MF.list_intersection(feature_matchList,
match_product_featureList))
final_score = (brand_score * 0.5 + price_score * 0.3 +
color_score * 0.2) * feature_match_score
else:
final_score = brand_score * 0.5 + price_score * 0.3 + color_score * 0.2
return final_score
def Prob_35(n):
count = 4
for i in range (10,n):
isPrime = MyFunctions.Prime(i)
sw = 1
if isPrime == True:
number = map(str,str(i))
perm = permutations(number)
for j in list(perm):
magic = lambda number: int(''.join(str(j) for j in number)) # Generator exp.
newnum = magic(j)
isPrime2 = MyFunctions.Prime(newnum)
if isPrime2 == True:
continue
else:
sw = 0
if sw == 1:
count += 1
return count
def updateProductInfo(productID, infoList1, infoList2):
print(productID)
infoList = infoList1.replace(',', '')
deslist = re.findall(r'[a-z]{3,}\b|[\u4e00-\u9fa5]+', infoList2)
descritpion = MF.list_to_string(deslist)
sql_string = 'UPDATE clean_data_zalora.product SET price = %s, description = %s WHERE idproduct = %s'
data = (infoList, descritpion, productID)
cursor.execute(sql_string, data)
db.commit()#需要这一句才能保存到数据库中
def main(self):
results_files = my.mylistdir(self.results_file_path, bit='.csv')
df1 = self.read_in_results(results_files[0])
if len(results_files) > 1:
for f in results_files[1:]:
dfx = self.read_in_results(f)
df1 = pd.concat([df1, dfx], axis=0)
self.df = df1
self.TransitionMatrix = self.get_dist(self.df)
self.len_df = len(self.TransitionMatrix)
def calculate_match_score_for_each_product(target_productInfoList, productID, feature_matchList, feature_scoreList):
final_score = 0
brand_score = 0
price_score = 0
color_score = 0
feature_match_score = 0
match_product_info = find_product_info(productID)
# sequence is idproduct, breadcrumbs, title, brand, gender, price, description, colour, category, feature
if target_productInfoList[3] == match_product_info[3]:
brand_score = 1
if colour_similarity(target_productInfoList[7], match_product_info[7]) == 1:
color_score = 1
if brand_score == 0 and color_score == 0:
final_score
mapping_price_target_product, price_level_target = find_mapping_product_price(target_productInfoList)
mapping_price_match_product, price_level_match = find_mapping_product_price(match_product_info)
if price_level_target == 0 or price_level_match == 0:
price_score = 1 - abs(float(match_product_info[5]) - float(target_productInfoList[5]))/float(target_productInfoList[5])
if price_level_target == price_level_match:
if mapping_price_target_product == mapping_price_match_product:
price_score = 1
elif abs(mapping_price_target_product-mapping_price_match_product) == 20 or \
abs(mapping_price_target_product-mapping_price_match_product) == 33:
price_score = 0.5
else:
if abs(mapping_price_target_product-mapping_price_match_product) == 12 or \
abs(mapping_price_target_product-mapping_price_match_product) == 6 or \
abs(mapping_price_target_product-mapping_price_match_product) == 1:
price_score = 1
if abs(mapping_price_target_product-mapping_price_match_product) == 7 or \
abs(mapping_price_target_product-mapping_price_match_product) == 26 or \
abs(mapping_price_target_product-mapping_price_match_product) == 14:
price_score = 0.5
match_product_featureList = match_product_info[9].split(';')
if len(list(set(match_product_featureList)&set(feature_matchList))) > 0:
if len(MF.list_intersection(feature_matchList, match_product_featureList)) > 0:
scorelist = []
for iNumber in range(len(feature_matchList)):
if feature_matchList[iNumber] in match_product_featureList:
scorelist.append(feature_scoreList[iNumber])
feature_match_score = sum(scorelist)/len(match_product_featureList)
final_score = (brand_score*0.5 + price_score*0.3 + color_score*0.2)*feature_match_score
else:
final_score = brand_score*0.5 + price_score*0.3 + color_score*0.2
return final_score
def main(UD: UploadDetails) -> str:
(startUTCstr, endUTCstr, videoID, videoName, event, outputContainer,
outputBlobStorageAccount, imagesCreatedCount) = UD
startUTC = datetime.strptime(startUTCstr, "%Y-%m-%d %H:%M:%S.%f")
# endUTC = datetime.strptime(endUTCstr,
# "%Y-%m-%d %H:%M:%S.%f")
endUTC = datetime.now()
logging.info("WriteToSQL started")
## Get information used to create connection string
username = '******'
password = os.getenv("sqlPassword")
driver = '{ODBC Driver 17 for SQL Server}'
server = os.getenv("sqlServer")
database = 'AzureCognitive'
table = 'AzureBlobVideoCompletes'
## Create connection string
connectionString = f'DRIVER={driver};SERVER={server};PORT=1433;DATABASE={database};UID={username};PWD={password}'
logging.info(f'Connection string created: {connectionString}')
## Create SQL query to use
cols0 = [
"StartUTC", "EndUTC", "VideoID", "VideoName", "Event",
"OutputContainer", "OutputBlobStorageAccount", "ImagesCreated"
]
cols = [MyFunctions.sqlColumn_ise(x) for x in cols0]
vals0 = [
MyFunctions.sqlDateTimeFormat(startUTC),
MyFunctions.sqlDateTimeFormat(endUTC), videoID, videoName, event,
outputContainer, outputBlobStorageAccount, imagesCreatedCount
]
vals = [MyFunctions.sqlValue_ise(x) for x in vals0]
sqlQuery = f"INSERT INTO {table} ({','.join(cols)}) VALUES ({','.join(vals)});"
with pyodbc.connect(connectionString) as conn:
with conn.cursor() as cursor:
logging.info("About to execute 'INSERT' query")
cursor.execute(sqlQuery)
logging.info("'INSERT' query executed")
return f"Row added to {table}"
def Ch5_Exa3(): N = 50 a = 0.0 b = 1.0 # Calculate the sample points and weights, then map them # to the required integration domain x,w = MyFunctions.gaussxwab(N,a,b) # Perform the integration s = 0.0 for k in range(N): s = s + w[k]*fun_to_int(x[k]) return (s)
def aggregate(folderpath):
'''Aggregate scripts here
Arguments:
folderpath {[String]} -- [the folder containing all raw files]
'''
fileNameList = MyFunctions.visitAllFile(folderpath) # fileNameList = [[]]
numOfFiles = len(fileNameList[0])
print('There are %d files in this folder' % numOfFiles)
count = 1
for filepath in fileNameList[0]:
print('This is %d file out of %d files' %
(count, len(fileNameList[0])))
print('Please wait for file %d\'s process...' % count)
filepath_new = folderpath + '/' + filepath # ‘../Download/BusLocation_09_05_2017/2017-09-04.csv’
handleEachFile(filepath_new)
count += 1
# Delete the folder generated in this process
# 'Fenced' & 'fenced_vehicle'
MyFunctions.deleteFolder('fenced')
MyFunctions.deleteFolder('fenced_vehicle')
print('Complete successfully')
def main(deck):
#Create the player
while True:
try:
#Ask for the Players name
pname = str(input("Please provide your name: "))
pname = pname.strip()
if len(pname) == 0:
continue
except TypeError:
print("A TypeError was found")
continue
except:
print("Generic Error")
continue
else:
# No error so do the else
player = Player.Player(name=pname, coins=50)
break
while True:
result = MyFunctions.newDeal(player)
if result == 2:
return False
else:
if len(deck) < 20:
#if the deck is running low on cards, lets reshuffle the deck
deck = MyFunctions.Deck()
MyFunctions.NewGame(dealer, player)
if player.coins <= 0:
#player is out of money :(
print(f"{player.name} has no more coins. GAME OVER")
return False
else:
Main_Game(dealer, player, deck)
def Prob_7():
sw = 1
x = 2
y = 3
while sw == 1:
isPrime = MyFunctions.Prime(y)
if isPrime == 1:
if x == 10001:
sw = 0
else:
y = y + 2
x = x + 1
else:
y = y + 2
return y
def Ch10_Exa2(): Z = 79 e = 1.602e-19 E = 7.7e6*e epsilon0 = 8.854e-12 a0 = 5.292e-11 sigma = a0/100 N = 1000000 count = 0 for i in range(N): x,y = MyFunctions.gaussian(sigma) b = sqrt(x*x+y*y) if b<Z*e*e/(2*pi*epsilon0*E): count += 1 print (count,"particles were reflected out of",N) return 'done'
def calculateDuration(filepath):
with open(filepath, 'r') as f:
r_f = csv.reader(f)
headline = next(r_f)
for row in r_f:
# start
if len(row) == 11:
if row[10] == 'service_start':
date = row[2].split('T')[0]
time = row[2].split('T')[1].split('.')[0]
if row[0] not in nodeid_times.keys():
nodeid_times[row[0]] = 0
nodeid_times[row[0]] += 1
nodeid_period[row[0]] = ['', '']
nodeid_period[row[0]][0] = time
nodeid_duration[row[0]] = []
else:
nodeid_times[row[0]] += 1
nodeid_period[row[0]][0] = time
# stop
if row[10] == 'service_stop':
date = row[2].split('T')[0]
time = row[2].split('T')[1].split('.')[0]
nodeid_period[row[0]][1] = time
if len(nodeid_period[row[0]]) == 2:
duration = MyFunctions.calculateDuration(
nodeid_period[row[0]][0], nodeid_period[row[0]][1])
# pprint.pprint(nodeid_period)
# if int(nodeid_period[row[0]][1].split(':')[2]) < int(nodeid_period[row[0]][0].split(':')[2]):
# period_sec = 60 - (int(nodeid_period[row[0]][0].split(':')[2]) - int(nodeid_period[row[0]][1].split(':')[2]))
# period_min = int(nodeid_period[row[0]][1].split(':')[1]) - int(nodeid_period[row[0]][0].split(':')[1]) - 1
# if period_min < 0:
# period_min = 60 + period_min
# else:
# period_sec = int(nodeid_period[row[0]][1].split(':')[2]) - int(nodeid_period[row[0]][0].split(':')[2])
# period_min = int(nodeid_period[row[0]][1].split(':')[1]) - int(nodeid_period[row[0]][0].split(':')[1])
# if period_min < 0:
# period_min = 60 + period_min
# # duration = ':'.join([str(period_min), str(period_sec)])
# duration = str(round(period_min + (period_sec / 60), 2))
nodeid_duration[row[0]].append(duration)
nodeid_period[row[0]] = ['', '']
else:
continue
print(nodeid_duration)
print('SUCCESS 1')
return (nodeid_duration)
def Ch5_Exa2(): N = 100 a = 0.0 b = 1.0 # Calculate the sample points and weights, then map them # to the required integration domain x,w = MyFunctions.gaussxw(N) xp = 0.5*(b-a)*x + 0.5*(b+a) wp = 0.5*(b-a)*w # Perform the integration s = 0.0 for k in range(N): s = s + wp[k]*fun_to_int(xp[k]) return (s)
def run(folderpath):
'''Run for Interval
Arguments:
folderpath {[String]} -- [The fianl_file's path]
'''
service = getService()
fileName = MyFunctions.visitAllFile(folderpath)
fileName[0].pop(fileName[0].index('.DS_Store'))
print('Modified file list is: ', fileName[0])
print('There are %d files' % len(fileName[0]))
count = 1
for item in fileName[0]:
read_filepath = folderpath + '/' + item # [eg: '../output/final_2017-09-04.csv']
print('This is %d file out of %d' % (count, len(fileName[0])))
handleOneFile(read_filepath, service)
count += 1
def codeToRun():
try:
MyFunctions.func1()
MyFunctions.func2()
except Exception, e:
print str(e)
# print " -- ", nameMatch['id'], nameMatch['last'], nameMatch['first'], nameMatch['lat'], nameMatch['long']
if row['lat'] == nameMatch['lat'] and row['long'] == nameMatch['long']:
matches += 1
print "MATCH FOUND! -- ", matches, nameMatch['first'], nameMatch['last']
insertmatchsql = "insert into libertarians_national (national_id, libertarian_id) values ('" + nameMatch['id'] + "', '" + row['SOS_VOTERID'] + "')"
# print insertmatchsql
cur.execute(insertmatchsql)
db.commit()
print "Number Processed: ", numberProcessed
print "Matches: ", matches
db.close()
if MyFunctions.query_yes_no("Copy libertarians_national table into default database for later use?"):
src_conn = MyFunctions.open_db(dbfile)
dest_conn = MyFunctions.open_db(defaultDBfile)
MyFunctions.copy_table('libertarians_national',src_conn,dest_conn)
end_time = time.time()
print("\n<--- Execution time: %s seconds --->" % (end_time - start_time))
print rowsProcessed, ": ", r.status_code, ": ", insertSQL
if rowsProcessed % 100 == 0:
db.commit() # Commit every 100 requests
except:
print "Uh oh! Something went wrong!"
print r.url
print r.json()
db.commit()
db.close()
raise
db.commit() # Do final commit
db.close()
if MyFunctions.query_yes_no("Copy geolocations into default database for later use?"):
src_conn = MyFunctions.open_db(dbfile)
dest_conn = MyFunctions.open_db(defaultDBfile)
MyFunctions.copy_table('locations',src_conn,dest_conn)
end_time = time.time()
print("\n<--- Execution time: %s seconds --->" % (end_time - start_time))
conn = sqlite3.connect(prelimDB)
cur = conn.cursor()
else:
print "Preliminary Database File " + prelimDB + " Does Not Exist"
sys.exit()
# get the list of fields from the database
cur.execute("PRAGMA table_info('libertarians')")
pragma = cur.fetchall()
# extract db field names from
dbfields = [str(x[1]) for x in pragma]
# Find some variables we'll need later
PARTY_AFFILIATION = MyFunctions.first_startswith('PARTY_AFFILIATION', dbfields)
FIRST_PRIMARY_FIELD = MyFunctions.first_startswith('PRIMARY', dbfields)
LAST_PRIMARY_FIELD = MyFunctions.last_startswith('PRIMARY', dbfields)
LAST_NAME = MyFunctions.first_startswith('LAST_NAME', dbfields)
FIRST_NAME = MyFunctions.first_startswith('FIRST_NAME', dbfields)
ZIP = MyFunctions.first_startswith('RESIDENTIAL_ZIP', dbfields)
fieldsInLine = len(dbfields)
# find the county data files, put them in an array & sort them.
csvfiles = glob.glob('../ZIP/*.zip')
csvfiles.sort()
# parse the datafiles
for onefile in csvfiles:
countyAffiliations = {}
databaseDirectory = "../Database/"
numCountiesExpected = 88
myZipFiles = glob.glob(zipDirectory + '*.zip')
myZipFiles.sort()
if len(myZipFiles) == numCountiesExpected:
print numCountiesExpected, "counties found - OK!"
else:
print numCountiesExpected, "counties expected"
print len(myZipFiles), "counties found"
numErrors = len(glob.glob(zipDirectory + '*.ERROR'))
if numErrors > 0:
print str(numErrors) + " error file(s) found"
if not MyFunctions.query_yes_no("Proceed?"):
print "Exiting due to user request: incorrect # of counties found!"
sys.exit(0)
fields = []
# Scan and see if files are consistent
errors = []
for onefile in myZipFiles:
# separate the filename
filenameparts = onefile.split('/')
# grab the actual filename
filename = filenameparts[-1]
def codeToRun():
MyFunctions.outerFunction()
cur = db.cursor()
cur.execute("PRAGMA table_info('national')")
pragma = cur.fetchall()
# extract db field names from national
dbfields = [str(x[1]) for x in pragma]
if dbfields == fields:
print "DB matches file format"
else:
print "DB does not match file format"
print len(dbfields), "fields in database"
print len(fields), "fields in files"
if MyFunctions.query_yes_no("Replace db schema with file schema?", "no"):
print "\nOk...."
print "Dropping table: national"
db.execute('''DROP TABLE IF EXISTS national''')
db.commit()
print "Recreating table: national with new file format"
sql = "CREATE TABLE national ("
fields_added = 0
for afield in fields:
if fields_added == 0:
fieldname = ""
else:
fieldname = ", "
print fieldname
if afield == "id":
optionText = " PRIMARY KEY"
prelimDB = filePath+filePrefix+fileSuffix
# Connect to the database
conn = sqlite3.connect(prelimDB)
cur = conn.cursor()
# get the list of fields from the database
cur.execute("PRAGMA table_info('libertarians')")
pragma = cur.fetchall()
# extract db field names from
dbfields = [str(x[1]) for x in pragma]
# Find some variables we'll need later
PARTY_AFFILIATION = MyFunctions.first_startswith('PARTY_AFFILIATION',dbfields)
LAST_PRIMARY_FIELD = MyFunctions.last_startswith('PRIMARY',dbfields)
CONGRESSIONAL_DISTRICT = MyFunctions.last_startswith('CONGRESSIONAL_DISTRICT',dbfields)
LAST_NAME = MyFunctions.first_startswith('LAST_NAME',dbfields)
FIRST_NAME = MyFunctions.first_startswith('FIRST_NAME',dbfields)
fieldsInLine = len(dbfields)
# find the county data files, put them in an array & sort them.
csvfiles = glob.glob('../ZIP/*.zip')
csvfiles.sort()
# parse the datafiles
for onefile in csvfiles:
countyCount = 1
# Set up variables needed later
def codeToRun():
MyFunctions.noParamsFunction()
def codeToRun():
MyFunctions.add(4, 5)
MyFunctions.subtract(4, 5)
def codeToRun():
MyFunctions.noParamsFunction()
z = MyFunctions.add(1, 2)
def codeToRun():
MyFunctions.add(4, 5)
sys.exit()
# Connect to the database
conn = sqlite3.connect(newDB)
cur = conn.cursor()
# get the list of fields from the database
cur.execute("PRAGMA table_info('libertarians')")
pragma = cur.fetchall()
# extract db field names from
dbfields = [str(x[1]) for x in pragma]
# Find some variables we'll need later
PARTY_AFFILIATION = MyFunctions.first_startswith('PARTY_AFFILIATION',dbfields)
LAST_PRIMARY_FIELD = MyFunctions.last_startswith('PRIMARY',dbfields)
LAST_NAME = MyFunctions.first_startswith('LAST_NAME',dbfields)
FIRST_NAME = MyFunctions.first_startswith('FIRST_NAME',dbfields)
fieldsInLine = len(dbfields)
# find the county data files, put them in an array & sort them.
csvfiles = glob.glob('../ZIP/*.zip')
csvfiles.sort()
# parse the datafiles
for onefile in csvfiles:
countyCount = 1
try:
zf = zipfile.ZipFile(onefile, 'r')
import shutil
import time
import subprocess
import zipfile
import MyFunctions
start_time = time.time()
ZIPFILEDIRECTORY = "../ZIP/"
errors = [] # list of lists, each sublist defined as [filename, error message to display]
countiesfile = 'counties.txt' # text file containing list of counties to download (one per line)
if os.listdir(ZIPFILEDIRECTORY):
if not MyFunctions.query_yes_no("Directory " + ZIPFILEDIRECTORY +
"is not empty. Proceed? \n(Will fill missing files, will not overwrite) -->",
"no"):
print "\nOk - exiting at user request"
sys.exit(0)
# open counties file
f = open(countiesfile, 'r')
# get the whole shebang
counties = f.readlines()
# standardize each line (remove trailing spaces, make all uppercase)
counties = [x.strip() for x in counties]
counties = [x.upper() for x in counties]
# cycle through counties in the list