def StartBash(modelType, branche):
Bm.bashFile(getdate(), modelType, branche)
time.sleep(1)
os.system("chmod -R 777 " + filelocation)
if Bm.bothCheked is 1:
os.system("./" + modelType + "00.bash ")
print("Done, file located at -->" + filelocation + "/bash")
if Bm.bothCheked is 2:
os.system("./" + modelType + "12.bash ")
print("Done, file located at -->" + filelocation + "/bash")
if Bm.bothCheked is 3:
os.system("./" + modelType + "00.bash ")
os.system("./" + modelType + "12.bash ")
print("Done, file located at -->" + filelocation + "/bash")
def generateFromDB(stationID):
lstofSpeciesFST = formattedParticuleString.split(" ")
lstofSpecies.clear()
for et in lstofSpeciesFST:
lstofSpecies.append(fstdDict[et])
templst = []
for s,sp in zip(lstofSpecies,lstofSpeciesFST):
for d in lstdays[:-1]: # skips last date, but lstfile contains it so it reads it. we just need the last 3h of the eDate in the last file
for l in lstfile:
print("searching db: "+ l)
connection = sql.connect(path+l)
c = connection.cursor()
c.execute("SELECT COUNT(*) FROM (SELECT _rowid_,* FROM main.header);")
c.execute("SELECT _rowid_,* FROM main.header WHERE id_stn LIKE '%0"+stationID+"%'")
for i in c.fetchall():
if i[6] == int(d.strftime("%Y%m%d")): # range of dates
print("TIME: " + str(i[7])+" DATE: "+d.strftime("%Y%m%d"))
c.execute("SELECT COUNT(*) FROM (SELECT _rowid_,* FROM main.data)")
c.execute("SELECT _rowid_,* FROM main.data WHERE id_obs =" + str(i[1]) + " AND species =" + s)
for p in c.fetchall():
#str(int(i[7]/10000))
if int(i[7]/10000) <10:
pre10 = d.strftime("%Y%m%d") + ",0" + str(int(i[7]/10000))+","+str((p[8]))+"\n"
templst.append(pre10)
else:
post10 = d.strftime("%Y%m%d") + "," + str(int(i[7] / 10000)) + "," + str((p[8])) + "\n"
templst.append(post10)
bb = sorted(templst)
if len(bb) <1:
print(Bm.FAIL + sp + " NOT found at station " + stationID + Bm.ENDC)
if len(lstofSpeciesFST)>1:
print("Trying other selected pollutants...")
time.sleep(2)
continue
print("Writing to file")
fileName = sDate.strftime("%Y%m%d") + "_" + eDate.strftime("%Y%m%d") +"_OBS_" + sp +"_" + Bm.returnName(stationID) + ".csv"
# open("output/OBS__ID" + stationID + "__" + sp + "__START" + sDate.strftime("%Y%m%d") + "__END" + eDate.strftime("%Y%m%d") + ".csv", "w+")
file = open("output_csv/" +fileName, "w+")
file.write("Date,Time(Z),Value\n")
for t in bb:
file.write(t)
templst.clear()
file.close()
Bm.generateExcel(fileName)
#raise Exception(Gm.FAIL+sp+" NOT found at station " + stationID+ Gm.ENDC)
print("\nJob done, see folder for csv file-->" + filelocation + "/output_csv")
print("\nJob done, see folder for excel file-->" + filelocation + "/output_excel")
def storeDB():
try:
os.remove("configuration")
except:
pass
if enteredDate.get() != "" and enteredEndDate != "":
dbFile = open("configuration", "ab")
UpdateEverything()
bashModelParameters("", "")
pickle.dump([
enteredDate.get(),
enteredEndDate.get(),
sHourcombo.current(),
eHourCombo.current(),
Bm.returnDateList(),
selectDate.current()
], dbFile)
dbFile.close()
print("Configuration Saved!")
def getDataAtLocation(locationID):
try:
stationCode = referenceDict[locationID]
for sub in os.listdir("prevision.csv"):
for file in os.listdir("prevision.csv/" + sub):
os.system("cat " + filelocation + "/prevision.csv/" + sub +
"/" + file + "| grep " + stationCode + " > " +
filelocation + "/UMOSTreating/" + file + sub)
for untreated in os.listdir("UMOSTreating"):
date = untreated.split("_")[0]
filename = date + "_UMOS_" + subconverterdict[
sub[:2]] + "_" + Bm.returnName(locationID) + ".csv"
# prevents having empty files
if os.stat("UMOSTreating/" + untreated).st_size > 500:
with open("UMOSTreating/" + untreated,
"r") as infile, open("output_csv/" + filename,
"w") as outfile:
# UMOS__ID"+locationID +"__"+ untreated + ".csv",'w'
outfile.write(
"Date_Orig,Date_Valid,Code_Stn(ID),Lat,Lon,Vertical,Var,Value\n"
)
for line in infile:
withcomma = line.replace('|', ',')
withoutspace = withcomma.replace(" ", "")
changeName = withoutspace.replace(
stationCode, locationID)
outfile.write(changeName)
toExcel(filename)
print("\nJob done, see folder for csv file-->" + filelocation +
"/output_csv")
print("\nJob done, see folder for excel file-->" + filelocation +
"/output_excel")
# removeAllfile(r'' + filelocation + "/UMOSTreating")
# shutil.rmtree("prevision.csv")
except KeyError:
print("The station chosen does not have a UMOS code.")
def getQuickData(stationID):
stationName = Bm.returnName(stationID)
if os.path.exists(AQHI_path + stationID) is True:
for files in sorted(
os.listdir(AQHI_path + stationID)):
for days in lstdays[:-1]:
if files.endswith(stationID + "_" + days.strftime("%Y%m%d") + ".csv"):
f = open(AQHI_path + stationID + "/" + files, "r")
csvFile = list(csv.reader(f))
for z in range(len(csvFile)):
row = csvFile[z]
if row != ['station', 'Date', 'UTC', 'AQHI', 'O3', 'NO2', 'PM2.5', 'PM10', 'SO2', 'H2S',
'CO',
'NO', 'TRS']:
station = row[0]
stationlst.append(station)
date = row[1]
datelst.append(date)
hour = row[2]
hourlst.append(hour)
NO2 = row[5]
NO2lst.append(NO2)
O3 = row[4]
O3lst.append(O3)
PM25 = row[6]
PM25lst.append(PM25)
else:
print("This station does not exist in db-AQHI, try using the archives...")
# use this line when pandas will be updated on the server:
# dfNO['Date(DD/MM/YYYY)'] = pd.to_datetime(dfNO['Date(DD/MM/YYYY)']).dt.strftime("%Y%m%d")
formatteddatelst = []
formattedhourlst = []
for day in datelst:
d = datetime.strptime(day, "%Y-%m-%d").strftime("%Y%m%d")
formatteddatelst.append(d)
for hour in hourlst:
h = datetime.strptime(hour, "%H:%M:%S").strftime("%H")
formattedhourlst.append(h)
columnheader = ["Date", "Hour(Z)", "Value"]
if "AF" in formattedParticuleString:
dfPM = pd.DataFrame(list(zip(formatteddatelst, formattedhourlst, PM25lst)), columns=columnheader)
fileNameAF = sDate.strftime("%Y%m%d")+"_" + eDate.strftime("%Y%m%d") +"_OBS_AF_" + stationName
dfPM.to_csv("output_csv/" + fileNameAF +".csv", sep=",", index=False)
dfPM.to_excel("output_excel/" + fileNameAF + ".xlsx", engine="xlsxwriter")
checkifDataisValid("output_csv/" + fileNameAF + ".csv", "PM25", "output_excel/" + fileNameAF + ".xlsx")
if "N2" in formattedParticuleString:
dfNO = pd.DataFrame(list(zip(formatteddatelst, formattedhourlst, NO2lst)), columns=columnheader)
fileNameN2 = sDate.strftime("%Y%m%d")+"_" + eDate.strftime("%Y%m%d") +"_OBS_N2_" + stationName
dfNO.to_csv("output_csv/" + fileNameN2+".csv", sep=",",index=False)
dfNO.to_excel("output_excel/"+ fileNameN2+ ".xlsx", engine="xlsxwriter")
checkifDataisValid("output_csv/" + fileNameN2+".csv", "NO2","output_excel/"+ fileNameN2+ ".xlsx")
if "O3" in formattedParticuleString:
dfO3 = pd.DataFrame(list(zip(formatteddatelst, formattedhourlst, O3lst)), columns=columnheader)
fileNameO3 = sDate.strftime("%Y%m%d")+"_" + eDate.strftime("%Y%m%d") +"_OBS_O3_" + stationName
dfO3.to_csv("output_csv/" +fileNameO3 +".csv", sep=",", index=False)
dfO3.to_excel("output_excel/" +fileNameO3+ ".xlsx", engine="xlsxwriter")
checkifDataisValid("output_csv/" + fileNameO3 +".csv", "O3", "output_excel/" +fileNameO3+ ".xlsx")
stationlst.clear()
datelst.clear()
hourlst.clear()
NO2lst.clear()
O3lst.clear()
PM25lst.clear()
formatteddatelst.clear()
formattedhourlst.clear()
print("\nJob done, see folder for csv file-->" + filelocation + "/output_csv")
print("\nJob done, see folder for excel file-->" + filelocation + "/output_excel")
def FwGetLocation():
fwFolder = "extracted" + fWorkModelType
bashModelParameters(fWorkModelType, fWorkBranch)
Bm.bashFile(getdate(), fWorkModelType, fWorkBranch)
StartBash(fWorkModelType, fWorkBranch)
time.sleep(1)
Bm.removeAllfile(r'' + filelocation + "/config" + fWorkModelType)
Bm.getEticket(fWorkModelType)
Bm.locationExtraction(getComboboxLocation(), getdate(), fWorkModelType)
Bm.launchTCL(fWorkModelType)
Bm.removeEmptyFile(r'' + filelocation + "/" + fwFolder)
Bm.sortAndGenerate(filelocation + "/" + fwFolder + "/", getdate(),
fWorkModelType)
def getLocationGemmach():
gemFolder = "extracted" + gemmachModelType
bashModelParameters(gemmachModelType, gemmachBranch)
Bm.bashFile(getdate(), gemmachModelType, gemmachBranch)
StartBash(gemmachModelType, gemmachBranch)
time.sleep(1)
Bm.removeAllfile(r'' + filelocation + "/config" + gemmachModelType)
Bm.getEticket(gemmachModelType)
Bm.locationExtraction(getComboboxLocation(), getdate(), gemmachModelType)
Bm.launchTCL(gemmachModelType)
Bm.removeEmptyFile(r'' + filelocation + "/" + gemFolder)
Bm.sortAndGenerate(filelocation + "/" + gemFolder + "/", getdate(),
gemmachModelType)
def MistGetLocation():
umistFolder = "extracted" + uMistModelType
bashModelParameters(uMistModelType, uMistBranch)
Bm.umosBashFile(getdate(), uMistModelType, uMistBranch)
StartBash(uMistModelType, uMistBranch)
time.sleep(1)
Bm.removeAllfile(r'' + filelocation + "/config" + uMistModelType)
Bm.getEticket(uMistModelType)
Bm.locationExtraction(getComboboxLocation(), getdate(), uMistModelType)
Bm.launchTCL(uMistModelType)
Bm.removeEmptyFile(r'' + filelocation + "/" + umistFolder)
Bm.sortAndGenerate(filelocation + "/" + umistFolder + "/", getdate(),
uMistModelType)
def SearchNameID():
userInput = stationSearchField.get()
dString = Bm.SearchNameID(userInput)
stationSearchLabel.config(text=dString)
def combined(event):
Bm.provlist.clear()
name = comboprov.get()
provlist = Bm.gettingprovlist(name)
combostations.config(values=provlist)
def getdate():
return Bm.returnDateList()[int(selectDate.current())]
def bashModelParameters(modelType, branche):
global particules
Bm.inputStartDate(enteredDate.get())
Bm.inputEndDate(enteredEndDate.get())
Bm.usertime(sHourcombo.get(), eHourCombo.get())
Bm.modelCheckbox(var_00.get(), var_12.get())
O3 = var_O3.get()
NO2 = var_NO2.get()
PM25 = var_PM25.get()
particules = Bm.particuleCheckBox(O3, NO2, "", PM25)
if modelType != "" and branche != "":
Bm.rarcFile(modelType, branche)
selectDate.config(values=Bm.returnDateList())
# others = otherVariable.get()
Bm.level("")
# stations
def combined(event):
Bm.provlist.clear()
name = comboprov.get()
provlist = Bm.gettingprovlist(name)
combostations.config(values=provlist)
locationLabel = ttk.Label(machTab, text="Select Station:")
# province combobox
comboprov = ttk.Combobox(machTab, values=Bm.prov, width=10, state='readonly')
comboprov.bind('<<ComboboxSelected>>', combined)
comboprov.current(0)
# stations from the province combobox
combostations = ttk.Combobox(machTab,
values=Bm.gettingprovlist("Province"),
width=30,
state='readonly')
combostations.current(0)
locationLabel.place(x=400, y=245)
comboprov.place(x=510, y=245)
combostations.place(x=625, y=245)
# suff for the search
displayString = ""
stationSearchField = ttk.Entry(machTab, width=15)
def SearchNameID():
userInput = stationSearchField.get()
dString = Bm.SearchNameID(userInput)