def main():
#####################
#Automatically set the file path according to the location of the installation package
#get the path of file
#set path
ddir=os.path.dirname(os.path.abspath(__file__))
dictionarypath=ddir[::-1].replace('py'[::-1],'dictionary'[::-1],1)[::-1]
parameterpath=ddir[::-1].replace('py'[::-1],'parameter'[::-1],1)[::-1]
Rawf_path=ddir[::-1].replace('py'[::-1],'aq/download'[::-1],1)[::-1]
pic_path=ddir[::-1].replace('py'[::-1],'aq/aqu_pic'[::-1],1)[::-1]
htmlpath=ddir[::-1].replace('py'[::-1],'html'[::-1],1)[::-1]
#HARDCODES
telemetrystatus_file=os.path.join(parameterpath,'telemetry_status.csv')
dictionaryfile=os.path.join(dictionarypath,'dictionary.json') # dictionary with endtime,doppio,gomofs,fvcom where each model has vesselname,lat,lon,time,temp
##############################
files=download_raw_file(ftppath='/Raw_Data/checked',localpath=Rawf_path)# UPDATE THE RAW csv FILE
starttime=datetime.datetime.now()-datetime.timedelta(days=30)
endtime=datetime.datetime.now()
telemetrystatus_df=rf.read_telemetrystatus(path_name=telemetrystatus_file)
emolt='http://www.nefsc.noaa.gov/drifter/emolt.dat' # this is the output of combining getap2s.py and getap3.py
print('get emolt df')
emolt_df=rf.screen_emolt(start_time=starttime,end_time=endtime,path=emolt)#get emolt data
# #run function
print('make png')
make_png(files,pic_path=pic_path,rootdir=Rawf_path,telemetry_status_df=telemetrystatus_df)# make time series image
print('upload png file')
up.sd2drf(local_dir=pic_path,remote_dir='/lei_aq_main/all_pic',filetype='png') #upload the picture to the studentdrifter
print('make html')
# #create the dictionary
cmd.update_dictionary(telemetrystatus_file,starttime,endtime,dictionaryfile) #that is a function use to update the dictionary
make_html(raw_path=Rawf_path,telemetrystatus_file=telemetrystatus_file,pic_path=pic_path,dictionary=dictionaryfile,df=emolt_df,htmlpath=htmlpath) #make the html
print('upload html')
up.sd2drf_update(local_dir=htmlpath,remote_dir='/lei_aq_main/html')
def main():
realpath=os.path.dirname(os.path.abspath(__file__))
parameterpath=realpath.replace('py','parameter')
#HARDCODING
raw_data_name_file=os.path.join(parameterpath,'raw_data_name.txt') #this data conclude the VP_NUM HULL_NUM VESSEL_NAME
output_path=realpath.replace('py','result') #use to save the data
telemetry_status=os.path.join(parameterpath,'telemetry_status.csv')
# below hardcodes is the informations to upload local data to student drifter.
subdir=['Matdata','checked']
mremote='/Raw_Data'
remote_subdir=['Matdata','checked']
###########################
end_time=datetime.now()
start_time,end_time=week_start_end(end_time,interval=1)
#download raw data from website
files=ftpdownload.download(os.path.join(output_path,'Matdata'),ftppath='/Matdata')
#classify the file by every boat
rdm.classify_by_boat(indir=os.path.join(output_path,'Matdata'),outdir=os.path.join(output_path,'classified'),pstatus=telemetry_status)
print('classfy finished!')
#check the reformat of every file:include header,heading,lat,lon,depth,temperature.
rdm.check_reformat_data(indir=os.path.join(output_path,'classified'),outdir=os.path.join(output_path,'checked'),startt=start_time,\
endt=end_time,pstatus=telemetry_status,rdnf=raw_data_name_file)
print('check format finished!')
for i in range(len(subdir)):
local_dir=os.path.join(output_path,subdir[i])
remote_dir=os.path.join(mremote,remote_subdir[i])
up.sd2drf(local_dir,remote_dir,filetype='csv',keepfolder=True)
#if __name__=='__main__':
# main()
def main():
realpath=os.path.dirname(os.path.abspath(__file__))
parameterpath=realpath.replace('py','parameter')
#HARDCODING
output_path=realpath.replace('py','result') #use to save the data
picture_save=output_path+'/stats/' #use to save the picture
emolt='https://www.nefsc.noaa.gov/drifter/emolt.dat' #this is download from https://www.nefsc.noaa.gov/drifter/emolt.dat,
telemetry_status=os.path.join(parameterpath,'telemetry_status.csv')
# below hardcodes is the informations to upload local data to student drifter.
subdir=['stats']
mremote='/Raw_Data'
remote_subdir=['stats']
###########################
end_time=datetime.now()
start_time,end_time=week_start_end(end_time,interval=1)
if not os.path.exists(picture_save):
os.makedirs(picture_save)
print('match telemetered and raw data!')
#match the telementry data with raw data, calculate the numbers of successful matched and the differnces of two data. finally , use the picture to show the result.
dict=rdm.match_tele_raw(os.path.join(output_path,'checked'),path_save=os.path.join(picture_save,'statistics'),telemetry_path=emolt,telemetry_status=telemetry_status,\
start_time=start_time,end_time=end_time,dpi=500)
tele_dict=dict['tele_dict']
raw_dict=dict['raw_dict']
record_file_df=dict['record_file_df']
index=tele_dict.keys()
print('match telemetered and raw data finished!')
print("start draw map")
raw_d=pd.DataFrame(data=None,columns=['time','filename','mean_temp','mean_depth','mean_lat','mean_lon'])
tele_d=pd.DataFrame(data=None,columns=['time','mean_temp','mean_depth','mean_lat','mean_lon'])
for i in index:
for j in range(len(record_file_df)): #find the location of data of this boat in record file
if i.lower()==record_file_df['Boat'][j].lower():
break
if len(raw_dict[i])==0 and len(tele_dict[i])==0:
continue
else:
raw_d=raw_d.append(raw_dict[i])
tele_d=tele_d.append(tele_dict[i])
rdm.draw_map(raw_dict[i],tele_dict[i],i,start_time,end_time,picture_save,dpi=300)
rdm.draw_time_series_plot(raw_dict[i],tele_dict[i],i,start_time,end_time,picture_save,record_file_df.iloc[j],dpi=300)
raw_d.index=range(len(raw_d))
tele_d.index=range(len(tele_d))
rdm.draw_map(raw_d,tele_d,'all_map',start_time,end_time,picture_save,dpi=300)
for i in range(len(subdir)):
local_dir=os.path.join(output_path,subdir[i])
remote_dir=os.path.join(mremote,remote_subdir[i])
up.sd2drf(local_dir,remote_dir,keepfolder=True) # need to keep subdirectry
def main():
realpath=os.path.dirname(os.path.abspath(__file__))
#realpath='E:/programe/raw_data_match/py'
parameterpath=realpath.replace('py','parameter')
#HARDCODING
raw_data_name_file=os.path.join(parameterpath,'raw_data_name.txt') #this data conclude the VP_NUM HULL_NUM VESSEL_NAME
#raw_data_name_file='E:/programe/raw_data_match/parameter/raw_data_name.txt'
output_path=realpath.replace('py','result') #use to save the data
telemetry_status=os.path.join(parameterpath,'telemetry_status.csv')
#telemetry_status='E:/programe/aqmain/parameter/telemetry_status.csv'
lack_data_path=os.path.join(output_path, 'lack_data.txt')
#lack_data_path='E:/programe/raw_data_match/result/lack_data.txt'#store the name of file that lacked data after 'classfy finished'
# below hardcodes is the informations to upload local data to student drifter.
subdir=['Matdata','checked']
mremote='/Raw_Data'
#mremote='\Raw_Data'
remote_subdir=['Matdata','checked']
###########################
end_time=datetime.utcnow()
#start_time,end_time=week_start_end(end_time,interval=1)
start_time=end_time-timedelta(weeks=1)
#download raw data from website
files=ftpdownload.download(localpath='E:\\programe\\raw_data_match\\result\\Matdata', ftppath='/Matdata')
#classify the file by every boats
rdm.classify_by_boat(indir='E:\\programe\\raw_data_match\\result\\Matdata',outdir='E:\\programe\\raw_data_match\\result\\classified',pstatus=telemetry_status)
print('classfy finished!')
#check the reformat of every file:include header,heading,lat,lon,depth,temperature.
rdm.check_reformat_data(indir='E:\\programe\\raw_data_match\\result\\classified',outdir='E:\\programe\\raw_data_match\\result\\checked',startt=start_time,\
endt=end_time,pstatus=telemetry_status,rdnf=raw_data_name_file,lack_data=lack_data_path)
print('check format finished!')
for i in range(len(subdir)):
local_dir=os.path.join(output_path,subdir[i])
#remote_dir=os.path.join(mremote,remote_subdir[i])
remote_dir=os.path.join(mremote,remote_subdir[i]).replace('\\', '/')
#up.sd2drf(local_dir,remote_dir,filetype='csv',keepfolder=True)
try:
up.sd2drf(local_dir, remote_dir, filetype='csv', keepfolder=True)
except TimeoutError:
print('Timeout Error')