def search_VSO(start_time, end_time): client = VSOClient() query_response = client.query_legacy(tstart=start_time, tend=end_time, instrument='HMI', physobs='intensity', sample=3600) results = client.fetch(query_response[:1], path='./tmp/{file}', site='rob') continuum_file = results.wait() query_response = client.query_legacy(tstart=start_time, tend=end_time, instrument='HMI', physobs='los_magnetic_field', sample=3600) results = client.fetch(query_response[:1], path='./tmp/{file}', site='rob') magnetic_file = results.wait() return continuum_file[0], magnetic_file[0]
def find_sdo_files(directory, wavelength='', time_limits=None, cadence=12, download=None, double_check='Yes'): """***This function no longer work. It will be updated or removed soon.*** Checks a directory for missing files from downloading AIA images and can check/download the files from the missing time. ***This function may need to run several times. It depends on how well the files are downloaded*** Parameters ---------- directory : Str A string of the path to the files to be checked. wavelength : Str The wavelength of the files to check. Only important if download = 'yes' or 'auto'. For HMI files then can have 'los_magnetic_field', 'intensity', etc. Default: '' time_limits : list A list of two entries for the start and end time of the observation. To check if any files were missed before the first and after the last file you have. Default: None cadence : Int An integer number of seconds that should be the temporal seperation of the files. Default: 12 download : Str Indicates whether missing files should be searched for/downloaded. If set to None then there will be a prompt to ask, enter 'Yes' or 'No'. Setting to 'auto' will search for the data automatically without user input. Default: None double_check : Str After checking for more files to download check again - without downloading - to see if there are any still missing, e.g. 'Yes' or 'No'. Can also have 'recursive' which keeps checking all files are found in case there is a large time gap in the middle *** BEWARE INFINITIES ***. Default: 'Yes' Returns ------- A list of the files with no friends cadence seconds after them. """ # this function no longer works at the moment print("This function no longer work. It will be updated or removed soon.") return files_list = list(os.listdir(directory)) files = [ f for f in files_list if f.endswith('.fits')] files.sort() if files == []: empty_but_download = input('The folder provided does not appear to have any \'.fits\' files within it. \nDo you want to download within the time range given in the form \"%Y-%m-%d %H:%M:%S\" (this will only work for SDO/AIA files at the moment)? ') if empty_but_download == 'Yes': client = VSOClient() query_response = client.query_legacy(tstart=time_limits[0], tend=time_limits[1], instrument='AIA', wave=wavelength) n = len(query_response) - 1 #will be used to index the list 'query_response' when downloading #Download the first two from ROB to /tmp folder and wait for download to complete results = client.get(query_response[0:n], path=directory, site='rob') fs = results.wait() still_no_friends = find_sdo_files(directory, wavelength, time_limits=time_limits, download='Yes', double_check='No') return else: assert files != [], f'No .fits files in {directory}.' #make sure there are files in the first place no_friends = [] #files that do not have a friend within the cadence time after it t_of_no_friends = [] #time of files that have no friends t_end_of_no_firends = [] for f in range(len(files)-2): #don't want to look at the last one as it will never have anything after it anyway time_0 = datetime.datetime.strptime(files[f][4:19], '%Y%m%d_%H%M%S') time_1 = datetime.datetime.strptime(files[f+1][4:19], '%Y%m%d_%H%M%S') if time_0 <= time_1 <= time_0 + timedelta(seconds=cadence): #if there is a file <=12s ahead move on continue else: #if there is not a file <=12s ahead add it to the no friends list no_friends.append(files[f]) t_of_no_friends.append(time_0) t_end_of_no_firends.append(time_1) if (download == None) and (len(t_of_no_friends) > 0 ): download = input('Would you like the times of the missing files checked (yea or nay)? ') if download in ['No', 'no', 'N', 'n', 'Nope', 'nope', 'Nay', 'nay']: download = 'No' elif download in ['Yes', 'yes', 'Y', 'y', 'Yip', 'yip', 'Yea', 'yea']: download = 'Yes' if len(t_of_no_friends) > 0: print('There are ', len(t_of_no_friends), ' time intervals of missing files.') if (download == 'Yes'): client = VSOClient() if len(t_of_no_friends) > 0: start_times = [t.strftime("%Y-%m-%d %H:%M:%S") for t in t_of_no_friends] #search a minute ahead end_times = [t.strftime("%Y-%m-%d %H:%M:%S") for t in t_end_of_no_firends] for ts, te in zip(start_times, end_times): if files[0][0:3] == 'aia': query_response = client.query_legacy(tstart=ts, tend=te, instrument='AIA', wave=wavelength) elif files[0][0:3] == 'hmi': query_response = client.query_legacy(tstart=st, tend=te, instrument='HMI', physobs=wavelength) n = len(query_response) - 1 #will be used to index the list 'query_response' when downloading #Download the first two from ROB to /tmp folder and wait for download to complete results = client.get(query_response[0:n], path=directory, site='rob') fs = results.wait() if time_limits != None: time_first = datetime.datetime.strptime(files[0][4:19], '%Y%m%d_%H%M%S') time_last = datetime.datetime.strptime(files[-1][4:19], '%Y%m%d_%H%M%S') time_limits_first = datetime.datetime.strptime(time_limits[0], "%Y-%m-%d %H:%M:%S") time_limits_last = datetime.datetime.strptime(time_limits[1], "%Y-%m-%d %H:%M:%S") if time_first - time_limits_first >= timedelta(seconds=cadence): #if the diff between the start time given and the first file's starting time print('Checking start time-gap.') if files[0][0:3] == 'aia': query_response = client.query_legacy(tstart=time_limits[0], tend=time_first.strftime("%Y-%m-%d %H:%M:%S"), instrument='AIA', wave=wavelength) elif files[0][0:3] == 'hmi': query_response = client.query_legacy(tstart=time_limits[0], tend=time_first.strftime("%Y-%m-%d %H:%M:%S"), instrument='HMI', physobs=wavelength) n = len(query_response) - 1 #will be used to index the list 'query_response' when downloading #Download the first two from ROB to /tmp folder and wait for download to complete results = client.get(query_response[0:n], path=directory, site='rob') fs = results.wait() if time_limits_last - time_last >= timedelta(seconds=cadence): #if the diff between the end time given and the end file's starting time print('Checking end time-gap.') if files[0][0:3] == 'aia': query_response = client.query_legacy(tstart=time_last.strftime("%Y-%m-%d %H:%M:%S"), tend=time_limits[1], instrument='AIA', wave=wavelength) elif files[0][0:3] == 'hmi': query_response = client.query_legacy(tstart=time_last.strftime("%Y-%m-%d %H:%M:%S"), tend=time_limits[1], instrument='HMI', physobs=wavelength) n = len(query_response) - 1 #will be used to index the list 'query_response' when downloading #Download the first two from ROB to /tmp folder and wait for download to complete results = client.get(query_response[0:n], path=directory, site='rob') fs = results.wait() duplicates = glob.glob(directory + '*.*.fits') #removes files that downloaded twice for each_file_path in duplicates: os.remove(each_file_path) if no_friends == []: #if there arent any files to check for then don't double check or anything, just stop print('All files here!') return if double_check == 'No': print(f'Here are files without friends {cadence} seconds ahead of them from directory \n{directory}:') print(no_friends) print('Please wait a few minutes and try this function again, it depends on the servers sometimes.') return no_friends elif double_check == 'Yes': #double check to see if we have all the files still_no_friends = find_sdo_files(directory, wavelength, time_limits=time_limits, download='Yes', double_check='No') elif double_check == 'recursive': still_no_friends = find_sdo_files(directory, wavelength, time_limits=time_limits, download='Yes', double_check='recursive') return still_no_friends