def _download_and_collect_entries(self, query_result, client=None, path=None, progress=False, methods=None, overwrite=False, **kwargs): if kwargs: k, v = kwargs.popitem() raise TypeError(f'unexpected keyword argument {k!r}') if client is None: client = VSOClient() remove_list = [] delete_entries = [] for qr in query_result: temp = tables.DatabaseEntry._from_query_result_block(qr) for database_entry in self: if database_entry.path is not None and temp._compare_attributes( database_entry, ["source", "provider", "physobs", "fileid", "observation_time_start", "observation_time_end", "instrument", "size", "wavemin", "wavemax"]): if not overwrite: remove_list.append(qr) else: delete_entries.append(database_entry) for temp in remove_list: query_result = [x for x in query_result if x != temp] for temp in delete_entries: self.remove(temp) paths = client.fetch(query_result, path) for (path, block) in zip(paths, query_result): qr_entry = tables.DatabaseEntry._from_query_result_block(block) if os.path.isfile(path): entries = tables.entries_from_file(path, self.default_waveunit) elif os.path.isdir(path): entries = tables.entries_from_dir(path, self.default_waveunit) else: raise ValueError('The path is neither a file nor directory') for entry in entries: entry.source = qr_entry.source entry.provider = qr_entry.provider entry.physobs = qr_entry.physobs entry.fileid = qr_entry.fileid entry.observation_time_start = qr_entry.observation_time_start entry.observation_time_end = qr_entry.observation_time_end entry.instrument = qr_entry.instrument entry.size = qr_entry.size entry.wavemin = qr_entry.wavemin entry.wavemax = qr_entry.wavemax entry.path = path entry.download_time = datetime.utcnow() yield entry
def search_VSO(start_time, end_time): client = VSOClient() query_response = client.query_legacy(tstart=start_time, tend=end_time, instrument='HMI', physobs='intensity', sample=3600) results = client.fetch(query_response[:1], path='./tmp/{file}', site='rob') continuum_file = results.wait() query_response = client.query_legacy(tstart=start_time, tend=end_time, instrument='HMI', physobs='los_magnetic_field', sample=3600) results = client.fetch(query_response[:1], path='./tmp/{file}', site='rob') magnetic_file = results.wait() return continuum_file[0], magnetic_file[0]
def _download_and_collect_entries(self, query_result, **kwargs): client = kwargs.pop('client', None) path = kwargs.pop('path', None) progress = kwargs.pop('progress', False) methods = kwargs.pop('methods', ('URL-FILE_Rice', 'URL-FILE')) overwrite = kwargs.pop('overwrite', False) if kwargs: k, v = kwargs.popitem() raise TypeError('unexpected keyword argument {0!r}'.format(k)) if client is None: client = VSOClient() remove_list = [] delete_entries = [] for qr in query_result: temp = tables.DatabaseEntry._from_query_result_block(qr) for database_entry in self: if database_entry.path is not None and temp._compare_attributes( database_entry, ["source", "provider", "physobs", "fileid", "observation_time_start", "observation_time_end", "instrument", "size", "wavemin", "wavemax"]): if not overwrite: remove_list.append(qr) else: delete_entries.append(database_entry) for temp in remove_list: query_result = [x for x in query_result if x != temp] for temp in delete_entries: self.remove(temp) paths = client.fetch(query_result, path).wait(progress=progress) for (path, block) in zip(paths, query_result): qr_entry = tables.DatabaseEntry._from_query_result_block(block) if os.path.isfile(path): entries = tables.entries_from_file(path, self.default_waveunit) elif os.path.isdir(path): entries = tables.entries_from_dir(path, self.default_waveunit) else: raise ValueError('The path is neither a file nor directory') for entry in entries: entry.source = qr_entry.source entry.provider = qr_entry.provider entry.physobs = qr_entry.physobs entry.fileid = qr_entry.fileid entry.observation_time_start = qr_entry.observation_time_start entry.observation_time_end = qr_entry.observation_time_end entry.instrument = qr_entry.instrument entry.size = qr_entry.size entry.wavemin = qr_entry.wavemin entry.wavemax = qr_entry.wavemax entry.path = path entry.download_time = datetime.utcnow() yield entry