def _download_and_collect_entries(self, query_result, client=None,
path=None, progress=False, methods=None,
overwrite=False, **kwargs):
if kwargs:
k, v = kwargs.popitem()
raise TypeError(f'unexpected keyword argument {k!r}')
if client is None:
client = VSOClient()
remove_list = []
delete_entries = []
for qr in query_result:
temp = tables.DatabaseEntry._from_query_result_block(qr)
for database_entry in self:
if database_entry.path is not None and temp._compare_attributes(
database_entry, ["source", "provider", "physobs", "fileid",
"observation_time_start", "observation_time_end",
"instrument", "size", "wavemin", "wavemax"]):
if not overwrite:
remove_list.append(qr)
else:
delete_entries.append(database_entry)
for temp in remove_list:
query_result = [x for x in query_result if x != temp]
for temp in delete_entries:
self.remove(temp)
paths = client.fetch(query_result, path)
for (path, block) in zip(paths, query_result):
qr_entry = tables.DatabaseEntry._from_query_result_block(block)
if os.path.isfile(path):
entries = tables.entries_from_file(path, self.default_waveunit)
elif os.path.isdir(path):
entries = tables.entries_from_dir(path, self.default_waveunit)
else:
raise ValueError('The path is neither a file nor directory')
for entry in entries:
entry.source = qr_entry.source
entry.provider = qr_entry.provider
entry.physobs = qr_entry.physobs
entry.fileid = qr_entry.fileid
entry.observation_time_start = qr_entry.observation_time_start
entry.observation_time_end = qr_entry.observation_time_end
entry.instrument = qr_entry.instrument
entry.size = qr_entry.size
entry.wavemin = qr_entry.wavemin
entry.wavemax = qr_entry.wavemax
entry.path = path
entry.download_time = datetime.utcnow()
yield entry
def search_VSO(start_time, end_time):
client = VSOClient()
query_response = client.query_legacy(tstart=start_time,
tend=end_time,
instrument='HMI',
physobs='intensity',
sample=3600)
results = client.fetch(query_response[:1], path='./tmp/{file}', site='rob')
continuum_file = results.wait()
query_response = client.query_legacy(tstart=start_time,
tend=end_time,
instrument='HMI',
physobs='los_magnetic_field',
sample=3600)
results = client.fetch(query_response[:1], path='./tmp/{file}', site='rob')
magnetic_file = results.wait()
return continuum_file[0], magnetic_file[0]
def _download_and_collect_entries(self, query_result, **kwargs):
client = kwargs.pop('client', None)
path = kwargs.pop('path', None)
progress = kwargs.pop('progress', False)
methods = kwargs.pop('methods', ('URL-FILE_Rice', 'URL-FILE'))
overwrite = kwargs.pop('overwrite', False)
if kwargs:
k, v = kwargs.popitem()
raise TypeError('unexpected keyword argument {0!r}'.format(k))
if client is None:
client = VSOClient()
remove_list = []
delete_entries = []
for qr in query_result:
temp = tables.DatabaseEntry._from_query_result_block(qr)
for database_entry in self:
if database_entry.path is not None and temp._compare_attributes(
database_entry, ["source", "provider", "physobs", "fileid",
"observation_time_start", "observation_time_end",
"instrument", "size", "wavemin", "wavemax"]):
if not overwrite:
remove_list.append(qr)
else:
delete_entries.append(database_entry)
for temp in remove_list:
query_result = [x for x in query_result if x != temp]
for temp in delete_entries:
self.remove(temp)
paths = client.fetch(query_result, path).wait(progress=progress)
for (path, block) in zip(paths, query_result):
qr_entry = tables.DatabaseEntry._from_query_result_block(block)
if os.path.isfile(path):
entries = tables.entries_from_file(path, self.default_waveunit)
elif os.path.isdir(path):
entries = tables.entries_from_dir(path, self.default_waveunit)
else:
raise ValueError('The path is neither a file nor directory')
for entry in entries:
entry.source = qr_entry.source
entry.provider = qr_entry.provider
entry.physobs = qr_entry.physobs
entry.fileid = qr_entry.fileid
entry.observation_time_start = qr_entry.observation_time_start
entry.observation_time_end = qr_entry.observation_time_end
entry.instrument = qr_entry.instrument
entry.size = qr_entry.size
entry.wavemin = qr_entry.wavemin
entry.wavemax = qr_entry.wavemax
entry.path = path
entry.download_time = datetime.utcnow()
yield entry