def get_cdas_url(date, vars, dataset, timeout=10):
starttime = datetime.combine(date, time.min)
endtime = datetime.combine(date, time.max)
dataview = 'sp_phys'
if vars is None:
try:
var_info = get_variables(dataset, timeout=timeout)
except requests.exceptions.ReadTimeout:
raise util.NoDataError(
'Connection to CDAweb timed out when downloading '
f'{dataset} data for date {date}.')
if not len(var_info):
raise util.NoDataError(
f'No {dataset} data available for date {date}')
vars = [v['Name'] for v in var_info['VariableDescription']]
uri = '/'.join([
'dataviews', dataview, 'datasets', dataset, 'data', ','.join([
starttime.strftime('%Y%m%dT%H%M%SZ'),
endtime.strftime('%Y%m%dT%H%M%SZ')
]), ','.join(vars)
])
url = '/'.join([CDAS_BASEURL, uri])
return url
def processing_func(f):
if 'error_message' in f.readline():
f.close()
os.remove(f.name)
raise util.NoDataError()
df = pd.read_csv(f, names=['Time', 'Bx', 'By', 'Bz'],
delim_whitespace=True,
parse_dates=[0], index_col=0)
return df
def load_local_file(self, interval):
f = open(self.local_path(interval))
if 'error_message' in f.readline():
f.close()
os.remove(f.name)
raise util.NoDataError()
data = pd.read_csv(f,
names=['Time', 'Bx', 'By', 'Bz', '|B|',
'X', 'Y', 'Z', 'Local hour', 'n points'],
delim_whitespace=True,
parse_dates=[0], index_col=0)
f.close()
return data
def get_data(dataset, date, vars=None, verbose=True):
"""
Download CDAS data.
Parameters
----------
dataset : string
Dataset identifier.
date : datetime.date
Date to download data for.
vars : list of str, optional
Variables to download. If ``None``, all variables for the given
dataset will be downloaded.
verbose : bool, optional
If ``True``, show a progress bar whilst downloading.
Returns
-------
data_path : str
Path to downloaded data (stored in a temporary directroy)
"""
starttime = datetime.combine(date, time.min)
endtime = datetime.combine(date, time.max)
dataview = 'sp_phys'
if vars is None:
var_info = get_variables(dataset)
vars = [v['Name'] for v in var_info['VariableDescription']]
uri = '/'.join([
'dataviews', dataview, 'datasets', dataset, 'data', ','.join([
starttime.strftime('%Y%m%dT%H%M%SZ'),
endtime.strftime('%Y%m%dT%H%M%SZ')
]), ','.join(vars)
])
url = '/'.join([CDAS_BASEURL, uri])
params = {}
ext = ''
params = {'format': 'cdf', 'cdfVersion': 3}
ext = 'cdf'
response = requests.get(url, params=params, headers=CDAS_HEADERS)
if 'FileDescription' in response.json():
print('Downloading {} for date {}'.format(dataset, date))
data_path = wget.download(
response.json()['FileDescription'][0]['Name'],
tempfile.gettempdir(),
bar=wget.bar_adaptive if verbose else None)
if verbose:
print('')
else:
raise util.NoDataError('No {} data available for date {}'.format(
dataset, date))
return data_path
def processing_func(f):
if 'error_message' in f.readline():
f.close()
os.remove(f.name)
raise util.NoDataError()
data = pd.read_csv(f,
names=[
'Time', 'Bx', 'By', 'Bz', '|B|', 'X', 'Y', 'Z',
'Local hour', 'n points'
],
delim_whitespace=True,
parse_dates=[0],
index_col=0)
f.close()
return data
def get_data(dataset, starttime, endtime, vars=None, timeout=100):
"""
Download CDAS data.
Parameters
----------
dataset : string
Dataset identifier.
starttime : datetime.datetime
Beginning of interval.
endtime : datetime.datetime
End of interval.
vars : list of str, optional
Variables to download. If ``None``, all variables for the given
dataset will be downloaded.
timeout : float, optional
Timeout on the CDAweb remote requests, in seconds. Defaults to 10s.
Returns
-------
data_path : str
Path to downloaded data (stored in a temporary directroy)
"""
url = get_cdas_url(starttime, endtime, vars, dataset, timeout=timeout)
params = {'format': 'cdf', 'cdfVersion': 3}
response = requests.get(url,
params=params,
headers=CDAS_HEADERS,
timeout=timeout)
if 'FileDescription' in response.json():
print(f'Downloading {dataset} for interval {starttime} - {endtime}')
url = response.json()['FileDescription'][0]['Name']
with tempfile.NamedTemporaryFile(delete=False) as temp_file:
with requests.get(url, stream=True) as request:
for chunk in tqdm.tqdm(request.iter_content(chunk_size=128)):
temp_file.write(chunk)
return temp_file.name
else:
raise util.NoDataError(f'No {dataset} data available for interval '
f'{starttime} - {endtime}')
def get_data(dataset, date, vars=None, verbose=True, timeout=10):
"""
Download CDAS data.
Parameters
----------
dataset : string
Dataset identifier.
date : datetime.date
Date to download data for.
vars : list of str, optional
Variables to download. If ``None``, all variables for the given
dataset will be downloaded.
verbose : bool, optional
If ``True``, show a progress bar whilst downloading.
timeout : float, optional
Timeout on the CDAweb remote requests, in seconds. Defaults to 10s.
Returns
-------
data_path : str
Path to downloaded data (stored in a temporary directroy)
"""
url = get_cdas_url(date, vars, dataset, timeout=timeout)
params = {'format': 'cdf', 'cdfVersion': 3}
response = requests.get(url,
params=params,
headers=CDAS_HEADERS,
timeout=timeout)
if 'FileDescription' in response.json():
print('Downloading {} for date {}'.format(dataset, date))
data_path = wget.download(
response.json()['FileDescription'][0]['Name'],
tempfile.gettempdir(),
bar=wget.bar_adaptive if verbose else None)
if verbose:
print('')
else:
raise util.NoDataError('No {} data available for date {}'.format(
dataset, date))
return data_path
