def download_and_load(remote_files,
local_dir,
download_only=False,
varformat=None,
get_support_data=False,
prefix='',
suffix=''):
""" Download cdf files.
Load cdf files into pytplot variables.
TODO: Loading files into pytplot has problems.
"""
result = []
""" # For debug only
msg = local_dir + str(download_only) + " ... " + '==='.join(remote_files)
title = 'Download Files'
show_my_message(title, msg)
print(msg)
"""
remotehttp = "https://cdaweb.sci.gsfc.nasa.gov/sp_phys/data"
count = 0
dcount = 0
for remotef in remote_files:
f = remotef.strip().replace(remotehttp, '', 1)
localf = local_dir + os.path.sep + f
resp, err, locafile = pyspedas.download_files(remotef, localf)
count += 1
if resp:
print(str(count) + '. File was downloaded. Location: ' + locafile)
dcount += 1
result.append(localf)
if not download_only:
try:
pytplot.cdf_to_tplot(locafile, varformat, get_support_data,
prefix, suffix, False, True)
except ValueError as err:
msg = "cdf_to_tplot could not load " + locafile
msg += "\n\n"
msg += "Error from pytplot: " + str(err)
print(msg)
show_my_message("Error!", msg)
else:
print(
str(count) + '. There was a problem. Could not download \
file: ' + remotef)
print(err)
print('Downloaded ' + str(dcount) + ' files.')
print('tplot variables:')
print(pyspedas.tplot_names())
return result
def test_cdf_swe_read():
pytplot.cdf_to_tplot(current_directory +
"/testfiles/mvn_swe_l2_svyspec_20170619_v04_r04.cdf")
pytplot.options('diff_en_fluxes', 'colormap', 'magma')
pytplot.options('diff_en_fluxes', 'ztitle', 'FLUX')
pytplot.options('diff_en_fluxes', 'ytitle', 'Energy')
pytplot.options("diff_en_fluxes", "spec", 1)
pytplot.options("diff_en_fluxes", "crosshair_y", "banana")
pytplot.options("diff_en_fluxes", "crosshair_z", "tomato")
pytplot.options("diff_en_fluxes", 'panel_size', 1)
pytplot.options('diff_en_fluxes', 'ylog', 1)
pytplot.options('diff_en_fluxes', 'zlog', 1)
pytplot.tplot(1, testing=True)
def load(trange=['2013-11-5', '2013-11-6'],
datatype='1min',
level='hro2',
suffix='',
get_support_data=False,
get_ignore_data=False,
varformat=None,
varnames=[],
downloadonly=False,
notplot=False,
no_update=False,
time_clip=True):
"""
This function loads OMNI (Combined 1AU IP Data; Magnetic and Solar Indices) data; this function is not meant
to be called directly; instead, see the wrapper:
pyspedas.omni.data
"""
if 'min' in datatype:
pathformat = level + '_' + datatype + '/%Y/omni_' + level + '_' + datatype + '_%Y%m01_v??.cdf'
elif 'hour' in datatype:
pathformat = 'hourly/%Y/omni2_h0_mrg1hr_%Y%m01_v??.cdf'
else:
raise TypeError("%r are invalid keyword arguments" % datatype)
# find the full remote path names using the trange
remote_names = dailynames(file_format=pathformat, trange=trange)
out_files = []
files = download(remote_file=remote_names,
remote_path=CONFIG['remote_data_dir'],
local_path=CONFIG['local_data_dir'],
no_download=no_update)
if files is not None:
for file in files:
out_files.append(file)
out_files = sorted(out_files)
if downloadonly:
return out_files
tvars = cdf_to_tplot(out_files,
suffix=suffix,
get_support_data=get_support_data,
get_ignore_data=get_ignore_data,
varformat=varformat,
varnames=varnames,
notplot=notplot)
if notplot:
return tvars
if time_clip:
for new_var in tvars:
tclip(new_var, trange[0], trange[1], suffix='')
return tvars
def omni_load(dates, level, downloadonly=False, varformat=None,
get_support_data=False, prefix='', suffix=''):
"""Loads OMNI data into pytplot variables"""
file_list = omni_filename(dates, level)
# 1. Download files
count = 0
downloaded_files = []
for remotef, localf in file_list:
count += 1
resp, err, localfile = pyspedas.download_files(remotef, localf)
if resp:
print(str(count) + '. File was downloaded. Location: ' + localfile)
downloaded_files.append(localfile)
else:
print(str(count) + '. Error: Could not download file: ' + remotef)
print(err)
print('Downloaded ' + str(len(downloaded_files)) + ' files.')
# 2. Load files into tplot
downloaded_vars = []
if not downloadonly:
try:
downloaded_vars = pytplot.cdf_to_tplot(downloaded_files, varformat,
get_support_data, prefix,
suffix, False, True)
except TypeError as e:
msg = "cdf_to_tplot could not load all data.\nError:\n" + str(e)
print(msg)
print('Loaded ' + str(len(downloaded_vars)) + ' variables.')
# 3. Time clip
if len(downloaded_vars) > 0:
pyspedas.time_clip(downloaded_vars, dates[0], dates[1], '')
def load(trange=['2019-02-01', '2019-02-02'],
site=None,
suffix='',
get_support_data=False,
varformat=None,
downloadonly=False,
notplot=False,
no_update=False,
time_clip=False):
"""
This function loads Magnetic Induction Coil Array (MICA) data; this function is not meant
to be called directly; instead, see the wrapper:
pyspedas.mica.induction
"""
if site == None:
print('A valid MICA site code name must be entered.')
print('Current site codes include: ')
print(
'NAL, LYR, LOR, ISR, SDY, IQA, SNK, MCM, SPA, JBS, NEV, HAL, PG2[3,4,5]'
)
return
pathformat = site.upper() + '/%Y/%m/mica_ulf_' + site.lower(
) + '_%Y%m%d_v??.cdf'
# find the full remote path names using the trange
remote_names = dailynames(file_format=pathformat, trange=trange)
out_files = []
files = download(remote_file=remote_names,
remote_path=CONFIG['remote_data_dir'],
local_path=CONFIG['local_data_dir'],
no_download=no_update)
if files is not None:
for file in files:
out_files.append(file)
out_files = sorted(out_files)
if downloadonly:
return out_files
tvars = cdf_to_tplot(out_files,
suffix='_' + site.upper() + suffix,
merge=True,
get_support_data=get_support_data,
varformat=varformat,
notplot=notplot)
if notplot:
return tvars
if time_clip:
for new_var in tvars:
tclip(new_var, trange[0], trange[1], suffix='')
return tvars
def load(trange=['2018-11-5', '2018-11-6'],
probe=['noaa19'],
instrument='sem',
datatype='*',
suffix='',
get_support_data=False,
varformat=None,
varnames=[],
downloadonly=False,
notplot=False,
no_update=False,
time_clip=False):
"""
This function loads POES Space Environment Monitor data. This function is
not meant to be called directly; instead, see the wrapper:
pyspedas.poes.sem
"""
if not isinstance(probe, list):
probe = [probe]
out_files = []
for prb in probe:
if instrument == 'sem':
pathformat = prb + '/sem2_fluxes-2sec/%Y/' + prb + '_poes-sem2_fluxes-2sec_%Y%m%d_v??.cdf'
# find the full remote path names using the trange
remote_names = dailynames(file_format=pathformat, trange=trange)
files = download(remote_file=remote_names,
remote_path=CONFIG['remote_data_dir'],
local_path=CONFIG['local_data_dir'],
no_download=no_update)
if files is not None:
for file in files:
out_files.append(file)
out_files = sorted(out_files)
if downloadonly:
return out_files
tvars = cdf_to_tplot(out_files,
suffix=suffix,
get_support_data=get_support_data,
varformat=varformat,
varnames=varnames,
notplot=notplot)
if notplot:
return tvars
if time_clip:
for new_var in tvars:
tclip(new_var, trange[0], trange[1], suffix='')
return tvars
def download(self,
remote_files,
local_dir,
download_only=False,
varformat=None,
get_support_data=False,
prefix='',
suffix=''):
""" Download cdf files.
Load cdf files into pytplot variables.
TODO: Loading files into pytplot sometimes does not work.
"""
result = []
loaded_vars = []
remotehttp = "https://cdaweb.gsfc.nasa.gov/sp_phys/data"
count = 0
dcount = 0
for remotef in remote_files:
tplot_loaded = 0
f = remotef.strip().replace(remotehttp, '', 1)
localf = local_dir + os.path.sep + f
localfile = download(remote_file=remotef, local_file=localf)
if localfile == None:
continue
localfile = localfile[0] # download returns an array
count += 1
if localfile != '':
dcount += 1
if not download_only:
try:
cdf_vars = pytplot.cdf_to_tplot(
localfile, varformat, get_support_data, prefix,
suffix, False, True)
if cdf_vars != [] and cdf_vars != None:
loaded_vars.extend(cdf_vars)
tplot_loaded = 1
except ValueError as err:
msg = "cdf_to_tplot could not load " + localfile
msg += "\n\n"
msg += "Error from pytplot: " + str(err)
print(msg)
tplot_loaded = 0
else:
print(
str(count) + '. There was a problem. Could not download \
file: ' + remotef)
tplot_loaded = -1
localfile = ''
result.append([remotef, localfile, tplot_loaded])
print('Downloaded ' + str(dcount) + ' files.')
if not download_only:
loaded_vars = list(set(loaded_vars))
print('tplot variables:')
for var in loaded_vars:
print(var)
return result
def load(trange=['2020-06-01', '2020-06-02'],
instrument='mag',
datatype='rtn-normal',
mode=None,
level='l2',
suffix='',
get_support_data=False,
varformat=None,
varnames=[],
downloadonly=False,
notplot=False,
no_update=False,
time_clip=False):
"""
This function loads data from the Solar Orbiter mission; this function is not meant
to be called directly; instead, see the wrappers:
pyspedas.solo.mag
pyspedas.solo.epd
pyspedas.solo.rpw
pyspedas.solo.swa
"""
if instrument == 'mag':
pathformat = instrument+'/science/'+level+'/'+datatype+'/%Y/solo_'+level+'_'+instrument+'-'+datatype+'_%Y%m%d_v??.cdf'
elif instrument == 'epd':
pathformat = instrument+'/science/'+level+'/'+datatype+'/'+mode+'/%Y/solo_'+level+'_'+instrument+'-'+datatype+'-'+mode+'_%Y%m%d_v??.cdf'
elif instrument == 'rpw':
pathformat = instrument+'/science/'+level+'/'+datatype+'/%Y/solo_'+level+'_'+instrument+'-'+datatype+'_%Y%m%d_v??.cdf'
elif instrument == 'swa':
if datatype == 'pas-eflux':
pathformat = instrument+'/science/'+level+'/'+datatype+'/%Y/solo_'+level+'_'+instrument+'-'+datatype+'_%Y%m%d_v??.cdf'
# find the full remote path names using the trange
remote_names = dailynames(file_format=pathformat, trange=trange)
out_files = []
files = download(remote_file=remote_names, remote_path=CONFIG['remote_data_dir'], local_path=CONFIG['local_data_dir'], no_download=no_update)
if files is not None:
for file in files:
out_files.append(file)
out_files = sorted(out_files)
if downloadonly:
return out_files
tvars = cdf_to_tplot(out_files, suffix=suffix, get_support_data=get_support_data, varformat=varformat, varnames=varnames, notplot=notplot)
if notplot:
return tvars
if time_clip:
for new_var in tvars:
tclip(new_var, trange[0], trange[1], suffix='')
return tvars
def load(trange=['2013-11-5', '2013-11-6'],
instrument='dcb',
datatype='',
level='l2',
suffix='',
get_support_data=False,
varformat=None,
downloadonly=False,
notplot=False,
no_update=False,
time_clip=False):
"""
This function loads data from the FAST mission; this function is not meant
to be called directly; instead, see the wrappers:
pyspedas.fast.dcb
pyspedas.fast.acb
pyspedas.fast.esa
pyspedas.fast.teams
"""
file_resolution = 24*3600.
if instrument == 'dcb':
pathformat = 'dcf/'+level+'/'+instrument+'/%Y/%m/fast_hr_'+instrument+'_%Y%m%d%H????_v??.cdf'
file_resolution = 3600.
if instrument == 'acb':
pathformat = 'acf/'+level+'/%Y/fa_'+level+'_acf_%Y%m%d_v??.cdf'
elif instrument == 'esa':
pathformat = instrument+'/'+level+'/'+datatype+'/%Y/%m/fa_'+instrument+'_'+level+'_'+datatype+'_%Y%m%d??????_*_v??.cdf'
if instrument == 'teams':
pathformat = 'teams/'+level+'/%Y/fa_'+level+'_tms_%Y%m%d_v??.cdf'
# find the full remote path names using the trange
remote_names = dailynames(file_format=pathformat, trange=trange, res=file_resolution)
out_files = []
for remote_file in remote_names:
files = download(remote_file=remote_file, remote_path=CONFIG['remote_data_dir'], local_path=CONFIG['local_data_dir'], no_download=no_update)
if files is not None:
for file in files:
out_files.append(file)
out_files = sorted(out_files)
if downloadonly:
return out_files
tvars = cdf_to_tplot(out_files, suffix=suffix, merge=True, get_support_data=get_support_data, varformat=varformat, notplot=notplot)
if notplot:
return tvars
if time_clip:
for new_var in tvars:
tclip(new_var, trange[0], trange[1], suffix='')
return tvars
def load(trange=['2013-11-5', '2013-11-6'],
instrument='reptile',
datatype='flux',
level='l2',
suffix='',
get_support_data=False,
varformat=None,
downloadonly=False,
notplot=False,
no_update=False,
time_clip=False):
"""
This function loads data from the CSSWE mission; this function is not meant
to be called directly; instead, see the wrapper:
pyspedas.csswe.reptile
"""
if instrument == 'reptile':
pathformat = level + '/' + instrument + '/' + datatype + '/%Y/csswe_' + instrument + '_6sec-' + datatype + '-' + level + '_%Y%m%d_v??.cdf'
# find the full remote path names using the trange
remote_names = dailynames(file_format=pathformat, trange=trange)
out_files = []
for remote_file in remote_names:
files = download(remote_file=remote_file,
remote_path=CONFIG['remote_data_dir'],
local_path=CONFIG['local_data_dir'],
no_download=no_update)
if files is not None:
for file in files:
out_files.append(file)
out_files = sorted(out_files)
if downloadonly:
return out_files
tvars = cdf_to_tplot(out_files,
suffix=suffix,
merge=True,
get_support_data=get_support_data,
varformat=varformat,
notplot=notplot)
if notplot:
return tvars
if time_clip:
for new_var in tvars:
tclip(new_var, trange[0], trange[1], suffix='')
return tvars
def load(trange=['2018-11-5', '2018-11-6'],
probe='1',
instrument='lad',
datatype='',
suffix='',
get_support_data=False,
varformat=None,
downloadonly=False,
notplot=False,
no_update=False,
time_clip=False):
"""
This function loads TWINS data; this function is not meant
to be called directly; instead, see the wrappers:
pyspedas.twins.lad
pyspedas.twins.ephemeris
pyspedas.twins.imager
"""
if not isinstance(probe, list):
probe = [probe]
probe = [str(prb) for prb in probe]
out_files = []
for prb in probe:
if instrument == 'lad':
pathformat = 'twins' + prb + '/' + instrument + '/%Y/twins' + prb + '_l1_lad_%Y%m%d_v??.cdf'
elif instrument == 'imager':
pathformat = 'twins' + prb + '/' + instrument + '/%Y/twins' + prb + '_l1_imager_%Y%m%d??_v??.cdf'
elif instrument == 'ephemeris':
pathformat = 'twins' + prb + '/' + instrument + '/' + datatype + '/%Y/twins' + prb + '_' + datatype + '_def_%Y%m%d_v??.cdf'
# find the full remote path names using the trange
remote_names = dailynames(file_format=pathformat, trange=trange)
for remote_file in remote_names:
files = download(remote_file=remote_file,
remote_path=CONFIG['remote_data_dir'],
local_path=CONFIG['local_data_dir'],
no_download=no_update)
if files is not None:
for file in files:
out_files.append(file)
out_files = sorted(out_files)
if downloadonly:
return out_files
tvars = cdf_to_tplot(out_files,
suffix=suffix,
merge=True,
get_support_data=get_support_data,
varformat=varformat,
notplot=notplot)
if notplot:
return tvars
if time_clip:
for new_var in tvars:
tclip(new_var, trange[0], trange[1], suffix='')
return tvars
def load(trange=['1997-01-03', '1997-01-04'],
instrument='mfe',
datatype='k0',
suffix='',
get_support_data=False,
varformat=None,
varnames=[],
downloadonly=False,
notplot=False,
no_update=False,
time_clip=False):
"""
This function loads data from the Polar mission; this function is not meant
to be called directly; instead, see the wrappera:
pyspedas.polar.mfe
pyspedas.polar.efi
pyspedas.polar.pwi
pyspedas.polar.hydra
pyspedas.polar.tide
pyspedas.polar.timas
pyspedas.polar.cammice
pyspedas.polar.ceppad
pyspedas.polar.uvi
pyspedas.polar.vis
pyspedas.polar.pixie
pyspedas.polar.orbit
"""
if instrument == 'mfe':
pathformat = instrument + '/' + instrument + '_' + datatype + '/%Y/po_' + datatype + '_' + instrument + '_%Y%m%d_v??.cdf'
elif instrument == 'efi':
pathformat = instrument + '/' + instrument + '_' + datatype + '/%Y/po_' + datatype + '_' + instrument + '_%Y%m%d_v??.cdf'
elif instrument == 'pwi':
pathformat = instrument + '/' + instrument + '_' + datatype + '/%Y/po_' + datatype + '_' + instrument + '_%Y%m%d_v??.cdf'
elif instrument == 'hydra':
pathformat = instrument + '/' + instrument + '_' + datatype + '/%Y/po_' + datatype + '_hyd_%Y%m%d_v??.cdf'
elif instrument == 'tide':
pathformat = instrument + '/' + instrument + '_' + datatype + '/%Y/po_' + datatype + '_tid_%Y%m%d_v??.cdf'
elif instrument == 'timas':
pathformat = instrument + '/' + instrument + '_' + datatype + '/%Y/po_' + datatype + '_tim_%Y%m%d_v??.cdf'
elif instrument == 'cammice':
pathformat = instrument + '/' + instrument + '_' + datatype + '/%Y/po_' + datatype + '_cam_%Y%m%d_v??.cdf'
elif instrument == 'ceppad':
pathformat = instrument + '/' + instrument + '_' + datatype + '/%Y/po_' + datatype + '_cep_%Y%m%d_v??.cdf'
elif instrument == 'uvi':
pathformat = instrument + '/' + instrument + '_' + datatype + '/%Y/po_' + datatype + '_' + instrument + '_%Y%m%d_v??.cdf'
elif instrument == 'vis':
pathformat = instrument + '/' + instrument + '_' + datatype + '/%Y/po_' + datatype + '_' + instrument + '_%Y%m%d_v??.cdf'
elif instrument == 'pixie':
pathformat = instrument + '/' + instrument + '_' + datatype + '/%Y/po_' + datatype + '_pix_%Y%m%d_v??.cdf'
elif instrument == 'spha':
pathformat = 'orbit/' + instrument + '_' + datatype + '/%Y/po_' + datatype + '_' + instrument + '_%Y%m%d_v??.cdf'
# find the full remote path names using the trange
remote_names = dailynames(file_format=pathformat, trange=trange)
out_files = []
files = download(remote_file=remote_names,
remote_path=CONFIG['remote_data_dir'],
local_path=CONFIG['local_data_dir'],
no_download=no_update)
if files is not None:
for file in files:
out_files.append(file)
out_files = sorted(out_files)
if downloadonly:
return out_files
tvars = cdf_to_tplot(out_files,
suffix=suffix,
get_support_data=get_support_data,
varformat=varformat,
varnames=varnames,
notplot=notplot)
if notplot:
return tvars
if time_clip:
for new_var in tvars:
tclip(new_var, trange[0], trange[1], suffix='')
return tvars
def load(trange=['2013-11-5', '2013-11-6'],
instrument='fgm',
datatype='k0',
suffix='',
get_support_data=False,
varformat=None,
downloadonly=False,
notplot=False,
no_update=False,
varnames=[],
time_clip=False):
"""
This function loads data from the ACE mission; this function is not meant
to be called directly; instead, see the wrappers:
pyspedas.ace.mfi
pyspedas.ace.swe
pyspedas.ace.epam
pyspedas.ace.cris
pyspedas.ace.sis
pyspedas.ace.uleis
pyspedas.ace.sepica
pyspedas.ace.swics
"""
if instrument == 'fgm':
pathformat = 'mag/level_2_cdaweb/mfi_' + datatype + '/%Y/ac_' + datatype + '_mfi_%Y%m%d_v??.cdf'
elif instrument == 'swe':
pathformat = 'swepam/level_2_cdaweb/swe_' + datatype + '/%Y/ac_' + datatype + '_swe_%Y%m%d_v??.cdf'
elif instrument == 'epm':
pathformat = 'epam/level_2_cdaweb/epm_' + datatype + '/%Y/ac_' + datatype + '_epm_%Y%m%d_v??.cdf'
elif instrument == 'cris':
pathformat = 'cris/level_2_cdaweb/cris_' + datatype + '/%Y/ac_' + datatype + '_cris_%Y%m%d_v??.cdf'
elif instrument == 'sis':
pathformat = 'sis/level_2_cdaweb/sis_' + datatype + '/%Y/ac_' + datatype + '_sis_%Y%m%d_v??.cdf'
elif instrument == 'ule':
pathformat = 'uleis/level_2_cdaweb/ule_' + datatype + '/%Y/ac_' + datatype + '_ule_%Y%m%d_v??.cdf'
elif instrument == 'sep':
pathformat = 'sepica/level_2_cdaweb/sep_' + datatype + '/%Y/ac_' + datatype + '_sep_%Y%m%d_v??.cdf'
elif instrument == 'swics':
filename_dtype = datatype.split('_')[1] + '_' + datatype.split('_')[0]
pathformat = 'swics/level_2_cdaweb/' + datatype + '/%Y/ac_' + filename_dtype + '_%Y%m%d_v??.cdf'
# find the full remote path names using the trange
remote_names = dailynames(file_format=pathformat, trange=trange)
out_files = []
files = download(remote_file=remote_names,
remote_path=CONFIG['remote_data_dir'],
local_path=CONFIG['local_data_dir'],
no_download=no_update)
if files is not None:
for file in files:
out_files.append(file)
out_files = sorted(out_files)
if downloadonly:
return out_files
tvars = cdf_to_tplot(out_files,
suffix=suffix,
get_support_data=get_support_data,
varformat=varformat,
varnames=varnames,
notplot=notplot)
if notplot:
return tvars
if time_clip:
for new_var in tvars:
tclip(new_var, trange[0], trange[1], suffix='')
return tvars
def load(trange=['2013-11-5', '2013-11-6'],
probe='a',
instrument='mag',
level='l2',
datatype='8hz',
coord='RTN',
suffix='',
get_support_data=False,
varformat=None,
downloadonly=False,
notplot=False,
no_update=False,
time_clip=False):
"""
This function loads data from the STEREO mission; this function is not meant
to be called directly; instead, see the wrappers:
pyspedas.stereo.mag
pyspedas.stereo.plastic
"""
out_files = []
if not isinstance(probe, list):
probe = [probe]
if datatype == '32hz':
burst = 'B'
else:
burst = ''
for prb in probe:
if prb == 'a':
direction = 'ahead'
elif prb == 'b':
direction = 'behind'
if instrument == 'mag':
pathformat = 'impact/level1/'+direction+'/mag/'+coord+'/%Y/%m/ST'+prb.upper()+'_L1_MAG'+burst+'_'+coord+'_%Y%m%d_V??.cdf'
elif instrument == 'plastic':
CONFIG['remote_data_dir'] = 'http://stereo-ssc.nascom.nasa.gov/data/ins_data/'
if level == 'l2':
pathformat = 'plastic/level2/Protons/Derived_from_1D_Maxwellian/'+direction+'/'+datatype+'/%Y/ST'+prb.upper()+'_L2_PLA_1DMax_'+datatype+'_%Y%m%d_V??.cdf'
# find the full remote path names using the trange
remote_names = dailynames(file_format=pathformat, trange=trange)
for remote_file in remote_names:
files = download(remote_file=remote_file, remote_path=CONFIG['remote_data_dir'], local_path=CONFIG['local_data_dir'], no_download=no_update)
if files is not None:
for file in files:
out_files.append(file)
out_files = sorted(out_files)
if downloadonly:
return out_files
tvars = cdf_to_tplot(out_files, suffix=suffix, merge=True, get_support_data=get_support_data, varformat=varformat, notplot=notplot)
if notplot:
return tvars
if time_clip:
for new_var in tvars:
tclip(new_var, trange[0], trange[1], suffix='')
return tvars
def mms_part_des_photoelectrons(dist_var):
"""
Loads and returns the FPI/DES photoelectron model based on stepper ID
Input
----------
dist_var: str
tplot variable containing DES distribution data
Notes
----------
For more information on the model, see:
https://agupubs.onlinelibrary.wiley.com/doi/full/10.1002/2017JA024518
Returns
----------
Dictionary containing the photoelectron model
"""
data = get_data(dist_var)
if data is None:
logging.error('Problem reading DES distribution variable')
return
# get the metadata for the 'energy_table_name' from the global attributes
metadata = get_data(dist_var, metadata=True)
try:
table_name = metadata['CDF']['GATT']['Energy_table_name']
except KeyError:
logging.error('Problem extracting the energy table name from the DF metadata')
return
stepper_id = table_name.replace('.txt', '').split('energies_des_')[-1]
# we'll need the data rate
data_rate = dist_var.split('_')[-1]
# download the model file from the SDC
pe_model = download(
last_version=True,
remote_path='https://lasp.colorado.edu/mms/sdc/public/data/models/fpi/',
remote_file='mms_fpi_'+data_rate+'_l2_des-bgdist_v?.?.?_p'+stepper_id+'.cdf',
local_path=os.path.join(CONFIG['local_data_dir'], 'mms'+os.path.sep+'sdc'+os.path.sep+'public'+os.path.sep+'data'+os.path.sep+'models'+os.path.sep+'fpi'+os.path.sep+''))
if len(pe_model) != 1:
logging.error('Problem downloading DES model from the SDC')
return
model_vars = cdf_to_tplot(pe_model[0], get_support_data=True)
if data_rate == 'fast':
bg_dist = get_data('mms_des_bgdist_fast')
nphoto = get_data('mms_des_numberdensity_fast')
return {'bg_dist': bg_dist, 'n': nphoto}
elif data_rate == 'brst':
bg_dist_0 = get_data('mms_des_bgdist_p0_brst')
bg_dist_1 = get_data('mms_des_bgdist_p1_brst')
nphoto_0 = get_data('mms_des_numberdensity_p0_brst')
nphoto_1 = get_data('mms_des_numberdensity_p1_brst')
return {'bgdist_p0': bg_dist_0,
'bgdist_p1': bg_dist_1,
'n_0': nphoto_0,
'n_1': nphoto_1}
# shouldn't get here
logging.error('Error: something went wrong with the photoelectron model')
return
def load(trange=['2018-11-5', '2018-11-6'],
instrument='fields',
datatype='mag_rtn',
level='l2',
suffix='',
get_support_data=False,
varformat=None,
varnames=[],
downloadonly=False,
notplot=False,
no_update=False,
time_clip=False):
"""
This function loads Parker Solar Probe data into tplot variables; this function is not
meant to be called directly; instead, see the wrappers:
psp.fields: FIELDS data
psp.spc: Solar Probe Cup data
psp.spe: SWEAP/SPAN-e data
psp.spi: SWEAP/SPAN-i data
psp.epihi: ISoIS/EPI-Hi data
psp.epilo: ISoIS/EPI-Lo data
psp.epi ISoIS/EPI (merged Hi-Lo) data
"""
file_resolution = 24*3600.
if instrument == 'fields':
pathformat = instrument + '/' + level + '/' + datatype + '/%Y/psp_fld_' + level + '_' + datatype + '_%Y%m%d%H_v??.cdf'
file_resolution = 6*3600.
elif instrument == 'spc':
pathformat = 'sweap/spc/' + level + '/' + datatype + '/%Y/psp_swp_spc_' + datatype + '_%Y%m%d_v??.cdf'
elif instrument == 'spe':
pathformat = 'sweap/spe/' + level + '/' + datatype + '/%Y/psp_swp_sp?_*_%Y%m%d_v??.cdf'
elif instrument == 'spi':
pathformat = 'sweap/spi/' + level + '/' + datatype + '/%Y/psp_swp_spi_*_%Y%m%d_v??.cdf'
elif instrument == 'epihi':
pathformat = 'isois/epihi/' + level + '/' + datatype + '/%Y/psp_isois-epihi_' + level + '*_%Y%m%d_v??.cdf'
elif instrument == 'epilo':
pathformat = 'isois/epilo/' + level + '/' + datatype + '/%Y/psp_isois-epilo_' + level + '*_%Y%m%d_v??.cdf'
elif instrument == 'epi':
pathformat = 'isois/merged/' + level + '/' + datatype + '/%Y/psp_isois_' + level + '-' + datatype + '_%Y%m%d_v??.cdf'
# find the full remote path names using the trange
remote_names = dailynames(file_format=pathformat, trange=trange, res=file_resolution)
out_files = []
files = download(remote_file=remote_names, remote_path=CONFIG['remote_data_dir'], local_path=CONFIG['local_data_dir'], no_download=no_update)
if files is not None:
for file in files:
out_files.append(file)
out_files = sorted(out_files)
if downloadonly:
return out_files
tvars = cdf_to_tplot(out_files, suffix=suffix, get_support_data=get_support_data, varformat=varformat, varnames=varnames, notplot=notplot)
if notplot:
return tvars
if time_clip:
for new_var in tvars:
tclip(new_var, trange[0], trange[1], suffix='')
return tvars
def load(trange=['1998-04-06', '1998-04-07'],
instrument='mam',
datatype='pp',
suffix='',
get_support_data=False,
varformat=None,
varnames=[],
downloadonly=False,
notplot=False,
no_update=False,
time_clip=False):
"""
This function loads data from the Equator-S mission; this function is not meant
to be called directly; instead, see the wrappers:
pyspedas.equator_s.mam
pyspedas.equator_s.edi
pyspedas.equator_s.esa (3DA)
pyspedas.equator_s.epi
pyspedas.equator_s.ici
pyspedas.equator_s.pcd
pyspedas.equator_s.sfd
"""
if instrument == 'mam':
pathformat = datatype + '/' + instrument + '/%Y/eq_' + datatype + '_' + instrument + '_%Y%m%d_v??.cdf'
elif instrument == 'edi':
pathformat = datatype + '/' + instrument + '/%Y/eq_' + datatype + '_' + instrument + '_%Y%m%d_v??.cdf'
elif instrument == '3da':
pathformat = datatype + '/' + instrument + '/%Y/eq_' + datatype + '_' + instrument + '_%Y%m%d_v??.cdf'
elif instrument == 'epi':
pathformat = datatype + '/' + instrument + '/%Y/eq_' + datatype + '_' + instrument + '_%Y%m%d_v??.cdf'
elif instrument == 'ici':
pathformat = datatype + '/' + instrument + '/%Y/eq_' + datatype + '_' + instrument + '_%Y%m%d_v??.cdf'
elif instrument == 'pcd':
pathformat = datatype + '/' + instrument + '/%Y/eq_' + datatype + '_' + instrument + '_%Y%m%d_v??.cdf'
elif instrument == 'sfd':
pathformat = datatype + '/' + instrument + '/%Y/eq_' + datatype + '_' + instrument + '_%Y%m%d_v??.cdf'
# find the full remote path names using the trange
remote_names = dailynames(file_format=pathformat, trange=trange)
out_files = []
files = download(remote_file=remote_names,
remote_path=CONFIG['remote_data_dir'],
local_path=CONFIG['local_data_dir'],
no_download=no_update)
if files is not None:
for file in files:
out_files.append(file)
out_files = sorted(out_files)
if downloadonly:
return out_files
tvars = cdf_to_tplot(out_files,
suffix=suffix,
get_support_data=get_support_data,
varformat=varformat,
varnames=varnames,
notplot=notplot)
if notplot:
return tvars
if time_clip:
for new_var in tvars:
tclip(new_var, trange[0], trange[1], suffix='')
return tvars
def load(trange=['2013-11-5', '2013-11-6'],
instrument='fgm',
datatype='h0',
suffix='',
get_support_data=False,
varformat=None,
varnames=[],
downloadonly=False,
notplot=False,
no_update=False,
time_clip=False):
"""
This function loads data from the WIND mission; this function is not meant
to be called directly; instead, see the wrappers:
pyspedas.wind.mfi
pyspedas.wind.swe
pyspedas.wind.sms
pyspedas.wind.threedp
pyspedas.wind.waves
pyspedas.wind.orbit
"""
if instrument == 'fgm':
pathformat = 'mfi/mfi_' + datatype + '/%Y/wi_' + datatype + '_mfi_%Y%m%d_v??.cdf'
elif instrument == 'swe':
pathformat = 'swe/swe_' + datatype + '/%Y/wi_' + datatype + '_swe_%Y%m%d_v??.cdf'
elif instrument == 'sms':
pathformat = 'sms/' + datatype + '/sms_' + datatype + '/%Y/wi_' + datatype + '_sms_%Y%m%d_v??.cdf'
elif instrument == 'waves':
pathformat = 'waves/wav_' + datatype + '/%Y/wi_' + datatype + '_wav_%Y%m%d_v??.cdf'
elif instrument == 'orbit':
pathformat = 'orbit/' + datatype + '/%Y/wi_' + datatype.split(
'_')[1] + '_' + datatype.split('_')[0] + '_%Y%m%d_v??.cdf'
elif instrument == '3dp':
if datatype == '3dp_emfits_e0':
pathformat = '3dp/' + datatype + '/%Y/wi_' + datatype.split(
'_')[1] + '_' + datatype.split('_')[2] + '_' + datatype.split(
'_')[0] + '_%Y%m%d_v??.cdf'
else:
pathformat = '3dp/' + datatype + '/%Y/wi_' + datatype.split(
'_')[1] + '_' + datatype.split('_')[0] + '_%Y%m%d_v??.cdf'
# find the full remote path names using the trange
remote_names = dailynames(file_format=pathformat, trange=trange)
out_files = []
files = download(remote_file=remote_names,
remote_path=CONFIG['remote_data_dir'],
local_path=CONFIG['local_data_dir'],
no_download=no_update,
last_version=True)
if files is not None:
for file in files:
out_files.append(file)
out_files = sorted(out_files)
if downloadonly:
return out_files
tvars = cdf_to_tplot(out_files,
suffix=suffix,
get_support_data=get_support_data,
varformat=varformat,
varnames=varnames,
notplot=notplot)
if notplot:
return tvars
if time_clip:
for new_var in tvars:
tclip(new_var, trange[0], trange[1], suffix='')
return tvars
def load(trange=['2013-11-5', '2013-11-6'],
instrument='fgm',
probe='c',
level='l2',
stations=None, # ground mag data
greenland=None, # also for ground mag data
suffix='',
get_support_data=False,
varformat=None,
varnames=[],
downloadonly=False,
notplot=False,
no_update=False,
time_clip=False):
"""
This function loads data from the THEMIS mission;
this function is not meant to be called directly;
instead, see the wrappers:
pyspedas.themis.fgm
pyspedas.themis.fit
pyspedas.themis.efi
pyspedas.themis.scm
pyspedas.themis.fft
pyspedas.themis.fbk
pyspedas.themis.esa
pyspedas.themis.sst
pyspedas.themis.mom
pyspedas.themis.gmom
pyspedas.themis.gmag
pyspedas.themis.state
"""
if not isinstance(probe, list):
probe = [probe]
out_files = []
for prb in probe:
if instrument == 'fgm':
pathformat = ('th' + prb + '/' + level + '/' + instrument
+ '/%Y/th' + prb + '_' + level + '_' + instrument
+ '_%Y%m%d_v??.cdf')
elif instrument == 'fit':
pathformat = ('th' + prb + '/' + level + '/' + instrument
+ '/%Y/th' + prb + '_' + level + '_' + instrument
+ '_%Y%m%d_v??.cdf')
elif instrument == 'efi':
if level == 'l2':
pathformat = ('th' + prb + '/' + level + '/' + instrument
+ '/%Y/th' + prb + '_' + level + '_' + instrument
+ '_%Y%m%d_v??.cdf')
elif level == 'l1':
pathformat = [('th' + prb + '/' + level + '/vaf/%Y/th' + prb
+ '_' + level + '_vaf_%Y%m%d_v??.cdf'),
('th' + prb + '/' + level + '/vap/%Y/th' + prb
+ '_' + level + '_vap_%Y%m%d_v??.cdf'),
('th' + prb + '/' + level + '/vaw/%Y/th' + prb
+ '_' + level + '_vaw_%Y%m%d_v??.cdf'),
('th' + prb + '/' + level + '/vbf/%Y/th' + prb
+ '_' + level + '_vbf_%Y%m%d_v??.cdf'),
('th' + prb + '/' + level + '/vbp/%Y/th' + prb
+ '_' + level + '_vbp_%Y%m%d_v??.cdf'),
('th' + prb + '/' + level + '/vbw/%Y/th' + prb
+ '_' + level + '_vbw_%Y%m%d_v??.cdf'),
('th' + prb + '/' + level + '/eff/%Y/th' + prb
+ '_' + level + '_eff_%Y%m%d_v??.cdf'),
('th' + prb + '/' + level + '/efw/%Y/th' + prb
+ '_' + level + '_efw_%Y%m%d_v??.cdf'),
('th' + prb + '/' + level + '/efp/%Y/th' + prb
+ '_' + level + '_efp_%Y%m%d_v??.cdf')]
elif instrument == 'scm':
if level == 'l2':
pathformat = ('th' + prb + '/' + level + '/' + instrument
+ '/%Y/th' + prb + '_' + level + '_' + instrument
+ '_%Y%m%d_v??.cdf')
elif level == 'l1':
pathformat = [('th' + prb + '/' + level + '/scp/%Y/th' + prb
+ '_' + level + '_scp_%Y%m%d_v??.cdf'),
('th' + prb + '/' + level + '/scf/%Y/th' + prb
+ '_' + level + '_scf_%Y%m%d_v??.cdf'),
('th' + prb + '/' + level + '/scw/%Y/th' + prb
+ '_' + level + '_scw_%Y%m%d_v??.cdf')]
elif instrument == 'fft':
if level == 'l2':
pathformat = ('th' + prb + '/' + level + '/' + instrument
+ '/%Y/th' + prb + '_' + level + '_' + instrument
+ '_%Y%m%d_v??.cdf')
elif level == 'l1':
pathformat = [('th' + prb + '/' + level + '/fff_16/%Y/th' + prb
+ '_' + level + '_fff_16_%Y%m%d_v??.cdf'),
('th' + prb + '/' + level + '/fff_32/%Y/th' + prb
+ '_' + level + '_fff_32_%Y%m%d_v??.cdf'),
('th' + prb + '/' + level + '/fff_64/%Y/th' + prb
+ '_' + level + '_fff_64_%Y%m%d_v??.cdf'),
('th' + prb + '/' + level + '/ffp_16/%Y/th' + prb
+ '_' + level + '_ffp_16_%Y%m%d_v??.cdf'),
('th' + prb + '/' + level + '/ffp_32/%Y/th' + prb
+ '_' + level + '_ffp_32_%Y%m%d_v??.cdf'),
('th' + prb + '/' + level + '/ffp_64/%Y/th' + prb
+ '_' + level + '_ffp_64_%Y%m%d_v??.cdf'),
('th' + prb + '/' + level + '/ffw_16/%Y/th' + prb
+ '_' + level + '_ffw_16_%Y%m%d_v??.cdf'),
('th' + prb + '/' + level + '/ffw_32/%Y/th' + prb
+ '_' + level + '_ffw_32_%Y%m%d_v??.cdf'),
('th' + prb + '/' + level + '/ffw_64/%Y/th' + prb
+ '_' + level + '_ffw_64_%Y%m%d_v??.cdf')]
elif instrument == 'fbk':
pathformat = ('th' + prb + '/' + level + '/' + instrument
+ '/%Y/th' + prb + '_' + level + '_' + instrument
+ '_%Y%m%d_v??.cdf')
elif instrument == 'esa':
pathformat = ('th' + prb + '/' + level + '/' + instrument
+ '/%Y/th' + prb + '_' + level + '_' + instrument
+ '_%Y%m%d_v??.cdf')
elif instrument == 'sst':
pathformat = ('th' + prb + '/' + level + '/' + instrument
+ '/%Y/th' + prb + '_' + level + '_' + instrument
+ '_%Y%m%d_v??.cdf')
elif instrument == 'mom':
pathformat = ('th' + prb + '/' + level + '/' + instrument
+ '/%Y/th' + prb + '_' + level + '_' + instrument
+ '_%Y%m%d_v??.cdf')
elif instrument == 'gmom':
pathformat = ('th' + prb + '/' + level + '/' + instrument
+ '/%Y/th' + prb + '_' + level + '_' + instrument
+ '_%Y%m%d_v??.cdf')
elif instrument == 'state':
pathformat = ('th' + prb + '/' + level + '/' + instrument
+ '/%Y/th' + prb + '_' + level + '_' + instrument
+ '_%Y%m%d.cdf')
elif instrument == 'gmag':
if stations is None:
logging.error('No stations specified')
return
else:
pathformat = []
for site, in_greenland in zip(stations, greenland):
if in_greenland:
pathformat.append('thg/greenland_gmag/' + level
+ '/mag/' + site + '/%Y/thg_' + level
+ '_mag_' + site + '_%Y%m%d_v??.cdf')
else:
pathformat.append('thg/' + level + '/mag/' + site
+ '/%Y/thg_' + level + '_mag_' + site
+ '_%Y%m%d_v??.cdf')
if not isinstance(pathformat, list):
pathformat = [pathformat]
for file_format in pathformat:
# find the full remote path names using the trange
remote_names = dailynames(file_format=file_format, trange=trange)
files = download(remote_file=remote_names,
remote_path=CONFIG['remote_data_dir'],
local_path=CONFIG['local_data_dir'],
no_download=no_update,
last_version=True)
if files is not None:
for file in files:
out_files.append(file)
out_files = sorted(out_files)
if downloadonly:
return out_files
tvars = cdf_to_tplot(out_files,
suffix=suffix,
merge=True,
get_support_data=get_support_data,
varformat=varformat,
varnames=varnames,
notplot=notplot)
if notplot:
return tvars
if time_clip:
for new_var in tvars:
tclip(new_var, trange[0], trange[1], suffix='')
return tvars
def load(trange=['2017-03-27', '2017-03-28'],
instrument='mgf',
datatype='8sec',
mode=None,
site=None,
level='l2',
suffix='',
get_support_data=False,
varformat=None,
varnames=[],
downloadonly=False,
notplot=False,
no_update=False,
uname=None,
passwd=None,
time_clip=False):
"""
This function is not meant to be called directly; please see the instrument specific wrappers:
pyspedas.erg.mgf()
pyspedas.erg.hep()
pyspedas.erg.orb()
pyspedas.erg.lepe()
pyspedas.erg.lepi()
pyspedas.erg.mepe()
pyspedas.erg.mepi()
pyspedas.erg.pwe_ofa()
pyspedas.erg.pwe_efd()
pyspedas.erg.pwe_hfa()
pyspedas.erg.xep()
"""
prefix = 'erg_' + instrument + '_' + level + '_'
file_res = 24 * 3600.
if instrument == 'mgf':
pathformat = 'satellite/erg/' + instrument + '/' + level + '/' + datatype + '/%Y/%m/erg_' + instrument + '_' + level + '_' + datatype + '_%Y%m%d_v??.??.cdf'
elif instrument == 'hep':
pathformat = 'satellite/erg/' + instrument + '/' + level + '/' + datatype + '/%Y/%m/erg_' + instrument + '_' + level + '_' + datatype + '_%Y%m%d_v??_??.cdf'
elif instrument == 'orb':
pathformat = 'satellite/erg/' + instrument + '/' + level + '/opq/%Y/%m/erg_' + instrument + '_' + level + '_op_%Y%m%d_v??.cdf'
elif instrument == 'lepe':
pathformat = 'satellite/erg/' + instrument + '/' + level + '/' + datatype + '/%Y/%m/erg_' + instrument + '_' + level + '_' + datatype + '_%Y%m%d_v??_??.cdf'
elif instrument == 'lepi':
pathformat = 'satellite/erg/' + instrument + '/' + level + '/' + datatype + '/%Y/%m/erg_' + instrument + '_' + level + '_' + datatype + '_%Y%m%d_v??_??.cdf'
elif instrument == 'mepe':
pathformat = 'satellite/erg/' + instrument + '/' + level + '/' + datatype + '/%Y/%m/erg_' + instrument + '_' + level + '_' + datatype + '_%Y%m%d_v??_??.cdf'
elif instrument == 'mepi':
pathformat = 'satellite/erg/' + instrument + '/' + level + '/' + datatype + '/%Y/%m/erg_' + instrument + '_' + level + '_' + datatype + '_%Y%m%d_v??_??.cdf'
elif instrument == 'pwe_ofa':
pathformat = 'satellite/erg/pwe/ofa/' + level + '/' + datatype + '/%Y/%m/erg_' + instrument + '_' + level + '_' + datatype + '_%Y%m%d_v??_??.cdf'
elif instrument == 'pwe_efd':
pathformat = 'satellite/erg/pwe/efd/' + level + '/' + datatype + '/%Y/%m/erg_' + instrument + '_' + level + '_' + datatype + '_%Y%m%d_v??_??.cdf'
elif instrument == 'pwe_hfa':
pathformat = 'satellite/erg/pwe/hfa/' + level + '/' + datatype + '/' + mode + '/%Y/%m/erg_' + instrument + '_' + level + '_' + datatype + '_' + mode + '_%Y%m%d_v??_??.cdf'
elif instrument == 'xep':
pathformat = 'satellite/erg/' + instrument + '/' + level + '/' + datatype + '/%Y/%m/erg_' + instrument + '_' + level + '_' + datatype + '_%Y%m%d_v??_??.cdf'
# find the full remote path names using the trange
remote_names = dailynames(file_format=pathformat,
trange=trange,
res=file_res)
out_files = []
files = download(remote_file=remote_names,
remote_path=CONFIG['remote_data_dir'],
local_path=CONFIG['local_data_dir'],
no_download=no_update,
last_version=True,
username=uname,
password=passwd)
if files is not None:
for file in files:
out_files.append(file)
out_files = sorted(out_files)
if downloadonly:
return out_files
tvars = cdf_to_tplot(out_files,
prefix=prefix,
suffix=suffix,
get_support_data=get_support_data,
varformat=varformat,
varnames=varnames,
notplot=notplot)
if notplot:
return tvars
if time_clip:
for new_var in tvars:
tclip(new_var, trange[0], trange[1], suffix='')
return tvars
def load_csa(trange=['2001-02-01', '2001-02-03'],
probes=['C1'],
datatypes=['CP_CIS-CODIF_HS_H1_MOMENTS'],
downloadonly=False,
time_clip=True,
suffix='',
get_support_data=False,
varformat=None,
varnames=[],
notplot=False):
"""Load data using the Cluster Science Data archive.
Parameters:
trange : list of str
Time range [start, end].
probes : list of str
List of Cluster probes.
Use ['*'] to load all. See cl_master_probes().
datatypes : list of str
List of Cluster data types.
Use ['*'] to load all. See cl_master_datatypes().
downloadonly: bool
If true, do not use cdf_to_tplot.
time_clip: bool
If true, apply time clip to data.
suffix: str (for pytplot)
The tplot variable names will be given this suffix. By default,
no suffix is added.
get_support_data: bool (for pytplot)
Data with an attribute "VAR_TYPE" with a value of "support_data"
will be loaded into tplot. By default, only loads in data with a
"VAR_TYPE" attribute of "data".
varformat : str (for pytplot)
The file variable formats to load into tplot. Wildcard character
"*" is accepted. By default, all variables are loaded in.
varnames: str or list of str (for pytplot)
Load these variables only. If [] or ['*'], then load everything.
notplot: bool (for pytplot)
If True, then data are returned in a hash table instead of
being stored in tplot variables (useful for debugging, and
access to multi-dimensional data products)
Returns:
List of tplot variables created (unless notplot keyword is used).
"""
# Empty output in case of errors.
tvars = []
# Start and end dates
start_date = cl_format_time(trange[0])
end_date = cl_format_time(trange[1])
# Delivery format
delivery_format = 'CDF_ISTP'
# Delivery interval
delivery_interval = 'ALL'
# TODO: Create a function that can resolve wildcards
# similar to IDL spedas ssl_check_valid_name
# my_datatypes=ssl_check_valid_name(uc_datatypes,master_datatypes)
# my_probes=ssl_check_valid_name(uc_probes,master_probes)
if not probes: # list is empty
return tvars
elif probes[0] == '*': # load all probes
probes = cl_master_probes()
if not datatypes: # list is empty
return tvars
elif datatypes[0] == '*': # load all probes
datatypes = cl_master_datatypes()
# Construct the query string
base_url = 'https://csa.esac.esa.int/csa-sl-tap/data?'
query_string = ('retrieval_type=PRODUCT&START_DATE=' + start_date +
'&END_DATE=' + end_date +
'&DELIVERY_FORMAT=' + delivery_format +
'&DELIVERY_INTERVAL=' + delivery_interval +
'&NON_BROWSER')
for p in probes:
for d in datatypes:
query_string += '&DATASET_ID=' + p + '_' + d
# Encode the url urllib.parse.quote
url = base_url + (query_string)
local_path = CONFIG['local_data_dir']
Path(local_path).mkdir(parents=True, exist_ok=True)
out_gz = local_path + 'temp_cluster_file.tar.gz' # Temp file name
# Download the file.
print("Downloading Cluster data, please wait....")
try:
r = requests.get(url, allow_redirects=True)
r.raise_for_status()
except requests.exceptions.HTTPError as err:
print("Download HTTP error: ", err)
return tvars
except requests.exceptions.RequestException as e:
print("Download error: ", e)
return tvars
print("Download complete.")
# Open the downloaded file.
with open(out_gz, 'wb') as w:
w.write(r.content)
# Extract the tar archive.
tar = tarfile.open(out_gz, "r:gz")
f = tar.getnames()
tar.extractall(path=local_path)
tar.close()
# Remove the tar.gz file but keep the extracted.
os.remove(out_gz)
# Get unique set of files.
f_set = set(f)
# File list with full path.
out_files = [local_path+s for s in list(f_set)]
out_files = sorted(out_files)
if downloadonly:
return out_files
# Load data into tplot
try:
tvars = cdf_to_tplot(out_files,
suffix=suffix,
get_support_data=get_support_data,
varformat=varformat,
varnames=varnames,
notplot=notplot)
except IndexError as e:
print("cdf_to_tplot cannot load Cluster cdf file.")
print("File: ", out_files[0])
print("IndexError:", e)
return tvars
except TypeError as e:
print("cdf_to_tplot cannot load Cluster cdf file.")
print("File: ", out_files[0])
print("TypeError:", e)
return tvars
if notplot:
return tvars
if time_clip:
for new_var in tvars:
tclip(new_var, trange[0], trange[1], suffix='')
return tvars
def load(
trange=['2018-11-5', '2018-11-6'],
probe='a',
instrument='emfisis',
level='l3',
datatype='magnetometer',
suffix='',
cadence='4sec', # for EMFISIS mag data
coord='sm', # for EMFISIS mag data
wavetype='waveform', # for EMFISIS waveform data
rel='rel04', # for ECT data
get_support_data=False,
varformat=None,
varnames=[],
downloadonly=False,
notplot=False,
no_update=False,
time_clip=False):
"""
This function loads Van Allen Probes (RBSP) data; this function is not meant
to be called directly; instead, see the wrappers:
pyspedas.rbsp.emfisis
pyspedas.rbsp.rbspice
pyspedas.rbsp.efw
pyspedas.rbsp.mageis
pyspedas.rbsp.hope
pyspedas.rbsp.rept
pyspedas.rbsp.rps
"""
if not isinstance(probe, list):
probe = [probe]
out_files = []
for prb in probe:
if instrument == 'emfisis':
if datatype == 'density' or datatype == 'housekeeping' or datatype == 'wna-survey':
pathformat = 'rbsp' + prb + '/' + level + '/' + instrument + '/' + datatype + '/%Y/rbsp-' + prb + '_' + datatype + '_' + instrument + '-' + level + '_%Y%m%d_v*.cdf'
elif datatype == 'wfr' or datatype == 'hfr':
pathformat = 'rbsp' + prb + '/' + level + '/' + instrument + '/' + datatype + '/' + wavetype + '/%Y/rbsp-' + prb + '_' + datatype + '-' + wavetype + '_' + instrument + '-' + level + '_%Y%m%d*_v*.cdf'
else:
if level == 'l2' and datatype == 'magnetometer':
pathformat = 'rbsp' + prb + '/' + level + '/' + instrument + '/' + datatype + '/uvw/%Y/rbsp-' + prb + '_' + datatype + '_uvw_' + instrument + '-' + level + '_%Y%m%d*_v*.cdf'
else:
pathformat = 'rbsp' + prb + '/' + level + '/' + instrument + '/' + datatype + '/' + cadence + '/' + coord + '/%Y/rbsp-' + prb + '_' + datatype + '_' + cadence + '-' + coord + '_' + instrument + '-' + level + '_%Y%m%d_v*.cdf'
elif instrument == 'rbspice':
pathformat = 'rbsp' + prb + '/' + level + '/' + instrument + '/' + datatype + '/%Y/rbsp-' + prb + '-' + instrument + '_lev-' + str(
level[-1]) + '?' + datatype + '_%Y%m%d_v*.cdf'
elif instrument == 'efw':
if level == 'l3':
pathformat = 'rbsp' + prb + '/' + level + '/' + instrument + '/%Y/rbsp' + prb + '_' + instrument + '-' + level + '_%Y%m%d_v??.cdf'
else:
pathformat = 'rbsp' + prb + '/' + level + '/' + instrument + '/' + datatype + '/%Y/rbsp' + prb + '_' + instrument + '-' + level + '_' + datatype + '_%Y%m%d_v??.cdf'
elif instrument == 'mageis':
pathformat = 'rbsp' + prb + '/' + level + '/ect/' + instrument + '/sectors/' + rel + '/%Y/rbsp' + prb + '_' + rel + '_ect-mageis-' + level + '_%Y%m%d_v*.cdf'
elif instrument == 'hope':
if datatype == 'moments':
pathformat = 'rbsp' + prb + '/' + level + '/ect/' + instrument + '/' + datatype + '/' + rel + '/%Y/rbsp' + prb + '_' + rel + '_ect-hope-mom-' + level + '_%Y%m%d_v*.cdf'
elif datatype == 'pitchangle':
pathformat = 'rbsp' + prb + '/' + level + '/ect/' + instrument + '/' + datatype + '/' + rel + '/%Y/rbsp' + prb + '_' + rel + '_ect-hope-pa-' + level + '_%Y%m%d_v*.cdf'
elif datatype == 'spinaverage':
pathformat = 'rbsp' + prb + '/' + level + '/ect/' + instrument + '/' + datatype + '/' + rel + '/%Y/rbsp' + prb + '_' + rel + '_ect-hope-sci-' + level + 'sa_%Y%m%d_v*.cdf'
elif instrument == 'rept':
pathformat = 'rbsp' + prb + '/' + level + '/ect/' + instrument + '/sectors/' + rel + '/%Y/rbsp' + prb + '_' + rel + '_ect-rept-sci-' + level + '_%Y%m%d_v*.cdf'
elif instrument == 'rps':
if datatype == 'rps-1min':
pathformat = 'rbsp' + prb + '/' + level + '/rps/psbr-rps-1min/%Y/rbsp' + prb + '_' + level + '-1min_psbr-rps_%Y%m%d_v*.cdf'
elif datatype == 'rps':
pathformat = 'rbsp' + prb + '/' + level + '/rps/psbr-rps/%Y/rbsp' + prb + '_' + level + '_psbr-rps_%Y%m%d_v*.cdf'
# find the full remote path names using the trange
remote_names = dailynames(file_format=pathformat, trange=trange)
files = download(remote_file=remote_names,
remote_path=CONFIG['remote_data_dir'],
local_path=CONFIG['local_data_dir'],
no_download=no_update)
if files is not None:
for file in files:
out_files.append(file)
out_files = sorted(out_files)
if downloadonly:
return out_files
tvars = cdf_to_tplot(out_files,
suffix=suffix,
get_support_data=get_support_data,
varformat=varformat,
varnames=varnames,
notplot=notplot)
if notplot:
return tvars
if time_clip:
for new_var in tvars:
tclip(new_var, trange[0], trange[1], suffix='')
return tvars
def mms_load_data(trange=['2015-10-16', '2015-10-17'],
probe='1',
data_rate='srvy',
level='l2',
instrument='fgm',
datatype='',
varformat=None,
prefix='',
suffix='',
get_support_data=False,
time_clip=False,
no_update=False,
center_measurement=False,
available=False,
notplot=False,
latest_version=False,
major_version=False,
min_version=None,
cdf_version=None,
spdf=False):
"""
This function loads MMS data into pyTplot variables
This function is not meant to be called directly. Please see the individual load routines for documentation and use.
"""
if not isinstance(probe, list): probe = [probe]
if not isinstance(data_rate, list): data_rate = [data_rate]
if not isinstance(level, list): level = [level]
if not isinstance(datatype, list): datatype = [datatype]
probe = [str(p) for p in probe]
# allows the user to pass in trange as list of datetime objects
if type(trange[0]) == datetime and type(trange[1]) == datetime:
trange = [
time_string(trange[0].timestamp()),
time_string(trange[1].timestamp())
]
# allows the user to pass in trange as a list of floats (unix times)
if isinstance(trange[0], float):
trange[0] = time_string(trange[0])
if isinstance(trange[1], float):
trange[1] = time_string(trange[1])
start_date = parse(trange[0]).strftime(
'%Y-%m-%d') # need to request full day, then parse out later
end_date = parse(time_string(time_double(trange[1]) - 0.1)).strftime(
'%Y-%m-%d-%H-%M-%S'
) # -1 second to avoid getting data for the next day
download_only = CONFIG['download_only']
no_download = False
if no_update or CONFIG['no_download']: no_download = True
if spdf:
return mms_load_data_spdf(trange=trange,
probe=probe,
data_rate=data_rate,
level=level,
instrument=instrument,
datatype=datatype,
varformat=varformat,
suffix=suffix,
get_support_data=get_support_data,
time_clip=time_clip,
no_update=no_update,
center_measurement=center_measurement,
notplot=notplot,
latest_version=latest_version,
major_version=major_version,
min_version=min_version,
cdf_version=cdf_version)
user = None
if not no_download:
sdc_session, user = mms_login_lasp()
out_files = []
available_files = []
for prb in probe:
for drate in data_rate:
for lvl in level:
for dtype in datatype:
if user is None:
url = 'https://lasp.colorado.edu/mms/sdc/public/files/api/v1/file_info/science?start_date=' + start_date + '&end_date=' + end_date + '&sc_id=mms' + prb + '&instrument_id=' + instrument + '&data_rate_mode=' + drate + '&data_level=' + lvl
else:
url = 'https://lasp.colorado.edu/mms/sdc/sitl/files/api/v1/file_info/science?start_date=' + start_date + '&end_date=' + end_date + '&sc_id=mms' + prb + '&instrument_id=' + instrument + '&data_rate_mode=' + drate + '&data_level=' + lvl
if dtype != '':
url = url + '&descriptor=' + dtype
if CONFIG['debug_mode']: logging.info('Fetching: ' + url)
if no_download == False:
# query list of available files
try:
with warnings.catch_warnings():
warnings.simplefilter("ignore",
category=ResourceWarning)
http_json = sdc_session.get(
url, verify=True).json()
if CONFIG['debug_mode']:
logging.info(
'Filtering the results down to your trange'
)
files_in_interval = mms_files_in_interval(
http_json['files'], trange)
if available:
for file in files_in_interval:
logging.info(
file['file_name'] + ' (' + str(
np.round(file['file_size'] /
(1024. * 1024),
decimals=1)) + ' MB)')
available_files.append(file['file_name'])
continue
for file in files_in_interval:
file_date = parse(file['timetag'])
if dtype == '':
out_dir = os.sep.join([
CONFIG['local_data_dir'], 'mms' + prb,
instrument, drate, lvl,
file_date.strftime('%Y'),
file_date.strftime('%m')
])
else:
out_dir = os.sep.join([
CONFIG['local_data_dir'], 'mms' + prb,
instrument, drate, lvl, dtype,
file_date.strftime('%Y'),
file_date.strftime('%m')
])
if drate.lower() == 'brst':
out_dir = os.sep.join(
[out_dir,
file_date.strftime('%d')])
out_file = os.sep.join(
[out_dir, file['file_name']])
if CONFIG['debug_mode']:
logging.info('File: ' + file['file_name'] +
' / ' + file['timetag'])
if os.path.exists(out_file) and str(
os.stat(out_file).st_size) == str(
file['file_size']):
if not download_only:
logging.info('Loading ' + out_file)
out_files.append(out_file)
continue
if user is None:
download_url = 'https://lasp.colorado.edu/mms/sdc/public/files/api/v1/download/science?file=' + file[
'file_name']
else:
download_url = 'https://lasp.colorado.edu/mms/sdc/sitl/files/api/v1/download/science?file=' + file[
'file_name']
logging.info('Downloading ' +
file['file_name'] + ' to ' +
out_dir)
with warnings.catch_warnings():
warnings.simplefilter(
"ignore", category=ResourceWarning)
fsrc = sdc_session.get(download_url,
stream=True,
verify=True)
ftmp = NamedTemporaryFile(delete=False)
with open(ftmp.name, 'wb') as f:
copyfileobj(fsrc.raw, f)
if not os.path.exists(out_dir):
os.makedirs(out_dir)
# if the download was successful, copy to data directory
copy(ftmp.name, out_file)
out_files.append(out_file)
fsrc.close()
ftmp.close()
except requests.exceptions.ConnectionError:
# No/bad internet connection; try loading the files locally
logging.error('No internet connection!')
if out_files == []:
if not download_only:
logging.info('Searching for local files...')
out_files = mms_get_local_files(
prb, instrument, drate, lvl, dtype, trange)
if out_files == [] and CONFIG[
'mirror_data_dir'] != None:
# check for network mirror; note: network mirrors are assumed to be read-only
# and we always copy the files from the mirror to the local data directory
# before trying to load into tplot variables
logging.info(
'No local files found; checking network mirror...'
)
out_files = mms_get_local_files(prb,
instrument,
drate,
lvl,
dtype,
trange,
mirror=True)
if not no_download:
sdc_session.close()
if available:
return available_files
if not download_only:
out_files = sorted(out_files)
filtered_out_files = mms_file_filter(out_files,
latest_version=latest_version,
major_version=major_version,
min_version=min_version,
version=cdf_version)
if filtered_out_files == []:
logging.info('No matching CDF versions found.')
return
new_variables = cdf_to_tplot(filtered_out_files,
varformat=varformat,
merge=True,
get_support_data=get_support_data,
prefix=prefix,
suffix=suffix,
center_measurement=center_measurement,
notplot=notplot)
if notplot:
return new_variables
if new_variables == []:
logging.warning('No data loaded.')
return
if time_clip:
for new_var in new_variables:
tclip(new_var, trange[0], trange[1], suffix='')
return new_variables
else:
return out_files
def load_data(filenames=None,
instruments=None,
level='l2',
insitu=True,
iuvs=False,
start_date='2014-01-01',
end_date='2020-01-01',
update_prefs=False,
only_update_prefs=False,
local_dir=None,
list_files=False,
new_files=False,
exclude_orbit_file=False,
download_only=False,
varformat=None,
prefix='',
suffix='',
get_support_data=False):
"""
This function downloads MAVEN data loads it into tplot variables, if applicable.
"""
# 1. Download files
maven_files = maven_filenames(filenames, instruments, level, insitu, iuvs,
start_date, end_date, update_prefs,
only_update_prefs, local_dir)
# Keep track of what files are downloaded
downloaded_files = []
for instr in maven_files.keys():
if maven_files[instr]:
s = maven_files[instr][0]
data_dir = maven_files[instr][1]
public = maven_files[instr][2]
if list_files:
for f in s:
print(f)
return
if new_files:
s = get_new_files(s, data_dir, instr, level)
print("Your request will download a total of: " + str(len(s)) +
" files for instrument " + str(instr))
print('Would you like to proceed with the download? ')
valid_response = False
cancel = False
while not valid_response:
response = (input('(y/n) > '))
if response == 'y' or response == 'Y':
valid_response = True
cancel = False
elif response == 'n' or response == 'N':
print('Cancelled download. Returning...')
valid_response = True
cancel = True
else:
print('Invalid input. Please answer with y or n.')
if cancel:
continue
if not exclude_orbit_file:
print(
"Before downloading data files, checking for updated orbit # file from naif.jpl.nasa.gov"
)
print("")
get_orbit_files()
i = 0
display_progress(i, len(s))
for f in s:
i = i + 1
full_path = create_dir_if_needed(f, data_dir, level)
get_file_from_site(f, public, full_path)
display_progress(i, len(s))
downloaded_files.append(os.path.join(full_path, f))
# 2. Load files into tplot
if downloaded_files:
# Flatten out downloaded files from list of lists of filenames
if isinstance(downloaded_files[0], list):
downloaded_files = [
item for sublist in downloaded_files for item in sublist
]
# Only load in files into tplot if we actually downloaded CDF files
cdf_files = [f for f in downloaded_files if '.cdf' in f]
if not download_only:
# Create tplot variables
downloaded_tplot_vars = pytplot.cdf_to_tplot(
cdf_files,
varformat=varformat,
get_support_data=get_support_data,
prefix=prefix,
suffix=suffix,
merge=True)
return downloaded_tplot_vars
def load(trange=['2018-11-5', '2018-11-6'],
probe='1',
instrument='fgm',
datatype='up',
suffix='',
get_support_data=False,
varformat=None,
varnames=[],
downloadonly=False,
notplot=False,
no_update=False,
time_clip=False):
"""
This function loads data from the Cluster mission; this function is not meant
to be called directly; instead, see the wrappers:
pyspedas.cluster.fgm
pyspedas.cluster.aspoc
pyspedas.cluster.cis
pyspedas.cluster.dwp
pyspedas.cluster.edi
pyspedas.cluster.efw
pyspedas.cluster.peace
pyspedas.cluster.rapid
pyspedas.cluster.staff
pyspedas.cluster.wbd
pyspedas.cluster.whi
"""
if not isinstance(probe, list):
probe = [probe]
probe = [str(prb)
for prb in probe] # these will need to be strings from now on
out_files = []
res = 24 * 3600
if instrument != 'wbd':
# note: can't use last_version with WBD data due to using wild cards for the times (and not just in the version)
last_version = True
else:
last_version = False
for prb in probe:
if instrument == 'fgm':
if datatype == 'cp':
pathformat = 'c' + prb + '/cp/%Y/c' + prb + '_cp_fgm_spin_%Y%m%d_v??.cdf'
else:
pathformat = 'c' + prb + '/' + datatype + '/' + instrument + '/%Y/c' + prb + '_' + datatype + '_' + instrument + '_%Y%m%d_v??.cdf'
elif instrument == 'aspoc':
pathformat = 'c' + prb + '/' + datatype + '/asp/%Y/c' + prb + '_' + datatype + '_asp_%Y%m%d_v??.cdf'
elif instrument == 'cis':
pathformat = 'c' + prb + '/' + datatype + '/' + instrument + '/%Y/c' + prb + '_' + datatype + '_' + instrument + '_%Y%m%d_v??.cdf'
elif instrument == 'dwp':
pathformat = 'c' + prb + '/' + datatype + '/' + instrument + '/%Y/c' + prb + '_' + datatype + '_' + instrument + '_%Y%m%d_v??.cdf'
elif instrument == 'edi':
pathformat = 'c' + prb + '/' + datatype + '/' + instrument + '/%Y/c' + prb + '_' + datatype + '_' + instrument + '_%Y%m%d_v??.cdf'
elif instrument == 'efw':
pathformat = 'c' + prb + '/' + datatype + '/' + instrument + '/%Y/c' + prb + '_' + datatype + '_' + instrument + '_%Y%m%d_v??.cdf'
elif instrument == 'peace':
pathformat = 'c' + prb + '/' + datatype + '/pea/%Y/c' + prb + '_' + datatype + '_pea_%Y%m%d_v??.cdf'
elif instrument == 'rapid':
pathformat = 'c' + prb + '/' + datatype + '/rap/%Y/c' + prb + '_' + datatype + '_rap_%Y%m%d_v??.cdf'
elif instrument == 'staff':
pathformat = 'c' + prb + '/' + datatype + '/sta/%Y/c' + prb + '_' + datatype + '_sta_%Y%m%d_v??.cdf'
elif instrument == 'whi':
pathformat = 'c' + prb + '/' + datatype + '/' + instrument + '/%Y/c' + prb + '_' + datatype + '_' + instrument + '_%Y%m%d_v??.cdf'
elif instrument == 'wbd':
pathformat = 'c' + prb + '/' + instrument + '/%Y/%m/c' + prb + '_' + datatype + '_' + instrument + '_%Y%m%d%H%M_v??.cdf'
res = 600.0
# find the full remote path names using the trange
remote_names = dailynames(file_format=pathformat,
trange=trange,
res=res)
files = download(remote_file=remote_names,
remote_path=CONFIG['remote_data_dir'],
local_path=CONFIG['local_data_dir'],
no_download=no_update,
last_version=last_version)
if files is not None:
for file in files:
out_files.append(file)
out_files = sorted(out_files)
if downloadonly:
return out_files
tvars = cdf_to_tplot(out_files,
suffix=suffix,
get_support_data=get_support_data,
varformat=varformat,
varnames=varnames,
notplot=notplot)
if notplot:
return tvars
if time_clip:
for new_var in tvars:
tclip(new_var, trange[0], trange[1], suffix='')
return tvars
def load(trange=['2009-01-01', '2009-01-02'],
instrument='vhm',
datatype='1min',
suffix='',
get_support_data=False,
varformat=None,
varnames=[],
downloadonly=False,
notplot=False,
no_update=False,
time_clip=False):
"""
This function is not meant to be called directly; please see the instrument specific wrappers in __init__.py
"""
if instrument == 'vhm':
pathformat = 'mag_cdaweb/vhm_'+datatype+'/%Y/uy_'+datatype+'_vhm_%Y%m%d_v??.cdf'
elif instrument == 'swoops':
if datatype in ['bai_m0', 'bai_m1', 'bae_m0']:
pathformat = 'plasma/swoops_cdaweb/'+datatype+'/%Y/uy_'+datatype.split('_')[1]+'_'+datatype.split('_')[0]+'_%Y%m%d_v??.cdf'
else:
pathformat = 'plasma/swoops_cdaweb/'+datatype+'/%Y/uy_'+datatype+'_%Y0101_v??.cdf'
elif instrument == 'swics':
pathformat = 'plasma/swics_cdaweb/'+datatype+'/%Y/uy_'+datatype.split('_')[1]+'_'+datatype.split('_')[0]+'_%Y%m%d_v??.cdf'
elif instrument == 'urap':
pathformat = 'radio/urap_cdaweb/'+datatype+'/%Y/uy_'+datatype.split('_')[1]+'_'+datatype.split('_')[0]+'_%Y%m%d_v??.cdf'
elif instrument == 'epac':
if datatype == 'epac_m1':
pathformat = 'particle/epac_cdaweb/'+datatype+'/%Y/uy_m1_epa_%Y%m%d_v??.cdf'
elif instrument == 'hiscale':
pathformat = 'particle/hiscale_cdaweb/'+datatype+'/%Y/uy_'+datatype.split('_')[1]+'_'+datatype.split('_')[0]+'_%Y%m%d_v??.cdf'
elif instrument == 'cospin':
pathformat = 'particle/cospin_cdaweb/'+datatype+'/%Y/uy_m0_'+datatype+'_%Y%m%d_v??.cdf'
elif instrument == 'grb':
pathformat = 'gamma/grb_cdaweb/'+datatype+'/%Y/uy_'+datatype.split('_')[1]+'_'+datatype.split('_')[0]+'_%Y%m%d_v??.cdf'
# find the full remote path names using the trange
remote_names = dailynames(file_format=pathformat, trange=trange)
out_files = []
files = download(remote_file=remote_names, remote_path=CONFIG['remote_data_dir'], local_path=CONFIG['local_data_dir'], no_download=no_update)
if files is not None:
for file in files:
out_files.append(file)
out_files = sorted(out_files)
if downloadonly:
return out_files
tvars = cdf_to_tplot(out_files, suffix=suffix, get_support_data=get_support_data, varformat=varformat, varnames=varnames, notplot=notplot)
if notplot:
return tvars
if time_clip:
for new_var in tvars:
tclip(new_var, trange[0], trange[1], suffix='')
return tvars
def load(trange=['2013-11-5', '2013-11-6'],
instrument='mgf',
datatype='k0',
suffix='',
get_support_data=False,
varformat=None,
varnames=[],
downloadonly=False,
notplot=False,
no_update=False,
time_clip=False):
"""
This function loads data from the Geotail mission; this function is not meant
to be called directly; instead, see the wrappers:
pyspedas.geotail.mgf
pyspedas.geotail.efd
pyspedas.geotail.lep
pyspedas.geotail.cpi
pyspedas.geotail.epi
pyspedas.geotail.pwi
"""
tvars_created = []
if instrument == 'mgf':
if datatype == 'k0':
pathformat = 'mgf/mgf_k0/%Y/ge_' + datatype + '_mgf_%Y%m%d_v??.cdf'
elif datatype == 'eda3sec' or datatype == 'edb3sec':
pathformat = 'mgf/' + datatype + '_mgf/%Y/ge_' + datatype + '_mgf_%Y%m%d_v??.cdf'
elif instrument == 'efd':
pathformat = instrument + '/' + instrument + '_' + datatype + '/%Y/ge_' + datatype + '_' + instrument + '_%Y%m%d_v??.cdf'
elif instrument == 'lep':
if datatype == 'k0':
pathformat = 'lep/lep_k0/%Y/ge_' + datatype + '_lep_%Y%m%d_v??.cdf'
elif instrument == 'cpi':
pathformat = instrument + '/' + instrument + '_' + datatype + '/%Y/ge_' + datatype + '_' + instrument + '_%Y%m%d_v??.cdf'
elif instrument == 'epi':
pathformat = 'epic/' + instrument + '_' + datatype + '/%Y/ge_' + datatype + '_' + instrument + '_%Y%m%d_v??.cdf'
elif instrument == 'pwi':
pathformat = instrument + '/' + instrument + '_' + datatype + '/%Y/ge_' + datatype + '_' + instrument + '_%Y%m%d_v??.cdf'
# find the full remote path names using the trange
remote_names = dailynames(file_format=pathformat, trange=trange)
out_files = []
files = download(remote_file=remote_names,
remote_path=CONFIG['remote_data_dir'],
local_path=CONFIG['local_data_dir'],
no_download=no_update)
if files is not None:
for file in files:
out_files.append(file)
out_files = sorted(out_files)
if downloadonly:
return out_files
tvars = cdf_to_tplot(out_files,
suffix=suffix,
get_support_data=get_support_data,
varformat=varformat,
varnames=varnames,
notplot=notplot)
if notplot:
return tvars
if time_clip:
for new_var in tvars:
tclip(new_var, trange[0], trange[1], suffix='')
return tvars
def load(trange=['2004-11-5', '2004-11-6'],
instrument='lena',
datatype='k0',
suffix='',
get_support_data=False,
varformat=None,
downloadonly=False,
notplot=False,
no_update=False,
time_clip=False):
"""
This function loads IMAGE data; this function is not meant
to be called directly; instead, see the wrappers:
pyspedas.image.lena
pyspedas.image.mena
pyspedas.image.hena
pyspedas.image.rpi
pyspedas.image.euv
pyspedas.image.fuv
"""
if instrument == 'lena':
pathformat = instrument+'/'+instrument+'_'+datatype+'/%Y/im_'+datatype+'_'+instrument+'_%Y%m%d_v??.cdf'
elif instrument == 'mena':
pathformat = instrument+'/'+instrument+'_'+datatype+'/%Y/im_'+datatype+'_'+instrument+'_%Y%m%d_v??.cdf'
elif instrument == 'hena':
pathformat = instrument+'/'+instrument+'_'+datatype+'/%Y/im_'+datatype+'_'+instrument+'_%Y%m%d_v??.cdf'
elif instrument == 'rpi':
pathformat = instrument+'/'+instrument+'_'+datatype+'/%Y/im_'+datatype+'_'+instrument+'_%Y%m%d_v??.cdf'
elif instrument == 'euv':
pathformat = instrument+'/'+instrument+'_'+datatype+'/%Y/im_'+datatype+'_'+instrument+'_%Y%m%d_v??.cdf'
elif instrument == 'fuv':
pathformat = instrument+'/wic_'+datatype+'/%Y/im_'+datatype+'_wic_%Y%m%d_v??.cdf'
elif instrument == 'orbit':
if datatype == 'def_or':
pathformat = instrument+'/def_or/%Y/im_or_def_%Y%m%d_v??.cdf'
elif datatype == 'pre_or':
pathformat = instrument+'/pre_or/%Y/im_or_pre_%Y%m%d_v??.cdf'
# find the full remote path names using the trange
remote_names = dailynames(file_format=pathformat, trange=trange)
out_files = []
files = download(remote_file=remote_names, remote_path=CONFIG['remote_data_dir'], local_path=CONFIG['local_data_dir'], no_download=no_update)
if files is not None:
for file in files:
out_files.append(file)
out_files = sorted(out_files)
if downloadonly:
return out_files
tvars = cdf_to_tplot(out_files, suffix=suffix, merge=True, get_support_data=get_support_data, varformat=varformat, notplot=notplot)
if notplot:
return tvars
if time_clip:
for new_var in tvars:
tclip(new_var, trange[0], trange[1], suffix='')
return tvars
def load_data(filenames=None,
instruments=None,
level='l2',
type=None,
insitu=True,
iuvs=False,
start_date='2014-01-01',
end_date='2020-01-01',
update_prefs=False,
only_update_prefs=False,
local_dir=None,
list_files=False,
new_files=True,
exclude_orbit_file=False,
download_only=False,
varformat=None,
varnames=[],
prefix='',
suffix='',
get_support_data=False,
auto_yes=False):
"""
This function downloads MAVEN data loads it into tplot variables, if applicable.
"""
if not isinstance(instruments, list) and instruments is not None:
instruments = [instruments]
if not isinstance(type, list) and type is not None:
type = [type]
if not isinstance(filenames, list) and filenames is not None:
filenames = [filenames]
# 1. Get a list of MAVEN files queries from the above seach parameters
maven_files = maven_filenames(filenames, instruments, level, insitu, iuvs,
start_date, end_date, update_prefs,
only_update_prefs, local_dir)
# If we are not asking for KP data, this flag ensures only ancillary data is loaded in from the KP files
if level != 'kp':
ancillary_only = True
else:
ancillary_only = False
# Convert to list
if not isinstance(type, list):
type = [type]
# Keep track of what files are downloaded
files_to_load = []
# Loop through all instruments, download files locally if needed
for instr in maven_files.keys():
bn_files_to_load = []
if maven_files[instr]:
s = maven_files[instr][0]
data_dir = maven_files[instr][1]
public = maven_files[instr][2]
# Add to list of files to load
for f in s:
# Filter by type
if type != [None] and instr != 'kp':
file_type_match = False
desc = l2_regex.match(f).group("description")
for t in type:
if t in desc:
file_type_match = True
if not file_type_match:
continue
# Check if the files are KP data
if instr == 'kp':
full_path = create_dir_if_needed(f, data_dir, 'insitu')
else:
full_path = create_dir_if_needed(f, data_dir, level)
bn_files_to_load.append(f)
files_to_load.append(os.path.join(full_path, f))
if list_files:
for f in s:
print(f)
return
if new_files:
if instr == 'kp':
s = get_new_files(bn_files_to_load, data_dir, instr,
'insitu')
else:
s = get_new_files(bn_files_to_load, data_dir, instr, level)
if len(s) == 0:
continue
print("Your request will download a total of: " + str(len(s)) +
" files for instrument " + str(instr))
print('Would you like to proceed with the download? ')
valid_response = False
cancel = False
if auto_yes:
valid_response = True
while not valid_response:
response = (input('(y/n) > '))
if response == 'y' or response == 'Y':
valid_response = True
cancel = False
elif response == 'n' or response == 'N':
print('Cancelled download. Returning...')
valid_response = True
cancel = True
else:
print('Invalid input. Please answer with y or n.')
if cancel:
continue
i = 0
display_progress(i, len(s))
for f in s:
i = i + 1
if instr == 'kp':
full_path = create_dir_if_needed(f, data_dir, 'insitu')
else:
full_path = create_dir_if_needed(f, data_dir, level)
get_file_from_site(f, public, full_path)
display_progress(i, len(s))
# 2. Load files into tplot
if files_to_load:
# Flatten out downloaded files from list of lists of filenames
if isinstance(files_to_load[0], list):
files_to_load = [
item for sublist in files_to_load for item in sublist
]
# Only load in files into tplot if we actually downloaded CDF files
cdf_files = [f for f in files_to_load if '.cdf' in f]
sts_files = [f for f in files_to_load if '.sts' in f]
kp_files = [f for f in files_to_load if '.tab' in f]
loaded_tplot_vars = []
if not download_only:
for f in cdf_files:
# Loop through CDF files
desc = l2_regex.match(os.path.basename(f)).group("description")
if desc != '' and suffix == '':
created_vars = pytplot.cdf_to_tplot(
f,
varformat=varformat,
varnames=varnames,
string_encoding='utf-8',
get_support_data=get_support_data,
prefix=prefix,
suffix=desc,
merge=True)
else:
created_vars = pytplot.cdf_to_tplot(
f,
varformat=varformat,
varnames=varnames,
string_encoding='utf-8',
get_support_data=get_support_data,
prefix=prefix,
suffix=suffix,
merge=True)
# Specifically for SWIA and SWEA data, make sure the plots have log axes and are spectrograms
instr = l2_regex.match(os.path.basename(f)).group("instrument")
if instr in ["swi", "swe"]:
pytplot.options(created_vars, 'spec', 1)
loaded_tplot_vars.append(created_vars)
for f in sts_files:
# Loop through STS (Mag) files
desc = l2_regex.match(os.path.basename(f)).group("description")
if desc != '' and suffix == '':
loaded_tplot_vars.append(
pytplot.sts_to_tplot(f,
prefix=prefix,
suffix=desc,
merge=True))
else:
loaded_tplot_vars.append(
pytplot.sts_to_tplot(f,
prefix=prefix,
suffix=suffix,
merge=True))
# Remove the Decimal Day column, not really useful
for tvar in loaded_tplot_vars:
if "DDAY_" in tvar:
pytplot.del_data(tvar)
del tvar
# Flatten out the list and only grab the unique tplot variables
flat_list = list(
set([
item for sublist in loaded_tplot_vars for item in sublist
]))
# Load in KP data specifically for all of the Ancillary data (position, attitude, Ls, etc)
if kp_files != []:
kp_data_loaded = maven_kp_to_tplot(
filename=kp_files,
ancillary_only=ancillary_only,
instruments=instruments)
# Link all created KP data to the ancillary KP data
for tvar in kp_data_loaded:
pytplot.link(tvar,
"mvn_kp::spacecraft::altitude",
link_type='alt')
pytplot.link(tvar,
"mvn_kp::spacecraft::mso_x",
link_type='x')
pytplot.link(tvar,
"mvn_kp::spacecraft::mso_y",
link_type='y')
pytplot.link(tvar,
"mvn_kp::spacecraft::mso_z",
link_type='z')
pytplot.link(tvar,
"mvn_kp::spacecraft::geo_x",
link_type='geo_x')
pytplot.link(tvar,
"mvn_kp::spacecraft::geo_y",
link_type='geo_y')
pytplot.link(tvar,
"mvn_kp::spacecraft::geo_z",
link_type='geo_z')
pytplot.link(tvar,
"mvn_kp::spacecraft::sub_sc_longitude",
link_type='lon')
pytplot.link(tvar,
"mvn_kp::spacecraft::sub_sc_latitude",
link_type='lat')
# Link all created tplot variables to the corresponding KP data
for tvar in flat_list:
pytplot.link(tvar,
"mvn_kp::spacecraft::altitude",
link_type='alt')
pytplot.link(tvar, "mvn_kp::spacecraft::mso_x", link_type='x')
pytplot.link(tvar, "mvn_kp::spacecraft::mso_y", link_type='y')
pytplot.link(tvar, "mvn_kp::spacecraft::mso_z", link_type='z')
pytplot.link(tvar,
"mvn_kp::spacecraft::geo_x",
link_type='geo_x')
pytplot.link(tvar,
"mvn_kp::spacecraft::geo_y",
link_type='geo_y')
pytplot.link(tvar,
"mvn_kp::spacecraft::geo_z",
link_type='geo_z')
pytplot.link(tvar,
"mvn_kp::spacecraft::sub_sc_longitude",
link_type='lon')
pytplot.link(tvar,
"mvn_kp::spacecraft::sub_sc_latitude",
link_type='lat')
# Return list of unique KP data
return flat_list
def load(trange=['2020-06-01', '2020-06-02'],
instrument='mag',
datatype='rtn-normal',
mode=None,
level='l2',
suffix='',
get_support_data=False,
varformat=None,
varnames=[],
downloadonly=False,
notplot=False,
no_update=False,
time_clip=False):
"""
This function loads data from the Solar Orbiter mission; this function is not meant
to be called directly; instead, see the wrappers:
pyspedas.solo.mag
pyspedas.solo.epd
pyspedas.solo.rpw
pyspedas.solo.swa
"""
# Defaults for L2, L3 data
science_or_low_latency = 'science'
date_format = '%Y%m%d'
cdf_version = '??'
res = 24*3600.
if level == 'll02':
science_or_low_latency = 'low_latency'
date_format = '%Y%m%dt%H%M??-*'
cdf_version = '???'
res = 60.0
if instrument == 'mag':
pathformat = instrument+'/'+science_or_low_latency+'/'+level+'/'+datatype+'/%Y/solo_'+level+'_'+instrument+'-'+datatype+'_'+date_format+'_v'+cdf_version+'.cdf'
elif instrument == 'epd':
pathformat = instrument+'/'+science_or_low_latency+'/'+level+'/'+datatype+'/'+mode+'/%Y/solo_'+level+'_'+instrument+'-'+datatype+'-'+mode+'_'+date_format+'_v'+cdf_version+'.cdf'
elif instrument == 'rpw':
pathformat = instrument+'/'+science_or_low_latency+'/'+level+'/'+datatype+'/%Y/solo_'+level+'_'+instrument+'-'+datatype+'_'+date_format+'_v'+cdf_version+'.cdf'
elif instrument == 'swa':
if level == 'l2' or level == 'll02':
if datatype == 'pas-eflux' or datatype == 'pas-grnd-mom' or datatype == 'pas-vdf':
pathformat = instrument+'/'+science_or_low_latency+'/'+level+'/'+datatype+'/%Y/solo_'+level+'_'+instrument+'-'+datatype+'_'+date_format+'_v'+cdf_version+'.cdf'
else:
date_format = '%Y%m%dt%H%M??-*'
res = 60.0
pathformat = instrument+'/'+science_or_low_latency+'/'+level+'/'+datatype+'/%Y/solo_'+level+'_'+instrument+'-'+datatype+'_'+date_format+'_v'+cdf_version+'.cdf'
elif level == 'l1':
if datatype == 'his-pha' or datatype == 'his-sensorrates' or datatype == 'pas-3d' or datatype == 'pas-cal' or datatype == 'pas-mom':
pathformat = instrument+'/'+science_or_low_latency+'/'+level+'/'+datatype+'/%Y/solo_'+level+'_'+instrument+'-'+datatype+'_'+date_format+'_v'+cdf_version+'.cdf'
else:
date_format = '%Y%m%dt%H%M??-*'
res = 60.0
pathformat = instrument+'/'+science_or_low_latency+'/'+level+'/'+datatype+'/%Y/solo_'+level+'_'+instrument+'-'+datatype+'_'+date_format+'_v'+cdf_version+'.cdf'
# find the full remote path names using the trange
remote_names = dailynames(file_format=pathformat, trange=trange, res=res)
out_files = []
files = download(remote_file=remote_names, remote_path=CONFIG['remote_data_dir'], local_path=CONFIG['local_data_dir'], no_download=no_update)
if files is not None:
for file in files:
out_files.append(file)
out_files = sorted(out_files)
if downloadonly:
return out_files
tvars = cdf_to_tplot(out_files, suffix=suffix, get_support_data=get_support_data, varformat=varformat, varnames=varnames, notplot=notplot)
if notplot:
return tvars
if time_clip:
for new_var in tvars:
tclip(new_var, trange[0], trange[1], suffix='')
return tvars