def get_drms_files(self):
"""
Downloads the requested data from class object using drms (i.e. JSOC).
"""
import drms
client = drms.Client(email=self.email,verbose=True)
fmt = '%Y.%m.%d_%H:%M'
self.t_qstr = self.series+'[{0}_TAI-{1}_TAI@{2}]'.format(self.start.strftime(fmt),self.end.strftime(fmt),self.cadence)
#create wavelength query string
self.w_qstr = '['
for i in self.wav: self.w_qstr = self.w_qstr+'{0},'.format(int(i.value))
#remove last , and add bracket
self.w_qstr = self.w_qstr[:-1]+']'
#make the series string
self.s_qstr = '{'+self.segment+'}'
#the full query
self.qstr = self.t_qstr+self.w_qstr+self.s_qstr
#IF ERRORS WITH URL ERROR IT IS BECAUSE THE DOWNLOAD FILE SIZE IS TOO LARGE
#export the data file list
self.expt = client.export(self.qstr)
#create an array of indexes to download
index = np.arange(np.size(self.expt.urls.url))
# get file from JSOC
#set directory to current if no path set
outf = self.expt.download(self.odir,index,fname_from_rec=True)
def getfitsfile(inst, day, l):
if (inst == 'hmi'):
qblank = 'hmi.V_sht_gf_72d[{:d}d][{:d}][{:d}][138240]'
elif (inst == 'mdi'):
qblank = 'mdi.vw_V_sht_gf_72d[{:d}d][{:d}][{:d}][103680]'
else:
print("Invalid instrument, try again.")
return None
fblank='%s.%id.%i.fits'
ffile=os.path.join(datadir,fblank % (inst,day,l))
if os.path.exists(ffile):
print("Fits file already exists.")
else:
qstr=qblank.format(day,l,l)
c = drms.Client()
f = c.query(qstr,seg='data')
if (len(f) == 0):
print("No data found for that day number and degree, try again.")
return None
url = 'http://jsoc.stanford.edu' + f.data[0]
try:
urlretrieve(url, ffile)
except:
print("Failure of urlretrieve() for %s." % url)
return None
return ffile
def request_data(self, jsoc_response, method='url', **kwargs):
"""
Request that JSOC stages the data for download. This method will not
wait for the request to be staged.
Parameters
----------
jsoc_response : `~sunpy.net.jsoc.jsoc.JSOCResponse` object
The results of a query
method : {'url', 'url-tar', 'url-quick'}
Method for requesting JSOC data, can be 'url-tar', 'url' (the default) and 'url-quick'
If 'url-tar' it will request JSOC to provide single .tar file which contains all data
If 'url' it will request JSOC to provide all data as separate .fits files
If 'url-quick' (only with protocol 'as-is') provide all data as separate files,
but only if data is online.
Returns
-------
requests : `~drms.client.ExportRequest` object or
a list of `~drms.client.ExportRequest` objects
Request Id can be accessed by requests.id
Request status can be accessed by requests.status
"""
requests = []
self.query_args = jsoc_response.query_args
supported_protocols = {'fits', 'as-is'}
supported_methods = {'url-tar', 'url', 'url-quick'}
for block in jsoc_response.query_args:
ds = self._make_recordset(**block)
cd = drms.Client(email=block.get('notify', ''))
protocol = block.get('protocol', 'fits')
cutout = block.get('cutout')
if protocol not in supported_protocols:
error_message = f"Protocols other than {','.join(supported_protocols)} "\
"are not supported."
raise TypeError(error_message)
if method not in supported_methods:
error_message = f"Methods other than {','.join(supported_methods)} "\
"are not supported."
raise TypeError(error_message)
process = {'im_patch': cutout} if cutout is not None else None
if method != 'url-tar':
method = 'url' if protocol == 'fits' else 'url_quick'
r = cd.export(ds,
method=method,
protocol=protocol,
process=process)
requests.append(r)
if len(requests) == 1:
return requests[0]
return requests
def test_client_init_defaults():
c = drms.Client()
assert isinstance(c._server, ServerConfig)
assert c._server.name.lower() == 'jsoc'
assert c.email is None
assert c.verbose is False
assert c.debug is False
def create_parse_jsoc_values():
"""
Makes a network call to the VSO API that returns what keywords they support.
We take this list and register all the keywords as corresponding Attrs.
"""
here = os.path.dirname(os.path.realpath(__file__))
client = drms.Client()
# Series we are after
data_sources = ["hmi", "mdi", "aia"]
# Now get all the information we want.
series_store = []
segments = []
for series in data_sources:
info = client.series(rf'{series}\.')
for item in info:
data = client.info(item)
series_store.append((data.name, data.note))
if not data.segments.empty:
for row in data.segments.iterrows():
segments.append((row[0], row[1][-1]))
series_store = list(set(series_store))
segments = list(set(segments))
with open(os.path.join(here, 'data', 'attrs.json'), 'w') as attrs_file:
keyword_info = {}
keyword_info["series_store"] = sorted(series_store)
keyword_info["segments"] = sorted(segments)
json.dump(keyword_info, attrs_file, indent=2)
def __init__(self, verbose=False):
self.series = 'aia.lev1_euv_12s'
# initialize the drms client
self.client = drms.Client(verbose=verbose)
# obtain ALL of the keys for this series (i believe this connects to JSOC)
self.allkeys = self.client.keys(self.series)
# ---------------------------------------------
# define the series specific default variables here
# ---------------------------------------------
# default keys for a normal query over a range of times
self.default_keys = [
'T_REC',
'T_OBS',
'WAVELNTH',
'EXPTIME',
'QUALITY',
]
# default filters to use for a query
self.default_filters = ['QUALITY=0', 'EXPTIME>1.0']
# default segments for a query
self.default_segments = ['image']
# header items that change from "as-is" to "fits" formats served by JSOC.
self.hdr_keys_to_delete = ['BLD_VERS', 'TRECROUN']
self.hdr_keys_to_add = ['TRECIDX']
if verbose:
print('### Initialized DRMS client for ' + self.series)
def test_client_registered_servers(server_name):
c = drms.Client(server_name)
assert isinstance(c._server, ServerConfig)
assert c._server.name.lower() == server_name
assert c.email is None
assert c.verbose is False
assert c.debug is False
def request_mfits_by_date_HMI(moment, email=EMAIL, path_to_save='HMIdataset', verbose=False):
"""
Function for request fits from JSOC database
moment: pd.datetime object
return: filepath to the magnetogram
"""
filename = 'hmi.m_720s.' + moment.strftime('%Y%m%d_%H%M%S_TAI.magnetogram.fits')
filepath = os.path.join(path_to_save, filename)
if os.path.exists(filepath):
pass
else:
c = drms.Client(email=email, verbose=verbose)
str_for_query = 'hmi.m_720s' + moment.strftime('[%Y.%m.%d_%H:%M:%S_TAI]{magnetogram}')
logger.info('Magnetogram: {} will be downloaded ... '.format(str_for_query))
r = c.export(str_for_query, method='url', protocol='fits')
logger.debug(r)
try:
r.wait()
logger.info(r.request_url)
except Exception as e:
logger.warning('Can not wait anymore, skip this. Get Exception: {}'.format(e))
try:
logger.info("Download data and save to path {}".format(filepath))
r.download(path_to_save, verbose=verbose)
except Exception as e:
logger.error('Get error while trying download: {}'.format(e))
logger.warning('Skip this date')
return filepath
def test_email_cmdopt_set(email):
if email is None:
pytest.skip('No email was specified using --email')
else:
c = drms.Client('jsoc')
c.email = email
assert c.email == email
def hmi_drms_shmi():
c = drms.Client(email='*****@*****.**', verbose=True)
ds = 'hmi.M_720s_nrt[2019.01.17_10:00_TAI]'
r = c.export(ds, method='url', protocol='fits')
r.wait()
r.status
r.request_url
r.download(out_dir)
def get_drms_files(self):
"""
Downloads the requested data from class object using drms (i.e. JSOC).
"""
import drms
client = drms.Client(email=self.email, verbose=False)
fmt = '%Y.%m.%d_%H:%M'
self.t_qstr = self.series + '[{0}_TAI-{1}_TAI@{2}]'.format(
self.start.strftime(fmt), self.end.strftime(fmt), self.cadence)
#create wavelength query string
self.w_qstr = '['
for i in self.wav:
self.w_qstr = self.w_qstr + '{0},'.format(int(i.value))
#remove last , and add bracket if length of string is greater than 1
if len(self.w_qstr) > 1:
self.w_qstr = self.w_qstr[:-1] + ']'
else:
self.w_qstr = self.w_qstr + ']'
#make the series string
self.s_qstr = '{' + self.segment + '}'
#the full query
self.qstr = self.t_qstr + self.w_qstr + self.s_qstr
#IF ERRORS WITH URL ERROR IT IS BECAUSE THE DOWNLOAD FILE SIZE IS TOO LARGE
#export the data file list
self.expt = client.export(self.qstr)
#create an array of indexes to download
index = np.arange(np.size(self.expt.urls.url))
#get output file names to check if file already exists
#check if it is hmi or aia if hmi change file format
if 'hmi' in self.qstr:
outf = self.expt.urls.record.astype(str).str.replace(
':', '_').str.replace('{', '.').str.replace(
'}', '.').str.replace('[', '.').str.replace(
']', '.').str.replace('-', '').str.replace(
'\.\.', '.').str.replace('\.2\.', '.').str.replace(
'M_45', 'm_45') + 'fits'
else:
outf = self.expt.urls.record.astype(str).str.replace(
':', '').str.replace('{', '.').str.replace(
'}', '.').str.replace('[', '.').str.replace(
']', '.').str.replace('-', '').str.replace(
'\.\.', '.') + 'fits'
#Find if file exits is not then set check file to true so it keeps index
check_file = [os.path.isfile(self.odir + i) == False for i in outf]
#removed indices of already downloaded files
index = index[check_file]
# get new files from JSOC
#set directory to current if no path set
outf = self.expt.download(self.odir, index, fname_from_rec=True)
def main():
import drms
args = parse_args(sys.argv[1:])
# Create a Client instance
client = drms.Client(args.server,
email=args.email,
verbose=args.verbose,
debug=args.debug)
print(f'client: {client}')
def getSHARPs(t_start, t_end, method='cm', maxlon=90, discardfile=''):
"""
Identify all SHARPs within given time range and identify individual timestamp, with at least one frame within +-maxlon of Central Meridian.
Set method='cm' to choose frames closest to Central Meridian, or
method='maxflux' to choose frames with the maximum unsigned flux.
Returns (1) sorted list of SHARP numbers, (2) timestamp of frame, and (3) corresponding emergence time (next noon) as a datetime object.
List of SHARPs failing the maxlon test is written to discardfile.
"""
# Identify SHARP numbers in time range:
t1 = t_start.strftime('%Y.%m.%d_%H:%M:%S')
t2 = t_end.strftime('%Y.%m.%d_%H:%M:%S')
print('Identifying SHARPs to be considered between ' + t1 + ' and ' + t2 +
'...')
c = drms.Client()
k = c.query('hmi.sharp_cea_720s[][' + t1 + '-' + t2 + '@1d]',
key='HARPNUM, T_REC')
try:
sharps1 = list(set(k.HARPNUM))
sharps1.sort()
except:
print('ERROR: no SHARPs found in time range.')
sys.exit()
# Initialize discard file:
dfile = open(discardfile, 'w')
dfile.write(
'List of SHARPs discarded due to Central Meridian distance > %g\n' %
maxlon)
# Find single "emergence time" for each SHARP:
t_rec = []
t_em = []
sharps = []
for h in sharps1:
print(h)
t_rec1, t_em1, ivalid = SHARPtime(h, method=method, maxlon=maxlon)
if ((t_em1 >= t_start) & (ivalid)):
t_rec.append(t_rec1)
t_em.append(t_em1)
sharps.append(h)
if (not ivalid):
# Write to discard file:
dfile.write('%i\n' % (h))
dfile.close()
# Sort into order:
idx = sorted(range(len(t_em)), key=t_em.__getitem__)
sharps = [sharps[i] for i in idx]
t_rec = [t_rec[i] for i in idx]
t_em = [t_em[i] for i in idx]
print('Number of SHARPs to consider: %i' % len(sharps))
return sharps, t_rec, t_em
def dark_query(self):
#use drms module to download from JSOC (https://pypi.python.org/pypi/drms)
client = drms.Client(email=self.email, verbose=False)
fmt = '%Y.%m.%d_%H:%M'
self.qstr = 'iris.lev1[{0}_TAI-{1}_TAI][][? IMG_TYPE ~ "DARK" ?]'.format(
self.sta_dark_dt.strftime(fmt), self.end_dark_dt.strftime(fmt))
self.expt = client.export(self.qstr)
#setup string to pass write to sswidl for download
### fmt = '%Y-%m-%dT%H:%M:%S'
### self.response = client.query(jsoc.Time(self.sta_dark_dt.strftime(fmt),self.end_dark_dt.strftime(fmt)),jsoc.Series('iris.lev1'),
### jsoc.Notify('*****@*****.**'),jsoc.Segment('image'))
###
self.get_darks(client)
def downdrms(harpnum, tstart, extent, cadence, out_dir=None):
'''
Unlike the previous function, this one uses the drms module:
https://drms.readthedocs.io/en/stable/intro.html
This function takes the harp number assigned to an AR and the initial and
final time desired to fetch the data from a 'hmi.sharp_cea_720s' data
series.
The format for tstart is: '2014-01-01T00:00:00' or '2016.05.18_00:00:00'.
The Harp number will be converted to a string within the code.
It will then download the magnetic field vector in cilindrical components.
'''
# Checking path
if out_dir is None:
# Defaulting for the hardrive connected on linux.
out_dir = stdconfig.readconfig('linux', 'data') + str(harpnum) + '/'
# Checking if the path exists.
if not os.path.exists(out_dir):
os.mkdir(out_dir)
# Creating the client instance.
client = drms.Client(email='*****@*****.**', verbose=True)
# Filtering the series.
client.series(r'hmi\.sharp_cea_720s')
# Querying.
ds = 'hmi.sharp_cea_720s[' + str(
harpnum
) + '][' + tstart + '_TAI/' + extent + '@' + cadence + ']{Br, Bp, Bt, bitmap}'
# Creating the request object it contains the query id and status.
requests = client.export(ds, method='url', protocol='fits')
# Getting the request.
requests.download(out_dir)
# Checking missing data.
#missing_files = check_missing_files(harpnum=harpnum, directory=out_dir, requests=requests)
# Feedback.
print('Downloads complete!')
return (out_dir) #, missing_files)
def getmfile(inst, daystr):
if (inst == 'hmi'):
qblank = 'hmi.V_sht_modes[{:d}d][0][300][138240]'
avgurl = 'http://sun.stanford.edu/~tplarson/audio/HMI/hmi.average.modes'
elif (inst == 'mdi'):
qblank = 'mdi.vw_V_sht_modes[{:d}d][0][300][103680]'
avgurl = 'http://sun.stanford.edu/~tplarson/audio/MDI/mdi.average.modes'
else:
print("Invalid instrument, try again.")
return None
if (daystr == 'average'):
mblank = '%s.average.modes'
mfile=os.path.join(datadir,mblank % inst)
if os.path.exists(mfile):
print("Mode fits file already exists.")
else:
try:
urlretrieve(avgurl, mfile)
except:
print("Failure of urlretrieve() for %s." % avgurl)
return None
else:
mblank = '%s.%sd.modes'
mfile=os.path.join(datadir,mblank % (inst, daystr))
if os.path.exists(mfile):
print("Mode fits file already exists.")
else:
try:
day=int(daystr)
except:
print("Invalid number, try again.")
return None
qstr=qblank.format(day)
c = drms.Client()
m = c.query(qstr, seg='m6')
if (len(m) == 0):
print("No data found for that day number, try again.")
return None
url = 'http://jsoc.stanford.edu' + m.m6[0]
try:
urlretrieve(url, mfile)
except:
print("Failure of urlretrieve() for %s." % url)
return None
return mfile
def checksegments():
'''
This function will print the segments
of the hmi.sharp_cea_720s series.
While not directly relevant for the code,
this will be kept here for future reference.
'''
# using the client
c = drms.Client()
# asking for the info
si = c.info('hmi.sharp_cea_720s')
# printing the info
print(si.segments.index.values)
return
def downbitmaps(harpnum, out_dir):
'''
Unlike the previous function, this one uses the drms module:
https://drms.readthedocs.io/en/stable/intro.html
This function takes the harp number assigned to an AR and the initial and
final time desired to fetch the data from a 'hmi.sharp_cea_720s' data
series.
The format for tstart is: '2014-01-01T00:00:00' or '2016.05.18_00:00:00'.
The Harp number will be converted to a string within the code.
It will then download the magnetic field vector in cilindrical components.
'''
# Getting the timestamp from the filename.
timestamp = sorted(glob.glob(out_dir + '*Br.fits'))[0].replace(
out_dir + f'hmi.sharp_cea_720s.{harpnum}.',
'').replace('_TAI.Br.fits', '')
# Checking for the start time and subtracting a day from it.
tstart = datetime.strptime(timestamp, '%Y%m%d_%H%M%S') - timedelta(days=1)
tstart = tstart.strftime('%Y.%m.%d_%H:%M:%S')
# Creating the client instance.
client = drms.Client(email='*****@*****.**', verbose=True)
# Filtering the series.
client.series(r'hmi\.sharp_cea_720s')
# Querying.
ds = 'hmi.sharp_cea_720s[' + str(
harpnum) + '][' + tstart + '_TAI/20d@720s]{bitmap}'
# Creating the request object it contains the query id and status.
requests = client.export(ds, method='url', protocol='fits')
# Getting the request.
requests.download(out_dir)
# Feedback.
print(f'{harpnum} download complete!')
return (out_dir)
def request_data(self, jsoc_response, **kwargs):
"""
Request that JSOC stages the data for download. This method will not
wait for the request to be staged.
Parameters
----------
jsoc_response : `~sunpy.net.jsoc.jsoc.JSOCResponse` object
The results of a query
Returns
-------
requests : `~drms.ExportRequest` object or
a list of `~drms.ExportRequest` objects
Request Id can be accessed by requests.id
Request status can be accessed by requests.status
"""
requests = []
self.query_args = jsoc_response.query_args
for block in jsoc_response.query_args:
ds = self._make_recordset(**block)
cd = drms.Client(email=block.get('notify', ''))
protocol = block.get('protocol', 'fits')
if protocol != 'fits' and protocol != 'as-is':
error_message = "Protocols other than fits and as-is are "\
"are not supported."
raise TypeError(error_message)
method = 'url' if protocol == 'fits' else 'url_quick'
r = cd.export(ds, method=method, protocol=protocol)
requests.append(r)
if len(requests) == 1:
return requests[0]
return requests
def request_mfits_by_date(moment,
email,
mtype='MDI',
path_to_safe='MDIdataset'):
"""
Function for request fits from JSOC database
moment: pd.datetime object
return: filepath to the magnetogram
"""
assert mtype in ['MDI', 'HMI'], 'mtype should be "HMI" or "MDI"'
if mtype == 'MDI':
filename = 'mdi.fd_M_96m_lev182.' + moment.strftime(
'%Y%m%d_%H%M%S_TAI.data.fits')
else:
filename = 'hmi.m_720s.' + moment.strftime(
'%Y%m%d_%H%M%S_TAI.1.magnetogram.fits')
filepath = os.path.join(path_to_safe, filename)
if os.path.exists(filepath):
print('Magnetogram already exists')
return filepath
else:
c = drms.Client(email=email, verbose=True)
if mtype == 'MDI':
str_for_query = 'mdi.fd_M_96m_lev182' + moment.strftime(
'[%Y.%m.%d_%H:%M:%S_TAI]')
else:
str_for_query = 'hmi.m_720s' + moment.strftime(
'[%Y.%m.%d_%H:%M:%S_TAI]{magnetogram}')
print('Magnetogram:', str_for_query, ' will be downloaded .. ')
r = c.export(str_for_query, method='url', protocol='fits')
print(r)
r.wait()
print(r.request_url)
print("Download data and save to path", filepath)
r.download(path_to_safe)
return filepath
def SHARPtime(sharpnum, method='cm', maxlon=90):
"""
For a given SHARP, identify a single frame with (if possible) longitude centroid within +-maxlon of Central Meridian.
- If method='cm', use the frame with timestamp closest to central meridian.
- If method='maxflux', use the frame with maximum unsigned flux.
Returns (1) timestamp of chosen frame, and (2) corresponding emergence time (next noon) as a datetime object, and (3) ivalid flag True if valid frame exists and False if frame is outside +-maxlon.
"""
c = drms.Client()
# Get time series of unsigned fluxes and longitudes of this SHARP (0 is central meridian):
k = c.query('hmi.sharp_cea_720s[%i][]' % sharpnum,
key='HARPNUM, T_REC, USFLUXL, LON_FWT')
# Find individual record:
if (method == 'cm'):
rec_cm = k.LON_FWT.abs().idxmin()
k_cm = k.loc[rec_cm]
if (np.abs(k.LON_FWT[rec_cm]) <= maxlon):
ivalid = True
else:
ivalid = False
if (method == 'maxflux'):
usfluxl = k.USFLUXL.where(np.abs(k.LON_FWT) <= maxlon, other=0)
if (usfluxl.max() > 0):
rec_cm = usfluxl.abs().idxmax()
k_cm = k.loc[rec_cm]
ivalid = True
else:
rec_cm = 0
ivalid = False
t_cm = drms.to_datetime(k.T_REC[rec_cm])
# Identify emergence (completion) time - next noon:
twelve_hrs = datetime.timedelta(hours=12)
t_em = t_cm + twelve_hrs
t_em = t_em.replace(hour=12, minute=0, second=0)
return k.T_REC[rec_cm], t_em, ivalid
def readmap(rot):
"""
Reads the synoptic map for Carrington rotation rot.
[no flux correction etc, so just for display]
ARGUMENTS:
rot is the number of the required Carrington rotation (e.g. 2190)
"""
# (1) READ IN DATA
# ----------------
# The seg='Mr_polfil' downloads the polar field corrected data --> without NaN values and errors due to projection effect
# Read "Polar Field Correction for HMI Line-of-Sight Synoptic Data" by Xudong Sun, 2018 to know more
# Link: https://arxiv.org/pdf/1801.04265.pdf
try:
c = drms.Client()
seg = c.query(('hmi.synoptic_mr_polfil_720s[%4.4i]' % rot),
seg='Mr_polfil')
except:
print(
'Error downloading HMI synoptic map -- required rotation: %4.4i' %
rot)
# Extract data array: (data is stored in 2nd slot with No. 1 and not in the PRIMARY (No. 0), thus [1].data)
# typical file structure:
# No. Name Ver Type Cards Dimensions Format
# 0 PRIMARY 1 PrimaryHDU 6 ()
# 1 1 CompImageHDU 13 (3600, 1440) int32
brm = (fits.open('http://jsoc.stanford.edu' + seg.Mr_polfil[0]))[1].data
# Coordinates of original map:
nsm = np.size(brm, axis=0)
npm = np.size(brm, axis=1)
dsm = 2.0 / nsm
dpm = 2 * np.pi / npm
scm = np.linspace(-1 + 0.5 * dsm, 1 - 0.5 * dsm, nsm)
pcm = np.linspace(0.5 * dpm, 2 * np.pi - 0.5 * dpm, npm)
return brm, scm, pcm
def get_Hmi_sharp():
c = drms.Client()
startDate = datetime.strptime('2010.05.01 00:00:00', "%Y.%m.%d %H:%M:%S")
while startDate.year < 2020:
startDateString = startDate.strftime('%Y.%m.%d_%H:%M:%S')
startDate = startDate + relativedelta(weeks=+1)
endDateString = startDate.strftime('%Y.%m.%d_%H:%M:%S')
dateString = startDateString + '-' + endDateString
print(dateString)
# variable = 'T_REC,HARPNUM,TOTUSJH,TOTPOT,TOTUSJZ,ABSNJZH,SAVNCPP,USFLUX,AREA_ACR,MEANPOT,R_VALUE,SHRGT45,NOAA_AR,NOAA_NUM,NOAA_ARS,QUALITY'
variable = 'T_REC,HARPNUM,TOTUSJH,TOTPOT,TOTUSJZ,ABSNJZH,SAVNCPP,USFLUX,AREA_ACR,MEANPOT,R_VALUE,SHRGT45,MEANSHR,MEANGAM,MEANGBT,MEANGBZ,MEANGBH,MEANJZH,MEANJZD,MEANALP,NOAA_AR,NOAA_NUM,NOAA_ARS,QUALITY'
df = c.query('hmi.sharp_720s[][' + dateString + ']', key=variable)
if (df.size == 0):
continue
df.T_REC = drms.to_datetime(df.T_REC)
conn = sqlite3.connect('HMI_SHARP_SWPC_FINAL.db')
df.to_sql('02_HMI_SHARP', conn, if_exists='append', index=False)
def get_cgemLorentz():
c = drms.Client()
startDate = datetime.strptime('2010.05.01 00:00:00', "%Y.%m.%d %H:%M:%S")
while startDate.year < 2020:
startDateString = startDate.strftime('%Y.%m.%d_%H:%M:%S')
startDate = startDate + relativedelta(weeks=+1)
endDateString = startDate.strftime('%Y.%m.%d_%H:%M:%S')
dateString = startDateString + '-' + endDateString
print(dateString)
variable = 'HARPNUM, T_REC, TOTBSQ, TOTFZ, EPSZ, TOTFY, TOTFX, EPSY, EPSX, QUALITY, NOAA_ARS, NOAA_AR, NOAA_NUM'
df = c.query('cgem.lorentz[][' + dateString + ']', key=variable)
if (df.size == 0):
continue
df.T_REC = drms.to_datetime(df.T_REC)
conn = sqlite3.connect('HMI_SHARP_SWPC_FINAL.db')
df.to_sql('02_CGEM_LORENTZ', conn, if_exists='append', index=False)
def get_data(self):
"""function: get_data
This function reads the appropriate data and metadata.
Returns
-------
result: list
List containing five items:
[0] The relevant WCS keywords as a pandas dataframe
[1] FITS file containing azimuthal component of magnetic field vector as an astropy HDUList object
[2] FITS file containing field strength of magnetic field vector as an astropy HDUList object
[3] FITS file containing inclination component of magnetic field vector as an astropy HDUList object
[4] FITS file containing disambiguation information as an astropy HDUList object
"""
c = drms.Client()
try:
keys, segments = c.query(
self.recordset,
key=
'T_REC, CRPIX1, CRPIX2, CRVAL1, CRVAL2, CDELT1, CRLN_OBS, CRLT_OBS, CROTA2, DSUN_REF, RSUN_REF',
seg='inclination, azimuth, field, disambig')
except:
print("Invalid recordset specification")
sys.exit(1)
if (len(keys) > 1):
print("Specify only one record")
sys.exit(1)
baseurl = 'http://jsoc.stanford.edu'
azimuth = fits.open(baseurl + segments.azimuth[0])
field = fits.open(baseurl + segments.field[0])
inclination = fits.open(baseurl + segments.inclination[0])
disambig = fits.open(baseurl + segments.disambig[0])
return [keys, azimuth, field, inclination, disambig]
def RetrieveHMIFile(year, month, day, hour, minute, second, segment):
query = 'hmi.'
series = ''
if segment == 'continuum':
series = 'Ic_45s'
else:
series = 'M_45s'
query += series + '[' + year + '.' + month + '.' + day + '_' + hour + ':' + minute + ':' + second + '_TAI]'
#query = 'hmi.M_45s[2018.05.09_01:30:00_TAI]'
client = drms.Client()
hmi_keys, segments = client.query(query, key=drms.const.all, seg=segment)
hmi_url = 'http://jsoc.stanford.edu'
if segment == 'continuum':
hmi_url += segments.continuum[0]
else:
hmi_url += segments.magnetogram[0]
data = fits.getdata(hmi_url)
header = dict(hmi_keys.iloc[0])
path = '/tmp/'
path += 'hmi.' + series + '.' + year + month + day + '_' + hour + minute + second + '.fits'
fits.writeto(path, data)
return data, header
import drms
# Series name, start time and data segment
series = 'hmi.v_sht_modes'
tstart = '2014.06.20_00:00:00_TAI'
segname = 'm6' # 'm6', 'm18' or 'm36'
# DRMS-Server URL (or shortcut) and data url (if any) for the data segment
drms_url, data_url = 'jsoc', 'http://jsoc.stanford.edu'
#drms_url, data_url = 'kis', ''
# DRMS query string
qstr = '%s[%s]' % (series, tstart)
# Create DRMS JSON client, use debug=True to see the query URLs
c = drms.Client(drms_url)
# Send request to the DRMS server
print('Querying keyword data...\n -> %s' % qstr)
k, s = c.query(qstr,
key=['T_START', 'T_STOP', 'LMIN', 'LMAX', 'NDT'],
seg=segname)
print(' -> %d lines retrieved.' % len(k))
# Use only the first line of the query result
k = k.iloc[0]
fname = data_url + s[segname][0]
# Read the data segment
print('Reading data from %r...' % fname)
a = np.genfromtxt(fname)
def _lookup_records(self, iargs):
"""
Do a LookData request to JSOC to workout what results the query returns.
"""
keywords_default = [
'T_REC', 'TELESCOP', 'INSTRUME', 'WAVELNTH', 'CAR_ROT'
]
isMeta = iargs.get('meta', False)
c = drms.Client()
if isMeta:
keywords = '**ALL**'
else:
keywords = iargs.get('keys', keywords_default)
if 'series' not in iargs:
error_message = "Series must be specified for a JSOC Query"
raise ValueError(error_message)
if not isinstance(keywords, list) and not isinstance(keywords, str):
error_message = "Keywords can only be passed as a list or "\
"comma-separated strings."
raise TypeError(error_message)
# Raise errors for PrimeKeys
# Get a set of the PrimeKeys that exist for the given series, and check
# whether the passed PrimeKeys is a subset of that.
pkeys = c.pkeys(iargs['series'])
pkeys_passed = iargs.get(
'primekey', None) # pkeys_passes is a dict, with key-value pairs.
if pkeys_passed is not None:
if not set(list(pkeys_passed.keys())) <= set(pkeys):
error_message = "Unexpected PrimeKeys were passed. The series {series} "\
"supports the following PrimeKeys {pkeys}"
raise ValueError(
error_message.format(series=iargs['series'], pkeys=pkeys))
# Raise errors for wavelength
wavelength = iargs.get('wavelength', '')
if wavelength:
if 'WAVELNTH' not in pkeys:
error_message = "The series {series} does not support wavelength attribute."\
"The following primekeys are supported {pkeys}"
raise TypeError(
error_message.format(series=iargs['series'], pkeys=pkeys))
# Raise errors for segments
# Get a set of the segments that exist for the given series, and check
# whether the passed segments is a subset of that.
si = c.info(iargs['series'])
segs = list(
si.segments.index.values) # Fetches all valid segment names
segs_passed = iargs.get('segment', None)
if segs_passed is not None:
if not isinstance(segs_passed, list) and not isinstance(
segs_passed, str):
error_message = "Segments can only be passed as a comma-separated"\
" string or a list of strings."
raise TypeError(error_message)
elif isinstance(segs_passed, str):
segs_passed = segs_passed.replace(' ', '').split(',')
if not set(segs_passed) <= set(segs):
error_message = "Unexpected Segments were passed. The series {series} "\
"contains the following Segments {segs}"
raise ValueError(
error_message.format(series=iargs['series'], segs=segs))
iargs['segment'] = segs_passed
# If Time has been passed as a PrimeKey, convert the Time object into TAI time scale,
# and then, convert it to datetime object.
ds = self._make_recordset(**iargs)
# Convert the list of keywords into comma-separated string.
if isinstance(keywords, list):
key = str(keywords)[1:-1].replace(' ', '').replace("'", '')
else:
key = keywords
r = c.query(ds, key=key, rec_index=isMeta)
# If the method was called from search_metadata(), return a Pandas Dataframe,
# otherwise return astropy.table
if isMeta:
return r
if r is None or r.empty:
return astropy.table.Table()
else:
return astropy.table.Table.from_pandas(r)
def _make_recordset(self,
series,
start_time='',
end_time='',
wavelength='',
segment='',
primekey={},
**kwargs):
"""
Take the query arguments and build a record string.
All the primekeys are now stored in primekey dict, including Time and Wavelength
which were passed through pre-defined attributes. The following piece of code,
extracts the passed prime-keys and arranges it in the order as it appears in the
JSOC database.
`pkeys_isTime` is a Pandas DataFrame, whose index values are the Prime-key names
and the column stores a boolean value, identifying whether the prime-key is a
Time-type prime-key or not. Since, time-type prime-keys exist by different names,
we made it uniform in the above piece of code, by storing the time-type primekey
with a single name `TIME`.
Considering an example, if the primekeys that exist for a given series are
['HARPNUM', 'T_OBS', 'WAVELNTH'], we will consider three different cases of the
passed primekeys.
pkeys_isTime.index.values = ['HARPNUM', 'T_OBS', 'WAVELNTH']
Case 1
------
primekey = {'T_OBS' : , '2014.01.01_00:00:45_TAI',
'HARPNUM' : '4864',
'WAVELNTH': '605'}
If the primekey dict is as above, then pkstr should be as:
pkstr = '{4864}{2014.01.01_00:00:45_TAI}{605}'
Case 2
------
primekey = {'T_OBS' : , '2014.01.01_00:00:45_TAI',
'WAVELNTH': '605'}
If the primekey dict is as above, then pkstr should be as:
pkstr = '{}{2014.01.01_00:00:45_TAI}{605}'
Case 3
------
primekey = {'T_OBS' : , '2014.01.01_00:00:45_TAI'}
If the primekey dict is as above, then pkstr should be as:
pkstr = '{}{2014.01.01_00:00:45_TAI}'
The idea behind this should be clear. We build up the `pkstr` string
containing the values of the prime-keys passed in the same order as
it occurs in the list `pkeys_isTime.index.values`, i.e. how it is stored
in the online database. Any missing prime-keys should be compensated by
an empty {}, if it occurs before any passed prime-key. Any empty curly braces
that is present at last of the pkstr, can be skipped.
"""
# Extract and format segment
# Convert list of segments into a comma-separated string
if segment:
if isinstance(segment, list):
segment = str(segment)[1:-1].replace(' ', '').replace("'", '')
segment = '{{{segment}}}'.format(segment=segment)
# Extract and format sample
sample = kwargs.get('sample', '')
if sample:
sample = '@{}s'.format(sample)
# Populate primekeys dict with Time and Wavelength values
if start_time and end_time:
# Check whether any primekey listed in PKEY_LIST_TIME has been passed through
# PrimeKey() attribute. If yes, raise an error, since Time can only be passed
# either through PrimeKey() attribute or Time() attribute.
if not any(x in PKEY_LIST_TIME for x in primekey):
timestr = '{start}-{end}{sample}'.format(
start=start_time.tai.strftime("%Y.%m.%d_%H:%M:%S_TAI"),
end=end_time.tai.strftime("%Y.%m.%d_%H:%M:%S_TAI"),
sample=sample)
else:
error_message = "Time attribute has been passed both as a Time()"\
" and PrimeKey(). Please provide any one of them"\
" or separate them by OR operator."
raise ValueError(error_message)
else:
# This is executed when Time has not been passed through Time() attribute.
# `match` stores all the time-type prime-keys that has been passed through
# PrimeKey() attribute. The length of `match` won't ever be greater than 1,
# but it is a good idea to keep a check.
match = set(primekey.keys()) & PKEY_LIST_TIME
if len(match) > 1:
error_message = "Querying of series, having more than 1 Time-type "\
"prime-keys is not yet supported. Alternative is to "\
"use only one of the primekey to query for series data."
raise ValueError(error_message)
if match:
timestr = '{0}'.format(primekey.pop(list(match)[0], ''))
else:
timestr = ''
if wavelength:
if not primekey.get('WAVELNTH', ''):
if isinstance(wavelength, list):
wavelength = [
int(np.ceil(wave.to(u.AA).value))
for wave in wavelength
]
wavelength = str(wavelength)
else:
wavelength = '{0}'.format(
int(np.ceil(wavelength.to(u.AA).value)))
else:
# This is executed when wavelength has been passed both through PrimeKey()
# and Wavelength().
error_message = "Wavelength attribute has been passed both as a Wavelength()"\
" and PrimeKey(). Please provide any one of them"\
" or separate them by OR operator."
raise ValueError(error_message)
else:
# This is executed when wavelength has been passed through PrimeKey().
wavelength = '{0}'.format(primekey.pop('WAVELNTH', ''))
# Populate primekey dict with formatted Time and Wavlength.
if timestr:
primekey['TIME'] = timestr
if wavelength:
primekey['WAVELNTH'] = wavelength
# Extract and format primekeys
pkstr = ''
c = drms.Client()
si = c.info(series)
pkeys_isTime = si.keywords.loc[si.primekeys].is_time
for pkey in pkeys_isTime.index.values:
# The loop is iterating over the list of prime-keys existing for the given series.
if len(primekey) > 0:
if pkeys_isTime[pkey]:
pkstr += '[{0}]'.format(primekey.pop('TIME', ''))
else:
pkstr += '[{0}]'.format(primekey.pop(pkey, ''))
else:
break
# break because we can skip adding {} at the end of pkstr, if the primekey
# dict is empty.
if not pkstr:
# pkstr cannot be totally empty
error_message = "Atleast one PrimeKey must be passed."
raise ValueError(error_message)
dataset = '{series}{primekey}{segment}'.format(series=series,
primekey=pkstr,
segment=segment)
return dataset
def get_request(self,
requests,
path=None,
overwrite=False,
progress=True,
downloader=None,
wait=True):
"""
Query JSOC to see if the request(s) is ready for download.
If the request is ready for download, it will then download it.
Parameters
----------
requests : `~drms.ExportRequest`, `str`, `list`
`~drms.ExportRequest` objects or `str` request IDs or lists
returned by `~sunpy.net.jsoc.jsoc.JSOCClient.request_data`.
path : `str`
Path to save data to, defaults to SunPy download dir.
progress : `bool`, optional
If `True` show a progress bar showing how many of the total files
have been downloaded. If `False`, no progress bar will be shown.
overwrite : `bool` or `str`, optional
Determine how to handle downloading if a file already exists with the
same name. If `False` the file download will be skipped and the path
returned to the existing file, if `True` the file will be downloaded
and the existing file will be overwritten, if `'unique'` the filename
will be modified to be unique.
downloader : `parfive.Downloader`, optional
The download manager to use.
wait : `bool`, optional
If `False` ``downloader.download()`` will not be called. Only has
any effect if `downloader` is not `None`.
Returns
-------
res: `~sunpy.net.download.Results`
A `~sunpy.net.download.Results` instance or `None` if no URLs to download
"""
c = drms.Client()
# Convert Responses to a list if not already
if isinstance(requests, str) or not isiterable(requests):
requests = [requests]
# Ensure all the requests are drms ExportRequest objects
for i, request in enumerate(requests):
if isinstance(request, str):
r = c.export_from_id(request)
requests[i] = r
# We only download if all are finished
if not all([r.has_succeeded() for r in requests]):
raise NotExportedError("Can not download as not all the requests "
"have been exported for download yet.")
# Ensure path has a {file} in it
if path is None:
default_dir = config.get("downloads", "download_dir")
path = os.path.join(default_dir, '{file}')
elif isinstance(path, str) and '{file}' not in path:
path = os.path.join(path, '{file}')
paths = []
for request in requests:
for filename in request.data['filename']:
# Ensure we don't duplicate the file extension
ext = os.path.splitext(filename)[1]
if path.endswith(ext):
fname = path.strip(ext)
else:
fname = path
fname = fname.format(file=filename)
fname = os.path.expanduser(fname)
paths.append(fname)
dl_set = True
if not downloader:
dl_set = False
downloader = Downloader(progress=progress, overwrite=overwrite)
urls = []
for request in requests:
if request.status == 0:
for index, data in request.data.iterrows():
url_dir = request.request_url + '/'
urls.append(urllib.parse.urljoin(url_dir,
data['filename']))
if urls:
if progress:
print_message = "{0} URLs found for download. Full request totalling {1}MB"
print(print_message.format(len(urls), request._d['size']))
for aurl, fname in zip(urls, paths):
downloader.enqueue_file(aurl, filename=fname)
if dl_set and not wait:
return Results()
results = downloader.download()
return results