def list_experiments(instrument_code,
dt_fr,
dt_to,
madrigal_url=default_madrigal_url,
database=None):
if database is None:
database = madrigalweb.MadrigalData(madrigal_url)
exp_list = database.getExperiments(instrument_code,
dt_fr.year,
dt_fr.month,
dt_fr.day,
dt_fr.hour,
dt_fr.minute,
dt_fr.second,
dt_to.year,
dt_to.month,
dt_to.day,
dt_to.hour,
dt_to.minute,
dt_to.second,
local=0)
if not list(exp_list):
raise ValueError(
'Cannot find the experiments for the database! Check the input values!'
)
if exp_list[0].id == -1:
madrigal_url = exp_list[0].madrigalUrl
mylog.simpleinfo.info("Madrigal sources has been relocated to %s",
madrigal_url)
database = madrigalweb.MadrigalData(madrigal_url)
exp_list = database.getExperiments(instrument_code,
dt_fr.year,
dt_fr.month,
dt_fr.day,
dt_fr.hour,
dt_fr.minute,
dt_fr.second,
dt_to.year,
dt_to.month,
dt_to.day,
dt_to.hour,
dt_to.minute,
dt_to.second,
local=0)
return exp_list, madrigal_url, database
def __init__(self, download_dir, user_name, user_email, user_affil):
super().__init__(download_dir)
if None in [user_name, user_email, user_affil]:
raise InvalidConfiguration("To download from Madrigal, user name, email, and affiliation must be specified")
self.user_name = user_name
self.user_email = user_email
self.user_affil = user_affil
logger.info("connecting to server")
self.server = madrigalWeb.MadrigalData("http://cedar.openmadrigal.org")
def fndwnld(request, experimentid):
#linktofile = get_object_or_404(Question, pk = experimentid)
madrigalUrl = 'http://isr.sri.com/madrigal'
testData = madrigalWeb.MadrigalData(madrigalUrl)
fileList = testData.getExperimentFiles(experimentid)
linktofile = get_object_or_404(
Question,
madid=experimentid) #raise an error if this object does not exist
return render(request, 'polls/fndwnld.html')
def index(request):
# if this is a POST request we need to process the form data
if request.method == 'POST':
# create a form instance and populate it with data from the request:
form = InputForm(request.POST)
# check whether it's valid:
if form.is_valid():
location = form.cleaned_data['location']
fromdate = form.cleaned_data['fromdate']
todate = form.cleaned_data['todate']
if (location == '1'):
madrigalUrl = 'http://isr.sri.com/madrigal'
else:
madrigalUrl = 'http://madrigal.haystack.mit.edu/madrigal'
user_fullname = 'Ashaki Gumbs'
user_email = '*****@*****.**'
user_affiliation = 'Boston University'
testData = madrigalWeb.MadrigalData(madrigalUrl)
expList = testData.getExperiments(61,
fromdate.year,
fromdate.month,
fromdate.day,
0,
0,
0,
todate.year,
todate.month,
todate.day,
0,
0,
0,
local=1)
# newcgiurl = madrigalWeb.MadrigalData(madrigalUrl)
# expList = newcgiurl.getExperiments(61, fromdate.year,fromdate.month,fromdate.day,0,0,0,
# todate.year,todate.month,todate.day,0,0,0, local=1)
Question.objects.all().delete()
for i in range(len(expList)):
q = Question(pub_date=timezone.now(),
madrigalUrl=expList[i].madrigalUrl,
name=expList[i].name,
realUrl=expList[i].realUrl,
url=expList[i].url,
madid=expList[i].id)
q.save()
# process the data in form.cleaned_data as required
# ...
# redirect to a new URL:
return HttpResponseRedirect('/polls/listexp/')
# if a GET (or any other method) we'll create a blank form
else:
form = InputForm()
return render(request, 'polls/inputinfo.html', {'form': form})
def list_all_instruments(madrigal_url=default_madrigal_url, database=None):
# list all the instruments from the madrigal sources
# get sources info
if database is None:
database = madrigalweb.MadrigalData(madrigal_url)
# List all instruments
inst_list = database.getAllInstruments()
mylog.simpleinfo.info(
"List all the instruments from the Madrigal sources:\n")
for inst in inst_list:
mylog.simpleinfo.info("%s: %s", str(inst.code), inst.name)
def list_instruments(self):
# list all the instruments from the madrigal database
# get database info
if hasattr(self, "database"):
database = self.database
else:
database = madrigalweb.MadrigalData(self.madrigalURL)
# List all instruments
inst_list = database.getAllInstruments()
mylog.simpleinfo.info(
"List all the instruments from the Madrigal database:\n")
for inst in inst_list:
mylog.simpleinfo.info("%s: %s", str(inst.code), inst.name)
def check_madrigal_url(self):
dt_st = self.dt_fr
dt_ed = self.dt_to
self.database = madrigalweb.MadrigalData(self.madrigalURL)
expList = self.database.getExperiments(self.instrument_code,
dt_st.year,
dt_st.month,
dt_st.day,
dt_st.hour,
dt_st.minute,
dt_st.second,
dt_ed.year,
dt_ed.month,
dt_ed.day,
dt_ed.hour,
dt_ed.minute,
dt_ed.second,
local=0)
if expList[0].id == -1:
self.madrigalURL = expList[0].madrigalUrl
mylog.simpleinfo.info("Madrigal database has been relocated to %s",
self.madrigalURL)
self.database = madrigalweb.MadrigalData(self.madrigalURL)
self.expList = self.database.getExperiments(self.instrument_code,
dt_st.year,
dt_st.month,
dt_st.day,
dt_st.hour,
dt_st.minute,
dt_st.second,
dt_ed.year,
dt_ed.month,
dt_ed.day,
dt_ed.hour,
dt_ed.minute,
dt_ed.second,
local=0)
def getdata(date):
url = "http://cedar.openmadrigal.org/"
username = "******"
email = "*****@*****.**"
affiliation = "Virginia Tech"
instrument = 8000 #TEC instruments
startyear = str(date.year)
startmonth = str(date.month)
startday = str(date.day)
starthour = '0'
startmin = '0'
startsec = '0'
MadrigalData = madrigalWeb.MadrigalData(url)
experimentarray = MadrigalData.getExperiments(
instrument, startyear, startmonth, startday, starthour, startmin,
startsec, startyear, startmonth, startday, starthour, startmin,
startsec, 1)
#save hdf5 format straight from madrigalweb (no parameters)
print("Downloading: ", startyear, startmonth, startday)
filearray = MadrigalData.getExperimentFiles(
experimentarray[1].id, getNonDefault=False
) #need to use index 1 since madrigal returns the previos day at index 0
tempfilename = 'tempfile.hdf5'
MadrigalData.downloadFile(filearray[0].name,
tempfilename,
username,
email,
affiliation,
format='hdf5')
print("raw hdf5 file downloaded successfully")
#save only 3d TEC matrix binned by lat, lon, and time
with h5py.File(tempfilename, 'r') as f:
data = f['Data']['Array Layout']['2D Parameters']
tec = data['tec']
dtec = data['dtec']
print("saving to compressed .npz file format")
np.savez_compressed('data.npz', tec=tec, dtec=dtec)
os.remove(tempfilename)
Arr_offset_to_download = 19 # Browse first to find this
if RISRN and RISRC:
raise Exception(
"You cannot choose both RISR_HDF5-N and C because their data lives in different places"
)
if not RISRN and not RISRC:
raise Exception("You have to to pick one of RISR_HDF5-N or C")
# NOTE: Different instruments are at different URLs
if RISRN:
madrigalUrl = "http://isr.sri.com/madrigal/"
else:
madrigalUrl = "https://madrigal.phys.ucalgary.ca/"
testData = madrigalWeb.MadrigalData(madrigalUrl)
# List all of the instruments
instList = testData.getAllInstruments()
inst_code = 0
radar_nmonic = ""
for inst in instList:
# 91: Resolute Bay North IS Radar (http://isr.sri.com/madrigal/)
if inst.code == 91 and RISRN:
inst_code = 91
print(inst)
radar_nmonic = inst.mnemonic
# 92: Resolute Bay Canada IS Radar (https://madrigal.phys.ucalgary.ca/)
if inst.code == 92 and RISRC:
inst_code = 92
print(inst)
'--institution',
help='your institution',
default='nobody')
p.add_argument('--download',
help='type of file to download',
default='hdf5')
p.add_argument('--madurl',
help='madrigal server url',
default='http://isr.sri.com/madrigal/')
p.add_argument('--outdir',
help='path to save madrigal data to',
default='files')
p = p.parse_args()
tic = time()
files = getFiles(p.filtername, limit)
print('filtered file list in {:.4f} sec.'.format(time() - tic))
try:
mkdir(p.outdir)
except OSError: #directory already exists
pass
#%%
print('connecting to {}'.format(p.madurl))
site = MW.MadrigalData(p.madurl)
for f in files:
print('processing {}'.format(f))
site.downloadFile(f, join(p.outdir,
basename(f) + '.h5'), p.username, p.email,
p.institution, p.download)
def B2entries(insts, byear, bmonth, bday, bhour, bmin, bsec, eyear, emonth,
eday, ehour, emin, esec):
from B2SHAREClient import EISCATmetadata, B2SHAREClient
import madrigalWeb.madrigalWeb as mw
import madPlots
import datetime
from os import path, scandir, unlink
from configparser import SafeConfigParser
import logging
## Hardcoded numbering, Get from madrigal metadata instead?
instMap = {
71: "kir",
72: "uhf",
73: "sod",
74: "vhf",
75: "kir",
76: "sod",
95: "lyr"
}
## Read config
config = SafeConfigParser(inline_comment_prefixes={'#'})
config.read('/usr/local/etc/eudat.conf')
tmpdir = config.get('Main', 'tempDir')
madurl = config.get('Madrigal', 'URL')
## Open Madrigal connection
logging.info("Opening connection to %s" % madurl)
try:
madData = mw.MadrigalData(madurl)
except:
raise IOError("Could not open Madrigal connection")
## Set up one B2SHARE client instance
b2url = config.get('B2', 'b2share_url')
logging.info("Connecting to B2SHARE instance at %s" % b2url)
try:
client = B2SHAREClient.B2SHAREClient(community_id=config.get(
'B2', 'community'),
url=b2url,
token=config.get('B2', 'token'))
except:
raise IOError("Could not open B2Share connection")
## List experiments
try:
logging.debug("Listing Madrigal experiments")
madExps = madData.getExperiments(insts,
byear,
bmonth,
bday,
bhour,
bmin,
bsec,
eyear,
emonth,
eday,
ehour,
emin,
esec,
local=1)
except:
raise IOError("Could not list Madrigal experiments")
## Main loop over Madrigal experiments
for thisExp in madExps:
## get the list of NCAR files
logging.debug("Working on %s" % str(thisExp.id))
expFiles = madData.getExperimentFiles(thisExp.id)
## Check for realtime experiments
if len(expFiles) == 1:
if (expFiles[0].category
== 4) or expFiles[0].name.endswith('.asc'):
# only a realtime file, skip to next experiment
logging.info("Skipping experiment %s: only one realtime file" %
str(thisExp.id))
continue
logging.debug("Continuing with experiment %s" % str(thisExp.id))
## Build argument list for EISCATmetadata
args = []
args.append(thisExp.id) # Experiment ID
args.append(thisExp.name) # Experiment name
antenna = instMap[thisExp.instcode]
args.append(antenna) # Instrument
args.append(None) # Resources
startTime = datetime.datetime(thisExp.startyear, thisExp.startmonth,
thisExp.startday, thisExp.starthour,
thisExp.startmin, thisExp.startsec)
args.append(startTime) # Start time
endTime = datetime.datetime(thisExp.endyear, thisExp.endmonth,
thisExp.endday, thisExp.endhour,
thisExp.endmin, thisExp.endsec)
args.append(endTime)
args.append(None) #Embargo time
args.append(None) # Info directory
## Create metadata
logging.debug("Creating JSON object")
metadata_json = EISCATmetadata.MetaDataJSON(
args, 3, thisExp.realUrl, config.get('B2', 'community'),
config.get('B2', 'community_specific'))
## Create B2SHARE draft
logging.info("Creating B2SHARE draft record for experiment %s" %
str(thisExp.id))
try:
draft_json = client.create_draft(metadata_json)
except:
raise IOError("B2share draft creation error")
## Upload the files
logging.debug("Uploading experiment files")
for expFile in expFiles:
if expFile.category == 1:
outFile = path.join(
tmpdir,
path.splitext(path.basename(expFile.name))[0] + '.hdf5')
logging.info("Getting Madrigal file as %s" % outFile)
try:
madData.downloadFile(expFile.name,
outFile,
'B2Share client',
'*****@*****.**',
'EISCAT Scientific Association',
format='hdf5')
except:
raise IOError(
"Could not download experiment file from Madrigal")
logging.info("Uploading the file to B2share file bucket")
try:
client.put_draft_file(draft_json, [outFile])
except:
raise IOError("Could not upload file")
# Add the parameters
logging.debug("Getting parameters in file %s" % expFile.name)
try:
expPars = madData.getExperimentFileParameters(expFile.name)
except:
raise IOError("Could not get parameters from Madrigal")
logging.debug("Creating JSON Patch with parameters")
param_json_patch = EISCATmetadata.ParamJSONpatch(
expPars, config.get('B2', 'community_specific'))
# Patch B2Share entry
logging.info(
"Inserting JSON patch with parameters in B2Share entry")
try:
client.update_draft(draft_json, param_json_patch)
except:
raise IOError("Could not upload JSON patch")
## Add plots
# Build true experiment URL
logging.debug("Searching for figures to upload")
expurl = thisExp.url
if (expurl.find('cgi-bin/madtoc') > 0):
expurl = expurl.replace('cgi-bin/madtoc/', '')
try:
plotFiles = madPlots.get_plots(expurl, tmpdir)
except:
raise IOError("Could not search for madrigal plots")
if (len(plotFiles) > 0):
logging.info("Uploading plots to B2Share: %s" % str(plotFiles))
try:
client.put_draft_file(draft_json, plotFiles)
except:
raise IOError("Could not upload the files")
## Clean up temp dir
logging.debug("Cleaning up files in %s" % tmpdir)
for tmpfile in scandir(tmpdir):
unlink(tmpfile.path)
def get_remote_filenames(inst_code=None, kindat=None, user=None,
password=None, web_data=None,
url="http://cedar.openmadrigal.org",
start=dt.datetime(1900,1,1), stop=dt.datetime.now(),
date_array=None):
"""Retrieve the remote filenames for a specified Madrigal instrument
(and experiment)
Parameters
----------
inst_code : string (None)
Madrigal instrument code(s), cast as a string. If multiple are used,
separate them with commas.
kindat : string (None)
Madrigal experiment code(s), cast as a string. If multiple are used,
separate them with commas. If not supplied, all will be returned.
data_path : string (None)
Path to directory to download data to.
user : string (None)
User string input used for download. Provided by user and passed via
pysat. If an account
is required for dowloads this routine here must error if user not
supplied.
password : string (None)
Password for data download.
web_data : MadrigalData (None)
Open connection to Madrigal database or None (will initiate using url)
url : string ('http://cedar.openmadrigal.org')
URL for Madrigal site
start : dt.datetime
Starting time for file list (defaults to 01-01-1900)
stop : dt.datetime
Ending time for the file list (defaults to time of run)
date_array : dt.datetime (None)
Array of datetimes to download data for. The sequence of dates need not
be contiguous and will be used instead of start and stop if supplied.
Returns
-------
Void : (NoneType)
Downloads data to disk.
Notes
-----
The user's names should be provided in field user. Ruby Payne-Scott should
be entered as Ruby+Payne-Scott
The password field should be the user's email address. These parameters
are passed to Madrigal when downloading.
The affiliation field is set to pysat to enable tracking of pysat
downloads.
"""
if inst_code is None:
raise ValueError("Must supply Madrigal instrument code")
if kindat is None:
kindat = []
else:
kindat = [int(kk) for kk in kindat.split(",")]
# currently passes things along if no user and password supplied
# need to do this for testing
# TODO, implement user and password values in test code
# specific to each instrument
if user is None:
print('No user information supplied for download.')
user = '******'
if password is None:
print('Please provide email address in password field.')
password = '******'
# If date_array supplied, overwrite start and stop
if date_array is not None:
if len(date_array) == 0:
raise ValueError('unknown date_array supplied: {:}'.format(
date_array))
start = date_array.min()
stop = date_array.max()
if start == stop:
stop += dt.timedelta(days=1)
# open connection to Madrigal
if web_data is None:
web_data = madrigalWeb.MadrigalData(url)
# get list of experiments for instrument from 1900 till now
exp_list = web_data.getExperiments(inst_code, start.year, start.month,
start.day, start.hour, start.minute,
start.second, stop.year, stop.month,
stop.day, stop.hour, stop.minute,
stop.second)
# iterate over experiments to grab files for each one
files = list()
print("Found {:d} Madrigral experiments".format(len(exp_list)))
for exp in exp_list:
if good_exp(exp, date_array=date_array):
file_list = web_data.getExperimentFiles(exp.id)
if len(kindat) == 0:
files.extend(file_list)
else:
for file_obj in file_list:
if file_obj.kindat in kindat:
files.append(file_obj)
return files
def download(date_array, inst_code=None, kindat=None, data_path=None,
user=None, password=None, url="http://cedar.openmadrigal.org",
file_format='hdf5'):
"""Downloads data from Madrigal.
Parameters
----------
date_array : array-like
list of datetimes to download data for. The sequence of dates need not
be contiguous.
inst_code : string (None)
Madrigal instrument code(s), cast as a string. If multiple are used,
separate them with commas.
kindat : string (None)
Experiment instrument code(s), cast as a string. If multiple are used,
separate them with commas.
data_path : string (None)
Path to directory to download data to.
user : string (None)
User string input used for download. Provided by user and passed via
pysat. If an account
is required for dowloads this routine here must error if user not
supplied.
password : string (None)
Password for data download.
url : string ('http://cedar.openmadrigal.org')
URL for Madrigal site
file_format : string ('hdf5')
File format for Madrigal data. Load routines currently only accept
'hdf5', but any of the Madrigal options may be used here.
Returns
--------
Void : (NoneType)
Downloads data to disk.
Notes
-----
The user's names should be provided in field user. Ruby Payne-Scott should
be entered as Ruby+Payne-Scott
The password field should be the user's email address. These parameters
are passed to Madrigal when downloading.
The affiliation field is set to pysat to enable tracking of pysat
downloads.
"""
if inst_code is None:
raise ValueError("Must supply Madrigal instrument code")
if kindat is None:
raise ValueError("Must supply Madrigal experiment code")
# currently passes things along if no user and password supplied
# need to do this for testing
# TODO, implement user and password values in test code
# specific to each instrument
if user is None:
logger.info('No user information supplied for download.')
user = '******'
if password is None:
logger.info('Please provide email address in password field.')
password = '******'
# Initialize the connection to Madrigal
web_data = madrigalWeb.MadrigalData(url)
# Get the list of desired remote files
start = date_array.min()
stop = date_array.max()
if start == stop:
stop += dt.timedelta(days=1)
files = get_remote_filenames(inst_code=inst_code, kindat=kindat, user=user,
password=password, web_data=web_data, url=url,
start=start, stop=stop)
for mad_file in files:
local_file = os.path.join(data_path, os.path.basename(mad_file.name))
if not os.path.isfile(local_file):
web_data.downloadFile(mad_file.name, local_file, user, password,
"pysat", format=file_format)
