def list_experiments(instrument_code, dt_fr, dt_to, madrigal_url=default_madrigal_url, database=None): if database is None: database = madrigalweb.MadrigalData(madrigal_url) exp_list = database.getExperiments(instrument_code, dt_fr.year, dt_fr.month, dt_fr.day, dt_fr.hour, dt_fr.minute, dt_fr.second, dt_to.year, dt_to.month, dt_to.day, dt_to.hour, dt_to.minute, dt_to.second, local=0) if not list(exp_list): raise ValueError( 'Cannot find the experiments for the database! Check the input values!' ) if exp_list[0].id == -1: madrigal_url = exp_list[0].madrigalUrl mylog.simpleinfo.info("Madrigal sources has been relocated to %s", madrigal_url) database = madrigalweb.MadrigalData(madrigal_url) exp_list = database.getExperiments(instrument_code, dt_fr.year, dt_fr.month, dt_fr.day, dt_fr.hour, dt_fr.minute, dt_fr.second, dt_to.year, dt_to.month, dt_to.day, dt_to.hour, dt_to.minute, dt_to.second, local=0) return exp_list, madrigal_url, database
def __init__(self, download_dir, user_name, user_email, user_affil): super().__init__(download_dir) if None in [user_name, user_email, user_affil]: raise InvalidConfiguration("To download from Madrigal, user name, email, and affiliation must be specified") self.user_name = user_name self.user_email = user_email self.user_affil = user_affil logger.info("connecting to server") self.server = madrigalWeb.MadrigalData("http://cedar.openmadrigal.org")
def fndwnld(request, experimentid): #linktofile = get_object_or_404(Question, pk = experimentid) madrigalUrl = 'http://isr.sri.com/madrigal' testData = madrigalWeb.MadrigalData(madrigalUrl) fileList = testData.getExperimentFiles(experimentid) linktofile = get_object_or_404( Question, madid=experimentid) #raise an error if this object does not exist return render(request, 'polls/fndwnld.html')
def index(request): # if this is a POST request we need to process the form data if request.method == 'POST': # create a form instance and populate it with data from the request: form = InputForm(request.POST) # check whether it's valid: if form.is_valid(): location = form.cleaned_data['location'] fromdate = form.cleaned_data['fromdate'] todate = form.cleaned_data['todate'] if (location == '1'): madrigalUrl = 'http://isr.sri.com/madrigal' else: madrigalUrl = 'http://madrigal.haystack.mit.edu/madrigal' user_fullname = 'Ashaki Gumbs' user_email = '*****@*****.**' user_affiliation = 'Boston University' testData = madrigalWeb.MadrigalData(madrigalUrl) expList = testData.getExperiments(61, fromdate.year, fromdate.month, fromdate.day, 0, 0, 0, todate.year, todate.month, todate.day, 0, 0, 0, local=1) # newcgiurl = madrigalWeb.MadrigalData(madrigalUrl) # expList = newcgiurl.getExperiments(61, fromdate.year,fromdate.month,fromdate.day,0,0,0, # todate.year,todate.month,todate.day,0,0,0, local=1) Question.objects.all().delete() for i in range(len(expList)): q = Question(pub_date=timezone.now(), madrigalUrl=expList[i].madrigalUrl, name=expList[i].name, realUrl=expList[i].realUrl, url=expList[i].url, madid=expList[i].id) q.save() # process the data in form.cleaned_data as required # ... # redirect to a new URL: return HttpResponseRedirect('/polls/listexp/') # if a GET (or any other method) we'll create a blank form else: form = InputForm() return render(request, 'polls/inputinfo.html', {'form': form})
def list_all_instruments(madrigal_url=default_madrigal_url, database=None): # list all the instruments from the madrigal sources # get sources info if database is None: database = madrigalweb.MadrigalData(madrigal_url) # List all instruments inst_list = database.getAllInstruments() mylog.simpleinfo.info( "List all the instruments from the Madrigal sources:\n") for inst in inst_list: mylog.simpleinfo.info("%s: %s", str(inst.code), inst.name)
def list_instruments(self): # list all the instruments from the madrigal database # get database info if hasattr(self, "database"): database = self.database else: database = madrigalweb.MadrigalData(self.madrigalURL) # List all instruments inst_list = database.getAllInstruments() mylog.simpleinfo.info( "List all the instruments from the Madrigal database:\n") for inst in inst_list: mylog.simpleinfo.info("%s: %s", str(inst.code), inst.name)
def check_madrigal_url(self): dt_st = self.dt_fr dt_ed = self.dt_to self.database = madrigalweb.MadrigalData(self.madrigalURL) expList = self.database.getExperiments(self.instrument_code, dt_st.year, dt_st.month, dt_st.day, dt_st.hour, dt_st.minute, dt_st.second, dt_ed.year, dt_ed.month, dt_ed.day, dt_ed.hour, dt_ed.minute, dt_ed.second, local=0) if expList[0].id == -1: self.madrigalURL = expList[0].madrigalUrl mylog.simpleinfo.info("Madrigal database has been relocated to %s", self.madrigalURL) self.database = madrigalweb.MadrigalData(self.madrigalURL) self.expList = self.database.getExperiments(self.instrument_code, dt_st.year, dt_st.month, dt_st.day, dt_st.hour, dt_st.minute, dt_st.second, dt_ed.year, dt_ed.month, dt_ed.day, dt_ed.hour, dt_ed.minute, dt_ed.second, local=0)
def getdata(date): url = "http://cedar.openmadrigal.org/" username = "******" email = "*****@*****.**" affiliation = "Virginia Tech" instrument = 8000 #TEC instruments startyear = str(date.year) startmonth = str(date.month) startday = str(date.day) starthour = '0' startmin = '0' startsec = '0' MadrigalData = madrigalWeb.MadrigalData(url) experimentarray = MadrigalData.getExperiments( instrument, startyear, startmonth, startday, starthour, startmin, startsec, startyear, startmonth, startday, starthour, startmin, startsec, 1) #save hdf5 format straight from madrigalweb (no parameters) print("Downloading: ", startyear, startmonth, startday) filearray = MadrigalData.getExperimentFiles( experimentarray[1].id, getNonDefault=False ) #need to use index 1 since madrigal returns the previos day at index 0 tempfilename = 'tempfile.hdf5' MadrigalData.downloadFile(filearray[0].name, tempfilename, username, email, affiliation, format='hdf5') print("raw hdf5 file downloaded successfully") #save only 3d TEC matrix binned by lat, lon, and time with h5py.File(tempfilename, 'r') as f: data = f['Data']['Array Layout']['2D Parameters'] tec = data['tec'] dtec = data['dtec'] print("saving to compressed .npz file format") np.savez_compressed('data.npz', tec=tec, dtec=dtec) os.remove(tempfilename)
Arr_offset_to_download = 19 # Browse first to find this if RISRN and RISRC: raise Exception( "You cannot choose both RISR_HDF5-N and C because their data lives in different places" ) if not RISRN and not RISRC: raise Exception("You have to to pick one of RISR_HDF5-N or C") # NOTE: Different instruments are at different URLs if RISRN: madrigalUrl = "http://isr.sri.com/madrigal/" else: madrigalUrl = "https://madrigal.phys.ucalgary.ca/" testData = madrigalWeb.MadrigalData(madrigalUrl) # List all of the instruments instList = testData.getAllInstruments() inst_code = 0 radar_nmonic = "" for inst in instList: # 91: Resolute Bay North IS Radar (http://isr.sri.com/madrigal/) if inst.code == 91 and RISRN: inst_code = 91 print(inst) radar_nmonic = inst.mnemonic # 92: Resolute Bay Canada IS Radar (https://madrigal.phys.ucalgary.ca/) if inst.code == 92 and RISRC: inst_code = 92 print(inst)
'--institution', help='your institution', default='nobody') p.add_argument('--download', help='type of file to download', default='hdf5') p.add_argument('--madurl', help='madrigal server url', default='http://isr.sri.com/madrigal/') p.add_argument('--outdir', help='path to save madrigal data to', default='files') p = p.parse_args() tic = time() files = getFiles(p.filtername, limit) print('filtered file list in {:.4f} sec.'.format(time() - tic)) try: mkdir(p.outdir) except OSError: #directory already exists pass #%% print('connecting to {}'.format(p.madurl)) site = MW.MadrigalData(p.madurl) for f in files: print('processing {}'.format(f)) site.downloadFile(f, join(p.outdir, basename(f) + '.h5'), p.username, p.email, p.institution, p.download)
def B2entries(insts, byear, bmonth, bday, bhour, bmin, bsec, eyear, emonth, eday, ehour, emin, esec): from B2SHAREClient import EISCATmetadata, B2SHAREClient import madrigalWeb.madrigalWeb as mw import madPlots import datetime from os import path, scandir, unlink from configparser import SafeConfigParser import logging ## Hardcoded numbering, Get from madrigal metadata instead? instMap = { 71: "kir", 72: "uhf", 73: "sod", 74: "vhf", 75: "kir", 76: "sod", 95: "lyr" } ## Read config config = SafeConfigParser(inline_comment_prefixes={'#'}) config.read('/usr/local/etc/eudat.conf') tmpdir = config.get('Main', 'tempDir') madurl = config.get('Madrigal', 'URL') ## Open Madrigal connection logging.info("Opening connection to %s" % madurl) try: madData = mw.MadrigalData(madurl) except: raise IOError("Could not open Madrigal connection") ## Set up one B2SHARE client instance b2url = config.get('B2', 'b2share_url') logging.info("Connecting to B2SHARE instance at %s" % b2url) try: client = B2SHAREClient.B2SHAREClient(community_id=config.get( 'B2', 'community'), url=b2url, token=config.get('B2', 'token')) except: raise IOError("Could not open B2Share connection") ## List experiments try: logging.debug("Listing Madrigal experiments") madExps = madData.getExperiments(insts, byear, bmonth, bday, bhour, bmin, bsec, eyear, emonth, eday, ehour, emin, esec, local=1) except: raise IOError("Could not list Madrigal experiments") ## Main loop over Madrigal experiments for thisExp in madExps: ## get the list of NCAR files logging.debug("Working on %s" % str(thisExp.id)) expFiles = madData.getExperimentFiles(thisExp.id) ## Check for realtime experiments if len(expFiles) == 1: if (expFiles[0].category == 4) or expFiles[0].name.endswith('.asc'): # only a realtime file, skip to next experiment logging.info("Skipping experiment %s: only one realtime file" % str(thisExp.id)) continue logging.debug("Continuing with experiment %s" % str(thisExp.id)) ## Build argument list for EISCATmetadata args = [] args.append(thisExp.id) # Experiment ID args.append(thisExp.name) # Experiment name antenna = instMap[thisExp.instcode] args.append(antenna) # Instrument args.append(None) # Resources startTime = datetime.datetime(thisExp.startyear, thisExp.startmonth, thisExp.startday, thisExp.starthour, thisExp.startmin, thisExp.startsec) args.append(startTime) # Start time endTime = datetime.datetime(thisExp.endyear, thisExp.endmonth, thisExp.endday, thisExp.endhour, thisExp.endmin, thisExp.endsec) args.append(endTime) args.append(None) #Embargo time args.append(None) # Info directory ## Create metadata logging.debug("Creating JSON object") metadata_json = EISCATmetadata.MetaDataJSON( args, 3, thisExp.realUrl, config.get('B2', 'community'), config.get('B2', 'community_specific')) ## Create B2SHARE draft logging.info("Creating B2SHARE draft record for experiment %s" % str(thisExp.id)) try: draft_json = client.create_draft(metadata_json) except: raise IOError("B2share draft creation error") ## Upload the files logging.debug("Uploading experiment files") for expFile in expFiles: if expFile.category == 1: outFile = path.join( tmpdir, path.splitext(path.basename(expFile.name))[0] + '.hdf5') logging.info("Getting Madrigal file as %s" % outFile) try: madData.downloadFile(expFile.name, outFile, 'B2Share client', '*****@*****.**', 'EISCAT Scientific Association', format='hdf5') except: raise IOError( "Could not download experiment file from Madrigal") logging.info("Uploading the file to B2share file bucket") try: client.put_draft_file(draft_json, [outFile]) except: raise IOError("Could not upload file") # Add the parameters logging.debug("Getting parameters in file %s" % expFile.name) try: expPars = madData.getExperimentFileParameters(expFile.name) except: raise IOError("Could not get parameters from Madrigal") logging.debug("Creating JSON Patch with parameters") param_json_patch = EISCATmetadata.ParamJSONpatch( expPars, config.get('B2', 'community_specific')) # Patch B2Share entry logging.info( "Inserting JSON patch with parameters in B2Share entry") try: client.update_draft(draft_json, param_json_patch) except: raise IOError("Could not upload JSON patch") ## Add plots # Build true experiment URL logging.debug("Searching for figures to upload") expurl = thisExp.url if (expurl.find('cgi-bin/madtoc') > 0): expurl = expurl.replace('cgi-bin/madtoc/', '') try: plotFiles = madPlots.get_plots(expurl, tmpdir) except: raise IOError("Could not search for madrigal plots") if (len(plotFiles) > 0): logging.info("Uploading plots to B2Share: %s" % str(plotFiles)) try: client.put_draft_file(draft_json, plotFiles) except: raise IOError("Could not upload the files") ## Clean up temp dir logging.debug("Cleaning up files in %s" % tmpdir) for tmpfile in scandir(tmpdir): unlink(tmpfile.path)
def get_remote_filenames(inst_code=None, kindat=None, user=None, password=None, web_data=None, url="http://cedar.openmadrigal.org", start=dt.datetime(1900,1,1), stop=dt.datetime.now(), date_array=None): """Retrieve the remote filenames for a specified Madrigal instrument (and experiment) Parameters ---------- inst_code : string (None) Madrigal instrument code(s), cast as a string. If multiple are used, separate them with commas. kindat : string (None) Madrigal experiment code(s), cast as a string. If multiple are used, separate them with commas. If not supplied, all will be returned. data_path : string (None) Path to directory to download data to. user : string (None) User string input used for download. Provided by user and passed via pysat. If an account is required for dowloads this routine here must error if user not supplied. password : string (None) Password for data download. web_data : MadrigalData (None) Open connection to Madrigal database or None (will initiate using url) url : string ('http://cedar.openmadrigal.org') URL for Madrigal site start : dt.datetime Starting time for file list (defaults to 01-01-1900) stop : dt.datetime Ending time for the file list (defaults to time of run) date_array : dt.datetime (None) Array of datetimes to download data for. The sequence of dates need not be contiguous and will be used instead of start and stop if supplied. Returns ------- Void : (NoneType) Downloads data to disk. Notes ----- The user's names should be provided in field user. Ruby Payne-Scott should be entered as Ruby+Payne-Scott The password field should be the user's email address. These parameters are passed to Madrigal when downloading. The affiliation field is set to pysat to enable tracking of pysat downloads. """ if inst_code is None: raise ValueError("Must supply Madrigal instrument code") if kindat is None: kindat = [] else: kindat = [int(kk) for kk in kindat.split(",")] # currently passes things along if no user and password supplied # need to do this for testing # TODO, implement user and password values in test code # specific to each instrument if user is None: print('No user information supplied for download.') user = '******' if password is None: print('Please provide email address in password field.') password = '******' # If date_array supplied, overwrite start and stop if date_array is not None: if len(date_array) == 0: raise ValueError('unknown date_array supplied: {:}'.format( date_array)) start = date_array.min() stop = date_array.max() if start == stop: stop += dt.timedelta(days=1) # open connection to Madrigal if web_data is None: web_data = madrigalWeb.MadrigalData(url) # get list of experiments for instrument from 1900 till now exp_list = web_data.getExperiments(inst_code, start.year, start.month, start.day, start.hour, start.minute, start.second, stop.year, stop.month, stop.day, stop.hour, stop.minute, stop.second) # iterate over experiments to grab files for each one files = list() print("Found {:d} Madrigral experiments".format(len(exp_list))) for exp in exp_list: if good_exp(exp, date_array=date_array): file_list = web_data.getExperimentFiles(exp.id) if len(kindat) == 0: files.extend(file_list) else: for file_obj in file_list: if file_obj.kindat in kindat: files.append(file_obj) return files
def download(date_array, inst_code=None, kindat=None, data_path=None, user=None, password=None, url="http://cedar.openmadrigal.org", file_format='hdf5'): """Downloads data from Madrigal. Parameters ---------- date_array : array-like list of datetimes to download data for. The sequence of dates need not be contiguous. inst_code : string (None) Madrigal instrument code(s), cast as a string. If multiple are used, separate them with commas. kindat : string (None) Experiment instrument code(s), cast as a string. If multiple are used, separate them with commas. data_path : string (None) Path to directory to download data to. user : string (None) User string input used for download. Provided by user and passed via pysat. If an account is required for dowloads this routine here must error if user not supplied. password : string (None) Password for data download. url : string ('http://cedar.openmadrigal.org') URL for Madrigal site file_format : string ('hdf5') File format for Madrigal data. Load routines currently only accept 'hdf5', but any of the Madrigal options may be used here. Returns -------- Void : (NoneType) Downloads data to disk. Notes ----- The user's names should be provided in field user. Ruby Payne-Scott should be entered as Ruby+Payne-Scott The password field should be the user's email address. These parameters are passed to Madrigal when downloading. The affiliation field is set to pysat to enable tracking of pysat downloads. """ if inst_code is None: raise ValueError("Must supply Madrigal instrument code") if kindat is None: raise ValueError("Must supply Madrigal experiment code") # currently passes things along if no user and password supplied # need to do this for testing # TODO, implement user and password values in test code # specific to each instrument if user is None: logger.info('No user information supplied for download.') user = '******' if password is None: logger.info('Please provide email address in password field.') password = '******' # Initialize the connection to Madrigal web_data = madrigalWeb.MadrigalData(url) # Get the list of desired remote files start = date_array.min() stop = date_array.max() if start == stop: stop += dt.timedelta(days=1) files = get_remote_filenames(inst_code=inst_code, kindat=kindat, user=user, password=password, web_data=web_data, url=url, start=start, stop=stop) for mad_file in files: local_file = os.path.join(data_path, os.path.basename(mad_file.name)) if not os.path.isfile(local_file): web_data.downloadFile(mad_file.name, local_file, user, password, "pysat", format=file_format)