def ima_coverage(recompute=False): """Returns a dictionary, key=orbit number, value = size in MB for the IMA data from that orbit""" results = {} try: with open(mex.data_directory + "nilsson_ima_coverage.pck", "r") as f: results = pickle.load(f) except Exception as e: recompute = True print(e) if recompute: now = celsius.now() directory = mex.data_directory + "aspera/nilsson/Mars_mat_files4/" results = {} for o in mex.orbits: t = mex.orbits[o].start if t > celsius.now(): break t1 = mex.orbits[o].finish size = 0 while t < t1: dt = celsius.spiceet_to_datetime(t) fname = directory + "fion" + "%4d%02d%02d%02d00.mat" % (dt.year, dt.month, dt.day, dt.hour) if os.path.exists(fname): size += os.path.getsize(fname) t += 3600.0 results[o] = size / (1024 * 1024) # MB with open(mex.data_directory + "nilsson_ima_coverage.pck", "w") as f: pickle.dump(results, f) return results
def plot_aspera_els(start, finish=None, verbose=False, ax=None, colorbar=True, vmin=None, vmax=None, cmap=None, safe=True): """docstring for plot_aspera_els""" if cmap is None: cmap = plt.cm.Spectral_r if ax is None: ax = plt.gca() plt.sca(ax) if finish is None: finish = start + 86400. if vmin is None: vmin = 5. if vmax is None: vmax = 9. no_days = (finish - start) / 86400. if verbose: print('Plotting ASPERA/ELS between %s and %s...' % (celsius.utcstr(start, 'ISOC'), celsius.utcstr(finish, 'ISOC'))) directory = mex.data_directory + 'aspera/els/' all_files_to_read = [] for et in np.arange(start - 10., finish + 10., 86400.): dt = celsius.spiceet_to_datetime(et) f_name = directory + 'MEX_ELS_EFLUX_%4d%02d%02d_*.cef' % (dt.year, dt.month, dt.day) all_day_files = glob.glob(f_name) if not all_day_files: if verbose: print("No files matched %s" % f_name) else: all_files_to_read.extend(all_day_files) success = False all_extents = [] for f_name in all_files_to_read: try: # Find energy bins: with open(f_name, 'r') as f: line_no = 0 while line_no < 43: line_no += 1 line = f.readline() if 'DATA = ' in line: energy_bins = np.fromstring(line[7:], sep=',') energy_bins.sort() break else: raise IOError("No ENERGY_BINS info found in header") data = np.loadtxt(f_name, skiprows = 43, converters={1:lambda x: celsius.utcstr_to_spiceet(x[:-1])}) if data.shape[1] != (energy_bins.shape[0] + 2): raise ValueError("Size of ENERGY_BINS and DATA doesn't match") # Check timing: dt = np.diff(data[:,1]) spacing = np.median(dt) # checks = abs(dt - spacing) > (spacing/100.) # if np.any(checks): # # raise ValueError("Spacing is not constant: %d differ by more than 1%% of %f:" % (np.sum(checks), spacing)) # print "Spacing is not constant: %d differ by more than 1%% of %f (Maximum = %f):" % (np.sum(checks), spacing, max(abs(dt - spacing))) # # if safe and (max(abs(dt - spacing)) > 10.): # print '-- To big spacing - dumping' # continue # Interpolate to constant spacing: n_records = int((data[-1,1] - data[0,1]) / spacing) new_data = np.empty((n_records, data.shape[1])) + np.nan new_data[:,1] = np.linspace(data[0,1], data[-1,1], n_records) for i in range(3, data.shape[1]): new_data[:,i] = np.interp(new_data[:,1],data[:,1], data[:,i], left=np.nan, right=np.nan) data = new_data extent = (data[0,1], data[-1,1], energy_bins[0], energy_bins[-1]) if (extent[0] > finish) or (extent[1] < start): if verbose: print("This block not within plot range - dumping") continue all_extents.append(extent) if verbose: print('Plotting ASPERA ELS block, Time: %s - %s, Energy: %f - %f' % ( celsius.utcstr(extent[0],'ISOC'), celsius.utcstr(extent[1],'ISOC'), extent[2], extent[3])) print('Shape = ', data.shape) plt.imshow(np.log10(data[:,3:].T), interpolation="nearest", aspect='auto', extent=extent, vmin=vmin, vmax=vmax, cmap=cmap) success = True except IOError as e: if verbose: print('Error reading %f' % f_name) print('--', e) continue if success and colorbar: plt.xlim(start, finish) plt.ylim(max([e[2] for e in all_extents]), min([e[3] for e in all_extents])) celsius.ylabel('E / eV') plt.yscale('log') cmap.set_under('w') old_ax = plt.gca() plt.colorbar(cax=celsius.make_colorbar_cax(), cmap=cmap, ticks=[5,6,7,8,9]) plt.ylabel(r'log$_{10}$ D.E.F.') plt.sca(old_ax)
def read_ima(start, finish=None, dataset="FION", verbose=False, aux=False): """Read Nilsson's ima files into a list. dataset="fion": best for heavy-ions dataset="ion": best for protons, but not much in it. dataset="aux": ancillary info note: setting aux=True also reads the aux files, and appends each files contents into the principal blocks being read by extending each dictionary.""" if aux: if dataset == "aux": raise ValueError("""Don't append aux data to the aux data. The whole universe will explode""") if finish is None: finish = start + 1.0 directory = mex.data_directory + "aspera/nilsson/Mars_mat_files4/" remote_path = "[email protected]:/irf/data/mars/aspera3/Mars_mat_files4/" et = start - 3600.0 # one second before out = [] error_files = [] remote_failed_count = 0 while True: if et > (finish + 3600.0): break dt = celsius.spiceet_to_datetime(et) fname = dataset.lower() + "%4d%02d%02d%02d00.mat" % (dt.year, dt.month, dt.day, dt.hour) if not os.path.exists(directory + fname): remote_fname = remote_path + fname fd, temp_f_name = tempfile.mkstemp(suffix="tmp.mat") command = ("scp", remote_fname, temp_f_name) if verbose: print("Fetching %s" % remote_fname) try: # os.spawnvp(os.P_WAIT, command[0], command) subprocess.check_call(command) # reset to zero on success remote_failed_count = 0 except subprocess.CalledProcessError as e: remote_failed_count += 1 # raise IOError("Retrieval from aurora failed: %s" % str(e)) print("Retrieval of %s from aurora failed" % remote_fname) if remote_failed_count > MAX_FAILED_REMOTE_FILES: print("Maximum failed remote tries reached") break else: d = os.path.dirname(directory + fname) if d and not os.path.exists(d): if verbose: print("Creating %s" % d) os.makedirs(d) command = ("mv", temp_f_name, directory + fname) try: # os.spawnvp(os.P_WAIT, command[0], command) subprocess.check_call(command) except subprocess.CalledProcessError as e: print(e) raise IOError("Error moving file to %s" % (directory + fname)) try: out.append(loadmat(directory + fname)) # Squeeze arrays for o in out[-1]: if isinstance(out[-1][o], np.ndarray): out[-1][o] = np.squeeze(out[-1][o]) if verbose: print( "Read: %s: %s - %s" % ( fname, celsius.time_convert(out[-1]["tmptime"][0], "UTCSTR", "MATLABTIME"), celsius.time_convert(out[-1]["tmptime"][-1], "UTCSTR", "MATLABTIME"), ) ) except IOError as e: error_files.append(fname) if verbose: print("Missing: %s" % fname) et += 60.0 * 60.0 if verbose: print("Read %d files - %d errors" % (len(out), len(error_files))) if aux: aux_data_blocks = read_ima(start, finish, dataset="aux", verbose=verbose) if len(aux_data_blocks) != len(out): raise IOError("Number of aux data files doesn't match the actual data files") for block, aux_block in zip(out, aux_data_blocks): out["aux"] = aux_block return out
def read_els(start, finish=None, verbose=False): """ Read ASPERA/ELS data into blocks """ if finish is None: finish = start + 1.0 directory = mex.data_directory + "aspera/nilsson/els/mat/" et = start - 3600.0 # one second before out = [] error_files = [] remote_failed_count = 0 # allow_remote = True while True: if et > (finish + 3600.0): break dt = celsius.spiceet_to_datetime(et) fname = "%4d/elec%4d%02d%02d%02d.mat" % (dt.year, dt.year, dt.month, dt.day, dt.hour) remote_path = "[email protected]:/irf/data/mars/aspera3/mars/elsdata/" if not os.path.exists(directory + fname): remote_fname = remote_path + "%04d%02d/%4d%02d%02d%02d0000.mat" remote_fname = remote_fname % (dt.year, dt.month, dt.year, dt.month, dt.day, dt.hour) fd, temp_f_name = tempfile.mkstemp(suffix="tmp.mat") command = ("scp", remote_fname, temp_f_name) if verbose: print("Fetching %s" % remote_fname) try: # os.spawnvp(os.P_WAIT, command[0], command) subprocess.check_call(command) # reset fail count on success remote_failed_count = 0 except subprocess.CalledProcessError as e: remote_failed_count += 1 # raise IOError("Retrieval from aurora failed: %s" % str(e)) print("Retrieval of %s from aurora failed" % remote_fname) if remote_failed_count > MAX_FAILED_REMOTE_FILES: print("Maximum failed remote tries reached") break else: d = os.path.dirname(directory + fname) if d and not os.path.exists(d): if verbose: print("Creating %s" % d) os.makedirs(d) command = ("mv", temp_f_name, directory + fname) try: # os.spawnvp(os.P_WAIT, command[0], command) subprocess.check_call(command) except subprocess.CalledProcessError as e: print(e) raise IOError("Error moving file to %s" % (directory + fname)) try: b = loadmat(directory + fname) except IOError as e: error_files.append(fname) if verbose: print("Missing: %s" % fname) et += 60.0 * 60.0 continue # Squeeze arrays for o in b: if isinstance(b[o], np.ndarray): b[o] = np.squeeze(b[o]) if verbose: print( "Read: %s: %s - %s" % ( fname, celsius.time_convert(b["elstimes"][0, 0], "UTCSTR", "MATLABTIME"), celsius.time_convert(b["elstimes"][0, -1], "UTCSTR", "MATLABTIME"), ) ) # if b['EElec'].size < 2: # print("""---------------------- # read_els: %s contains data that is effectively 1-dimensional (i.e. not an image) # shape is %s. Ignoring this! # ---------------------""" % (fname, str(b['fElec'].shape))) # et += 60. * 60. # continue et += 60.0 * 60.0 out.append(b) return out