def read_ima(start, finish=None, dataset="FION", verbose=False, aux=False):
"""Read Nilsson's ima files into a list.
dataset="fion": best for heavy-ions
dataset="ion": best for protons, but not much in it.
dataset="aux": ancillary info
note: setting aux=True also reads the aux files, and appends each files contents into the
principal blocks being read by extending each dictionary."""
if aux:
if dataset == "aux":
raise ValueError("""Don't append aux data to the aux data. The whole universe will explode""")
if finish is None:
finish = start + 1.0
directory = mex.data_directory + "aspera/nilsson/Mars_mat_files4/"
remote_path = "[email protected]:/irf/data/mars/aspera3/Mars_mat_files4/"
et = start - 3600.0 # one second before
out = []
error_files = []
remote_failed_count = 0
while True:
if et > (finish + 3600.0):
break
dt = celsius.spiceet_to_datetime(et)
fname = dataset.lower() + "%4d%02d%02d%02d00.mat" % (dt.year, dt.month, dt.day, dt.hour)
if not os.path.exists(directory + fname):
remote_fname = remote_path + fname
fd, temp_f_name = tempfile.mkstemp(suffix="tmp.mat")
command = ("scp", remote_fname, temp_f_name)
if verbose:
print("Fetching %s" % remote_fname)
try:
# os.spawnvp(os.P_WAIT, command[0], command)
subprocess.check_call(command)
# reset to zero on success
remote_failed_count = 0
except subprocess.CalledProcessError as e:
remote_failed_count += 1
# raise IOError("Retrieval from aurora failed: %s" % str(e))
print("Retrieval of %s from aurora failed" % remote_fname)
if remote_failed_count > MAX_FAILED_REMOTE_FILES:
print("Maximum failed remote tries reached")
break
else:
d = os.path.dirname(directory + fname)
if d and not os.path.exists(d):
if verbose:
print("Creating %s" % d)
os.makedirs(d)
command = ("mv", temp_f_name, directory + fname)
try:
# os.spawnvp(os.P_WAIT, command[0], command)
subprocess.check_call(command)
except subprocess.CalledProcessError as e:
print(e)
raise IOError("Error moving file to %s" % (directory + fname))
try:
out.append(loadmat(directory + fname))
# Squeeze arrays
for o in out[-1]:
if isinstance(out[-1][o], np.ndarray):
out[-1][o] = np.squeeze(out[-1][o])
if verbose:
print(
"Read: %s: %s - %s"
% (
fname,
celsius.time_convert(out[-1]["tmptime"][0], "UTCSTR", "MATLABTIME"),
celsius.time_convert(out[-1]["tmptime"][-1], "UTCSTR", "MATLABTIME"),
)
)
except IOError as e:
error_files.append(fname)
if verbose:
print("Missing: %s" % fname)
et += 60.0 * 60.0
if verbose:
print("Read %d files - %d errors" % (len(out), len(error_files)))
if aux:
aux_data_blocks = read_ima(start, finish, dataset="aux", verbose=verbose)
if len(aux_data_blocks) != len(out):
raise IOError("Number of aux data files doesn't match the actual data files")
for block, aux_block in zip(out, aux_data_blocks):
out["aux"] = aux_block
return out
def read_els(start, finish=None, verbose=False):
"""
Read ASPERA/ELS data into blocks
"""
if finish is None:
finish = start + 1.0
directory = mex.data_directory + "aspera/nilsson/els/mat/"
et = start - 3600.0 # one second before
out = []
error_files = []
remote_failed_count = 0
# allow_remote = True
while True:
if et > (finish + 3600.0):
break
dt = celsius.spiceet_to_datetime(et)
fname = "%4d/elec%4d%02d%02d%02d.mat" % (dt.year, dt.year, dt.month, dt.day, dt.hour)
remote_path = "[email protected]:/irf/data/mars/aspera3/mars/elsdata/"
if not os.path.exists(directory + fname):
remote_fname = remote_path + "%04d%02d/%4d%02d%02d%02d0000.mat"
remote_fname = remote_fname % (dt.year, dt.month, dt.year, dt.month, dt.day, dt.hour)
fd, temp_f_name = tempfile.mkstemp(suffix="tmp.mat")
command = ("scp", remote_fname, temp_f_name)
if verbose:
print("Fetching %s" % remote_fname)
try:
# os.spawnvp(os.P_WAIT, command[0], command)
subprocess.check_call(command)
# reset fail count on success
remote_failed_count = 0
except subprocess.CalledProcessError as e:
remote_failed_count += 1
# raise IOError("Retrieval from aurora failed: %s" % str(e))
print("Retrieval of %s from aurora failed" % remote_fname)
if remote_failed_count > MAX_FAILED_REMOTE_FILES:
print("Maximum failed remote tries reached")
break
else:
d = os.path.dirname(directory + fname)
if d and not os.path.exists(d):
if verbose:
print("Creating %s" % d)
os.makedirs(d)
command = ("mv", temp_f_name, directory + fname)
try:
# os.spawnvp(os.P_WAIT, command[0], command)
subprocess.check_call(command)
except subprocess.CalledProcessError as e:
print(e)
raise IOError("Error moving file to %s" % (directory + fname))
try:
b = loadmat(directory + fname)
except IOError as e:
error_files.append(fname)
if verbose:
print("Missing: %s" % fname)
et += 60.0 * 60.0
continue
# Squeeze arrays
for o in b:
if isinstance(b[o], np.ndarray):
b[o] = np.squeeze(b[o])
if verbose:
print(
"Read: %s: %s - %s"
% (
fname,
celsius.time_convert(b["elstimes"][0, 0], "UTCSTR", "MATLABTIME"),
celsius.time_convert(b["elstimes"][0, -1], "UTCSTR", "MATLABTIME"),
)
)
# if b['EElec'].size < 2:
# print("""----------------------
# read_els: %s contains data that is effectively 1-dimensional (i.e. not an image)
# shape is %s. Ignoring this!
# ---------------------""" % (fname, str(b['fElec'].shape)))
# et += 60. * 60.
# continue
et += 60.0 * 60.0
out.append(b)
return out
