def read_raw(filename):
"""Read *filename* without scaling it afterwards.
"""
form = XMLFormat(os.path.join(CONFIG_PATH, "eps_avhrrl1b_6.5.xml"))
grh_dtype = np.dtype([("record_class", "|i1"), ("INSTRUMENT_GROUP", "|i1"),
("RECORD_SUBCLASS", "|i1"),
("RECORD_SUBCLASS_VERSION", "|i1"),
("RECORD_SIZE", ">u4"), ("RECORD_START_TIME", "S6"),
("RECORD_STOP_TIME", "S6")])
dtypes = []
cnt = 0
with open(filename, "rb") as fdes:
while True:
grh = np.fromfile(fdes, grh_dtype, 1)
if not grh:
break
rec_class = record_class[int(grh["record_class"])]
sub_class = grh["RECORD_SUBCLASS"][0]
expected_size = int(grh["RECORD_SIZE"])
bare_size = expected_size - grh_dtype.itemsize
try:
the_type = form.dtype((rec_class, sub_class))
the_descr = grh_dtype.descr + the_type.descr
except KeyError:
the_type = np.dtype([('unknown', 'V%d' % bare_size)])
the_descr = grh_dtype.descr + the_type.descr
the_type = np.dtype(the_descr)
if the_type.itemsize < expected_size:
padding = [('unknown%d' % cnt,
'V%d' % (expected_size - the_type.itemsize))]
cnt += 1
the_descr += padding
dtypes.append(np.dtype(the_descr))
fdes.seek(expected_size - grh_dtype.itemsize, 1)
file_dtype = np.dtype([(str(num), the_dtype)
for num, the_dtype in enumerate(dtypes)])
records = np.memmap(fdes, mode='r', dtype=file_dtype, shape=1)[0]
return records, form
def read_raw(filename):
"""Read *filename* without scaling it afterwards.
"""
form = XMLFormat(os.path.join(CONFIG_PATH, "eps_avhrrl1b_6.5.xml"))
grh_dtype = np.dtype([("record_class", "|i1"),
("INSTRUMENT_GROUP", "|i1"),
("RECORD_SUBCLASS", "|i1"),
("RECORD_SUBCLASS_VERSION", "|i1"),
("RECORD_SIZE", ">u4"),
("RECORD_START_TIME", "S6"),
("RECORD_STOP_TIME", "S6")])
record_class = ["Reserved", "mphr", "sphr",
"ipr", "geadr", "giadr",
"veadr", "viadr", "mdr"]
records = []
with open(filename, "rb") as fdes:
while True:
grh = np.fromfile(fdes, grh_dtype, 1)
if not grh:
break
rec_class = record_class[int(grh["record_class"])]
sub_class = grh["RECORD_SUBCLASS"][0]
offset = fdes.tell()
try:
record = np.fromfile(fdes,
form.dtype((rec_class,
sub_class)),
count=1)
except KeyError:
fdes.seek(int(grh["RECORD_SIZE"]) - 20, 1)
else:
fdes.seek(offset + int(grh["RECORD_SIZE"]) - 20, 0)
records.append((rec_class, record, sub_class))
return records, form
def read_raw(filename):
"""Read *filename* without scaling it afterwards.
"""
form = XMLFormat(os.path.join(CONFIG_PATH, "eps_avhrrl1b_6.5.xml"))
grh_dtype = np.dtype([("record_class", "|i1"),
("INSTRUMENT_GROUP", "|i1"),
("RECORD_SUBCLASS", "|i1"),
("RECORD_SUBCLASS_VERSION", "|i1"),
("RECORD_SIZE", ">u4"),
("RECORD_START_TIME", "S6"),
("RECORD_STOP_TIME", "S6")])
record_class = ["Reserved", "mphr", "sphr",
"ipr", "geadr", "giadr",
"veadr", "viadr", "mdr"]
records = []
with open(filename, "rb") as fdes:
while True:
grh = np.fromfile(fdes, grh_dtype, 1)
if not grh:
break
rec_class = record_class[int(grh["record_class"])]
sub_class = grh["RECORD_SUBCLASS"][0]
offset = fdes.tell()
try:
record = np.fromfile(fdes,
form.dtype((rec_class,
sub_class)),
count=1)
except KeyError:
fdes.seek(int(grh["RECORD_SIZE"]) - 20, 1)
else:
fdes.seek(offset + int(grh["RECORD_SIZE"]) - 20, 0)
records.append((rec_class, record, sub_class))
return records, form
def read_records(filename):
"""Read *filename* without scaling it afterwards."""
form = XMLFormat(os.path.join(CONFIG_PATH, "eps_avhrrl1b_6.5.xml"))
grh_dtype = np.dtype([("record_class", "|i1"), ("INSTRUMENT_GROUP", "|i1"),
("RECORD_SUBCLASS", "|i1"),
("RECORD_SUBCLASS_VERSION", "|i1"),
("RECORD_SIZE", ">u4"), ("RECORD_START_TIME", "S6"),
("RECORD_STOP_TIME", "S6")])
max_lines = np.floor((CHUNK_SIZE**2) / 2048)
dtypes = []
cnt = 0
counts = []
classes = []
prev = None
with open(filename, "rb") as fdes:
while True:
grh = np.fromfile(fdes, grh_dtype, 1)
if grh.size == 0:
break
rec_class = record_class[int(grh["record_class"])]
sub_class = grh["RECORD_SUBCLASS"][0]
expected_size = int(grh["RECORD_SIZE"])
bare_size = expected_size - grh_dtype.itemsize
try:
the_type = form.dtype((rec_class, sub_class))
# the_descr = grh_dtype.descr + the_type.descr
except KeyError:
the_type = np.dtype([('unknown', 'V%d' % bare_size)])
the_descr = grh_dtype.descr + the_type.descr
the_type = np.dtype(the_descr)
if the_type.itemsize < expected_size:
padding = [('unknown%d' % cnt,
'V%d' % (expected_size - the_type.itemsize))]
cnt += 1
the_descr += padding
new_dtype = np.dtype(the_descr)
key = (rec_class, sub_class)
if key == prev:
counts[-1] += 1
else:
dtypes.append(new_dtype)
counts.append(1)
classes.append(key)
prev = key
fdes.seek(expected_size - grh_dtype.itemsize, 1)
sections = {}
offset = 0
for dtype, count, rec_class in zip(dtypes, counts, classes):
fdes.seek(offset)
if rec_class == ('mdr', 2):
record = da.from_array(np.memmap(fdes,
mode='r',
dtype=dtype,
shape=count,
offset=offset),
chunks=(max_lines, ))
else:
record = np.fromfile(fdes, dtype=dtype, count=count)
offset += dtype.itemsize * count
if rec_class in sections:
logger.debug('Multiple records for ', str(rec_class))
sections[rec_class] = np.hstack((sections[rec_class], record))
else:
sections[rec_class] = record
return sections, form