def sonobat2guano(fname):
"""Convert a file with Sonobat metadata to GUANO metadata"""
print '\n', fname
sb_md = extract_sonobat_metadata(fname)
if not sb_md:
print >> sys.stderr, 'Skipping non-Sonobat file: ' + fname
return False
pprint(sb_md)
gfile = GuanoFile(fname)
gfile['GUANO|Version'] = 1.0
if 'timestamp' in sb_md:
gfile['Timestamp'] = sb_md['timestamp']
if sb_md.get('te', 1) != 1:
gfile['TE'] = sb_md['te']
gfile['Length'] = sb_md['length']
gfile['Note'] = sb_md['note'].strip().replace('\r\n', '\\n').replace('\n', '\\n')
if sb_md.get('species', None):
gfile['Species Auto ID'] = sb_md['species']
if 'd500x' in sb_md:
for k, v in sb_md['d500x'].items():
gfile['PET', k] = v
if 'ar125' in sb_md:
for k, v in sb_md['ar125'].items():
gfile['BAT', k] = v
print gfile._as_string()
gfile.write()
def save(self):
fname = self.ui.file_name.text()
try:
g = GuanoFile(fname)
except:
msg = f"There was a problem loading the Guano MD from:\n{fname}\n\nPlease verify that it is a valid wav file"
QMessageBox.warning(self, "File Error", msg)
return None
for namespace_name, namespace_group in self.namespaces.items():
print(namespace_name)
namespace_data = namespace_group.get_data()
if namespace_name == 'guano_base':
namespace_name = ''
for k, v in namespace_data.items():
if v == '':
try:
del g[f"{namespace_name}|{k}"]
except KeyError:
pass
else:
g[f"{namespace_name}|{k}"] = v
g.write(make_backup=False)
def wamd2guano(fname):
"""Convert a Wildlife Acoustics WAMD metadata file to GUANO metadata format"""
wamd_md = wamd(fname)
pprint(wamd_md)
gfile = GuanoFile(fname)
gfile['GUANO|Version'] = 1.0
gfile['Timestamp'] = wamd_md.pop('timestamp')
gfile['Note'] = wamd_md.pop('notes', '')
gfile['Make'] = 'Wildlife Acoustics'
gfile['Model'] = wamd_md.pop('model', '')
gfile['Firmware Version'] = wamd_md.pop('firmware', '')
gfile['Species Auto ID'] = wamd_md.pop('auto_id', '')
gfile['Species Manual ID'] = wamd_md.pop('manual_id', '')
gfile['TE'] = wamd_md.pop('time_expansion', 1)
gfile['Samplerate'] = gfile.wav_params.framerate * gfile['TE']
gfile['Length'] = gfile.wav_params.nframes / float(gfile.wav_params.framerate) * gfile['TE']
if 'gpsfirst' in wamd_md:
lat, lon, alt = wamd_md.pop('gpsfirst')
gfile['Loc Position'] = lat, lon
gfile['Loc Elevation'] = alt
for k, v in wamd_md.items():
gfile['WA', k] = v
print(gfile.to_string())
gfile.write()
def batlogger2guano(fname):
"""Convert an Elekon BatLogger .WAV with sidecar .XML to GUANO metadata"""
xmlfname = os.path.splitext(fname)[0] + '.xml'
if not os.path.exists(xmlfname):
raise ValueError('Unable to find XML metadata file for %s' % fname)
g = GuanoFile(fname)
with open(xmlfname, 'rt') as f:
xml = ElementTree.parse(f)
g['Timestamp'] = get(xml, 'DateTime', lambda x: datetime.strptime(x, '%d.%m.%Y %H:%M:%S'))
g['Firmware Version'] = get(xml, 'Firmware')
g['Make'] = 'Elekon'
g['Model'] = 'BatLogger'
g['Serial'] = get(xml, 'SN')
g['Samplerate'] = get(xml, 'Samplerate', lambda x: int(x.split()[0]))
g['Length'] = get(xml, 'Duration', lambda x: float(x.split()[0]))
g['Original Filename'] = get(xml, 'Filename')
g['Temperature Ext'] = get(xml, 'Temperature', lambda x: float(x.split()[0]))
g['Loc Position'] = get(xml, 'GPS/Position', lambda x: tuple(map(float, x.split())))
g['Loc Elevation'] = get(xml, 'GPS/Altitude', lambda x: float(x.split()[0]))
g['Elekon|BattVoltage'] = get(xml, 'BattVoltage')
for node in xml.find('Trigger'):
g['Elekon|Trigger|%s' % node.tag] = node.text
for node in xml.find('GPS'):
g['Elekon|GPS|%s' % node.tag] = node.text
# for k, v in g.items():
# print('%s:\t%s' % (k, v))
print(g.to_string())
g.write()
os.remove(xmlfname)
return g
def wamd2guano(fname):
"""Convert a Wildlife Acoustics WAMD metadata file to GUANO metadata format"""
wamd_md = wamd(fname)
pprint(wamd_md)
gfile = GuanoFile(fname)
gfile['GUANO|Version'] = 1.0
gfile['Timestamp'] = wamd_md.pop('timestamp')
gfile['Note'] = wamd_md.pop('notes', '')
gfile['Make'] = 'Wildlife Acoustics'
gfile['Model'] = wamd_md.pop('model', '')
gfile['Firmware Version'] = wamd_md.pop('firmware', '')
gfile['Species Auto ID'] = wamd_md.pop('auto_id', '')
gfile['Species Manual ID'] = wamd_md.pop('manual_id', '')
gfile['TE'] = wamd_md.pop('time_expansion', 1)
gfile['Samplerate'] = gfile.wav_params.framerate * gfile['TE']
gfile['Length'] = gfile.wav_params.nframes / float(gfile.wav_params.framerate) * gfile['TE']
if 'gpsfirst' in wamd_md:
lat, lon, alt = wamd_md.pop('gpsfirst')
gfile['Loc Position'] = lat, lon
gfile['Loc Elevation'] = alt
for k, v in wamd_md.items():
gfile['WAC', k] = v
print gfile._as_string()
gfile.write()
def sonobat2guano(fname):
"""Convert a file with Sonobat metadata to GUANO metadata"""
print('\n', fname)
sb_md = extract_sonobat_metadata(fname)
if not sb_md:
print('Skipping non-Sonobat file: ' + fname, file=sys.stderr)
return False
pprint(sb_md)
gfile = GuanoFile(fname)
gfile['GUANO|Version'] = 1.0
if 'timestamp' in sb_md:
gfile['Timestamp'] = sb_md['timestamp']
if sb_md.get('te', 1) != 1:
gfile['TE'] = sb_md['te']
gfile['Length'] = sb_md['length']
gfile['Note'] = sb_md['note'].strip().replace('\r\n',
'\\n').replace('\n', '\\n')
if sb_md.get('species', None):
gfile['Species Auto ID'] = sb_md['species']
if 'd500x' in sb_md:
for k, v in sb_md['d500x'].items():
gfile['PET', k] = v
if 'ar125' in sb_md:
for k, v in sb_md['ar125'].items():
gfile['BAT', k] = v
print(gfile.to_string())
gfile.write()
def update_single_md(fname, guano_md={}, project_md={}, site_md={}):
g = GuanoFile(fname)
parts = parse_nabat_fname(fname)
# content from the NABat filename
g['NABat', 'Grid Cell GRTS ID'] = parts['GrtsId']
g['NABat', 'Site Name'] = parts['SiteName']
#content from the site_md
if guano_md is None:
g['GUANO', 'time_to_time'] = str(parts['datetime'])
else:
for k, v in guano_md.items():
g['GUANO', k] = v
#content from the project_md
if project_md is not None:
for k, v in project_md.items():
g['NABat', k] = v
#content from the site_md
if site_md is not None:
for k, v in site_md.items():
g['NABat', k] = v
g.write(make_backup=False)
def load_file(self):
for i in reversed(range(self.namespace_layout.count() - 1)):
self.namespace_layout.itemAt(i).widget().setParent(None)
for namespce_chk in self.namespace_chks.values():
namespce_chk.setChecked(False)
self.namespaces = OrderedDict()
fname = self.ui.file_name.text()
f = Path(fname)
try:
exists = f.exists()
except:
exists = False
if exists:
try:
g = GuanoFile(fname)
except:
msg = f"There was a problem loading the Guano MD from:\n{fname}\n\nPlease verify that it is a valid wav file"
QMessageBox.warning(self, "File Error", msg)
return None
self.guano_content = {key: {} for key in g.get_namespaces()}
for item in g.items_namespaced():
self.guano_content[item[0]][item[1]] = item[2]
for namespace in g.get_namespaces():
if namespace == '':
namespace = 'guano_base'
namespace_fname = utils.resource_path(
f"../resources/specs/{namespace}.csv")
if Path(namespace_fname).exists():
spec = utils.read_namespace(namespace_fname)
else:
# if we have a namespace we've never seen, load it up as if it was a complete spec
spec = [{
'tag': tag
} for tag in self.guano_content[namespace].keys()]
this_namespace = NamespaceGroup(namespace, spec)
if namespace == 'guano_base':
this_namespace.load_data(self.guano_content[''])
else:
this_namespace.load_data(self.guano_content[namespace])
index = self.ui.scrollAreaWidgetContents_2.layout().count() - 1
self.ui.scrollAreaWidgetContents_2.layout().insertWidget(
index, this_namespace)
self.namespaces[namespace] = this_namespace
try:
self.namespace_chks[namespace].setChecked(True)
except KeyError:
pass
def test_strict_mode(self):
md = '''GUANO|Version: 1.0
TE: no
Loc Position: 10N 567288E 4584472N
'''
try:
GuanoFile.from_string(md, strict=True)
except ValueError as e:
pass
g = GuanoFile.from_string(md, strict=False)
self.assertEqual(g.get('TE', None), 'no')
self.assertEqual(g.get('Loc Position', None), '10N 567288E 4584472N')
def save(self):
d = Path(self.ui.directory_name.text())
wavs = list(d.glob('**\*.wav'))
wavs += list(d.glob('**\*.zc'))
self.ui.progressBar.setVisible(True)
self.ui.progressBar.setMinimum(1)
self.ui.progressBar.setMaximum(len(wavs))
self.ui.progressBar.setVisible(1)
if self.ui.directory_name_output.text():
if not Path(self.ui.directory_name_output.text()).exists():
msq = r"The output directory specified does not exist! Please point to an existing directory."
QMessageBox.warning(self, "Output directory does not exist",
msg)
return None
out_dir = Path(self.ui.directory_name_output.text())
make_copy = True
else:
out_dir = Path(self.ui.directory_name.text())
make_copy = False
for f in wavs:
original_fname = f.name
new_fname = self.change_fname_function(original_fname, f=f)
full_name = f.parent.joinpath(new_fname)
full_name = str(full_name).replace(str(Path(self.ui.directory_name.text())), \
str(out_dir))
if not Path(full_name).exists():
Path(full_name).parent.mkdir(parents=True, exist_ok=True)
if make_copy:
shutil.copy(str(f), full_name)
else:
f.rename(full_name)
if original_fname != new_fname:
try:
g = GuanoFile(full_name)
g['Original Filename'] = original_fname
g.write(make_backup=False)
except:
pass
self.ui.progressBar.setValue(self.ui.progressBar.value() + 1)
msg = f"Finished renaming files in directory:\n\n{out_dir}"
QMessageBox.information(self, "Renaming Process Complete", msg)
self.ui.progressBar.setVisible(False)
def test_file_roundtrip(self):
"""Write a GUANO .WAV file containing Unicode data, re-read it and confirm value is identical"""
fname = 'test_guano.wav'
# write a fake .WAV file
g = GuanoFile.from_string(self.MD)
g.filename = fname
g.wav_params = wavparams(1, 2, 500000, 2, 'NONE', None)
g._wav_data = b'\01\02' # faking it, don't try this at home!
g._wav_data_size = 2
g.write()
# read it back in
g2 = GuanoFile(fname)
self.assertEqual(self.NOTE, g2['Note'])
def process_file(self, filename):
filepath = os.path.realpath(filename)
filestem = os.path.basename(filename).split('.')[0]
print(f'Loading {filepath}')
existing_audio = AudioRecording.objects.filter(audio_file=filepath)
result_count = len(existing_audio)
if result_count:
if result_count > 1:
print(f'Got duplicate records ({result_count}) for {filepath}')
audio = existing_audio[0]
audio.identifier = filestem
if audio.processed and not self.force:
print("Already processed this file, skipping")
return
print('Found incomplete existing record, trying to update')
else:
audio = AudioRecording(audio_file=filepath, identifier=filestem)
self.populate_audio_from_identifier(audio)
try:
# print(f'Looking for GUANO data for {filename}')
guano_file = GuanoFile(filepath)
if not guano_file:
# print(f'Empty GUANO data for {filename}')
guano_file = None
except ValueError:
print(f'Unable to load GUANO data for {filename}')
guano_file = None
wamd_file = None
if guano_file is not None:
self.populate_audio_from_guano(audio, guano_file)
else:
# print(f'Looking for WAMD data for {filename}')
try:
wamd_file = WamdFile(filepath)
# print(f'Loaded WAMD data for {filename}')
except ValueError:
print(f'Unable to load WAMD data for {filename}')
if wamd_file is not None:
try:
self.populate_audio_from_wamd(audio, wamd_file)
except ValueError:
print("Could not load fallback WAMD data")
if not audio.duration:
self.read_file_duration(filename, audio)
if self.subsample:
self.subsample_file(audio)
if self.spectrogram:
self.generate_spectrogram_file(audio)
audio.save()
def test_new_empty(self):
"""Verify that "new" GUANO file metadata is "falsey" but populated metadata is "truthy"."""
g = GuanoFile('nonexistent_file.wav')
self.assertFalse(g)
self.assertFalse('GUANO|Version' in g)
g['Foo'] = 'bar'
self.assertTrue(g)
self.assertTrue('GUANO|Version' in g)
def d500x2guano(fname):
"""Convert a file with raw D500X metadata to use GUANO metadata instead"""
print('\n', fname)
md = extract_d500x_metadata(fname)
if not md:
print('Skipping non-D500X file: ' + fname, file=sys.stderr)
return False
pprint(md)
gfile = GuanoFile(fname)
gfile['GUANO|Version'] = 1.0
gfile['Make'] = 'Pettersson'
gfile['Model'] = 'D500X'
gfile['Timestamp'] = md.pop('File Time')
gfile['Original Filename'] = md.pop('File Name')
gfile['Samplerate'] = md.pop('Samplerate')
gfile['Length'] = md.pop('Length')
if md.get('Profile HP', None) == 'Y':
gfile['Filter HP'] = 20
lat, lon = md.pop('LAT', None), md.pop('LON', None)
if lat and lon:
gfile['Loc Position'] = dms2decimal(lat), dms2decimal(lon)
for k, v in md.items():
gfile['PET', k] = v
print(gfile.to_string())
gfile.wav_data = gfile.wav_data[D500X_DATA_SKIP_BYTES:] # throw out the metadata bytes from 'data' chunk
unlock(fname) # D500X "locks" files as unwriteable, we must unlock before we can modify
gfile.write()
def d500x2guano(fname):
"""Convert a file with raw D500X metadata to use GUANO metadata instead"""
print "\n", fname
md = extract_d500x_metadata(fname)
if not md:
print >> sys.stderr, "Skipping non-D500X file: " + fname
return False
pprint(md)
gfile = GuanoFile(fname)
gfile["GUANO|Version"] = 1.0
gfile["Make"] = "Pettersson"
gfile["Model"] = "D500X"
gfile["Timestamp"] = md.pop("File Time")
gfile["Samplerate"] = md.pop("Samplerate")
gfile["Length"] = md.pop("Length")
if md.get("Profile HP", None) == "Y":
gfile["Filter HP"] = 20
lat, lon = md.pop("LAT", None), md.pop("LON", None)
if lat and lon:
gfile["Loc Position"] = dms2decimal(lat), dms2decimal(lon)
for k, v in md.items():
gfile["PET", k] = v
print gfile._as_string()
gfile.wav_data = gfile.wav_data[D500X_DATA_SKIP_BYTES:] # throw out the metadata bytes from 'data' chunk
unlock(fname) # D500X "locks" files as unwriteable, we must unlock before we can modify
gfile.write()
def test_delete_namespaced(self):
"""Verify that we can delete namespaced fields"""
g = GuanoFile()
g['Foo|Bar'] = 'xyz'
self.assertTrue('Foo|Bar' in g)
self.assertTrue('Foo' in g.get_namespaces())
del g['Foo|Bar']
self.assertFalse('Foo|Bar' in g)
self.assertFalse('Foo' in g.get_namespaces())
try:
del g['Foo|Bar']
self.fail('Deleting a deleted key should throw KeyError')
except KeyError:
pass
g['Foo|Bar1'] = 'xyz'
g['Foo|Bar2'] = 'abc'
del g['Foo|Bar1']
self.assertFalse('Foo|Bar1' in g)
self.assertTrue('Foo|Bar2' in g)
self.assertTrue('Foo' in g.get_namespaces())
def test_delete_simple(self):
"""Verify that we can delete fields"""
g = GuanoFile()
g['Foo'] = 'xyz'
self.assertTrue('Foo' in g)
del g['Foo']
self.assertFalse('Foo' in g)
try:
del g['Foo']
self.fail('Deleting a deleted key should throw KeyError')
except KeyError:
pass
def run(self):
md = self.zc.metadata
timestamp = md.get('timestamp', None)
species = md.get('species', '')
note1 = md.get('note1', '')
if note1:
note2 = 'Myotisoft ZCANT'
else:
note1, note2 = 'Myotisoft ZCANT', ''
if self.zc.supports_amplitude:
log.debug('Adding GUANO metadata :-)')
guano = GuanoFile()
guano['ZCANT|Amplitudes'] = self.zc.amplitudes
else:
log.debug('Not adding GUANO metadata :-(')
guano = None
log.debug('Saving %s ...', self.fname)
outdir = os.path.dirname(self.fname)
if not os.path.exists(outdir):
log.debug('Creating outdir %s ...', outdir)
os.makedirs(outdir)
with AnabatFileWriter(self.fname) as out:
out.write_header(timestamp,
self.divratio,
species=species,
note1=note1,
note2=note2,
guano=guano)
time_indexes_us = self.zc.times * 1000000
intervals_us = np.diff(time_indexes_us)
intervals_us = intervals_us.astype(
int
) # TODO: round before int cast; consider casting before diff for performance
out.write_intervals(intervals_us)
def populate_audio_from_guano(audio: AudioRecording,
guano_file: GuanoFile):
guano_dict = {k: v for k, v in guano_file.items()}
for k, v in guano_dict.items():
if isinstance(v, datetime):
guano_dict[k] = v.strftime('%Y-%m-%d %H:%M:%S')
audio.guano_data = json.dumps(guano_dict)
# Species Auto ID: PIPPYG
# Species Manual ID:
if 'Species Manual ID' in guano_dict and len(
guano_dict['Species Manual ID']) > 0:
populate_audio_identification(audio,
guano_dict['Species Manual ID'])
elif 'Species Auto ID' in guano_dict and len(
guano_dict['Species Auto ID']) > 0:
populate_audio_identification(audio, guano_dict['Species Auto ID'])
if 'Loc Position' in guano_dict:
audio.latitude, audio.longitude = guano_dict['Loc Position']
if 'Serial' in guano_file:
audio.recorder_serial = guano_file['Serial']
if 'Length' in guano_file:
audio.duration = guano_file['Length']
try:
if 'Timestamp' in guano_file and guano_file[
'Timestamp'] is not None:
if guano_file['Timestamp'].utcoffset() is None:
print('Timestamp with no tzinfo from guano '
'cannot be saved')
else:
audio.set_recording_time(guano_file['Timestamp'])
except ValueError:
print('Cannot use timestamp from guano - no valid TZ?')
finally:
audio.processed = True
def setUp(self):
GuanoFile.register('User', 'Answer', int)
self.md = GuanoFile.from_string(self.MD)
def setUp(self):
self.g = g = GuanoFile()
g['A'] = 'A value'
g['Foo Bar'] = 'Foo Bar value'
g['NS|C'] = 'C value'
g['NS|Foo Bar'] = 'Namespaced Foo Bar value'
def extract_anabat(fname, hpfilter_khz=8.0, **kwargs):
"""Extract (times, frequencies, amplitudes, metadata) from Anabat sequence file"""
amplitudes = None
with open(fname, 'rb') as f, contextlib.closing(
mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)) as m:
size = len(m)
# parse header
data_info_pointer, file_type, tape, date, loc, species, spec, note1, note2 = struct.unpack_from(
ANABAT_129_HEAD_FMT, m)
data_pointer, res1, divratio, vres = struct.unpack_from(
ANABAT_129_DATA_INFO_FMT, m, data_info_pointer)
species = [_s(species).split('(', 1)[0]] if '(' in species else [
s.strip() for s in _s(species).split(',')
] # remove KPro junk
metadata = dict(date=date,
loc=_s(loc),
species=species,
spec=_s(spec),
note1=_s(note1),
note2=_s(note2),
divratio=divratio)
if file_type >= 132:
year, month, day, hour, minute, second, second_hundredths, microseconds, id_code, gps_data = struct.unpack_from(
ANABAT_132_ADDL_DATA_INFO_FMT, m, 0x120)
try:
timestamp = datetime(year, month, day, hour, minute, second,
second_hundredths * 10000 + microseconds)
except ValueError as e:
log.exception('Failed extracting timestamp')
timestamp = None
metadata.update(
dict(timestamp=timestamp, id=_s(id_code), gps=_s(gps_data)))
if data_pointer - 0x150 > 12: # and m[pos:pos+5] == 'GUANO':
try:
guano = GuanoFile.from_string(m[0x150:data_pointer])
log.debug(guano.to_string())
amplitudes = guano.get('ZCANT|Amplitudes', None)
except:
log.exception('Failed parsing GUANO metadata block')
else:
log.debug('No GUANO metadata found')
log.debug(
'file_type: %d\tdata_info_pointer: 0x%3x\tdata_pointer: 0x%3x',
file_type, data_info_pointer, data_pointer)
log.debug(metadata)
if res1 != 25000:
raise ValueError(
'Anabat files with non-standard RES1 (%s) not yet supported!' %
res1)
# parse actual sequence data
i = data_pointer # byte index as we scan through the file (data starts at 0x150 for v132, 0x120 for older files)
intervals_us = np.empty(2**14, np.dtype('uint32'))
offdots = OrderedDict() # dot index -> number of subsequent dots
int_i = 0 # interval index
while i < size:
if int_i >= len(intervals_us):
# Anabat files were formerly capped at 16384 dots, but may now be larger; grow
intervals_us = np.concatenate(
(intervals_us, np.empty(2**14, np.dtype('uint32'))))
byte = Byte.unpack_from(m, i)[0]
if byte <= 0x7F:
# Single byte is a 7-bit signed two's complement offset from previous interval
offset = byte if byte < 2**6 else byte - 2**7 # clever two's complement unroll
if int_i > 0:
intervals_us[int_i] = intervals_us[int_i - 1] + offset
int_i += 1
else:
log.warning(
'Sequence file starts with a one-byte interval diff! Skipping byte %x',
byte)
#intervals.append(offset) # ?!
elif 0x80 <= byte <= 0x9F:
# time interval is contained in 13 bits, upper 5 from the remainder of this byte, lower 8 bits from the next byte
accumulator = (byte & 0b00011111) << 8
i += 1
accumulator |= Byte.unpack_from(m, i)[0]
intervals_us[int_i] = accumulator
int_i += 1
elif 0xA0 <= byte <= 0xBF:
# interval is contained in 21 bits, upper 5 from the remainder of this byte, next 8 from the next byte and the lower 8 from the byte after that
accumulator = (byte & 0b00011111) << 16
i += 1
accumulator |= Byte.unpack_from(m, i)[0] << 8
i += 1
accumulator |= Byte.unpack_from(m, i)[0]
intervals_us[int_i] = accumulator
int_i += 1
elif 0xC0 <= byte <= 0xDF:
# interval is contained in 29 bits, the upper 5 from the remainder of this byte, the next 8 from the following byte etc.
accumulator = (byte & 0b00011111) << 24
i += 1
accumulator |= Byte.unpack_from(m, i)[0] << 16
i += 1
accumulator |= Byte.unpack_from(m, i)[0] << 8
i += 1
accumulator |= Byte.unpack_from(m, i)[0]
intervals_us[int_i] = accumulator
int_i += 1
elif 0xE0 <= byte <= 0xFF:
# status byte which applies to the next n dots
status = byte & 0b00011111
i += 1
dotcount = Byte.unpack_from(m, i)[0]
if status == DotStatus.OFF:
offdots[int_i] = dotcount
else:
log.debug(
'UNSUPPORTED: Status %X for %d dots at dot %d (file offset 0x%X)',
status, dotcount, int_i, i)
else:
raise Exception('Unknown byte %X at offset 0x%X' % (byte, i))
i += 1
intervals_us = intervals_us[:int_i] # TODO: should we free unused memory?
intervals_s = intervals_us * 1e-6
times_s = np.cumsum(intervals_s)
freqs_hz = 1 / (times_s[2:] - times_s[:-2]) * divratio
freqs_hz[freqs_hz == np.inf] = 0 # TODO: fix divide-by-zero
freqs_hz[freqs_hz < 4000] = 0
freqs_hz[freqs_hz > 250000] = 0
if offdots:
n_offdots = sum(offdots.values())
log.debug('Throwing out %d off-dots of %d (%.1f%%)', n_offdots,
len(times_s),
float(n_offdots) / len(times_s) * 100)
off_mask = np.zeros(len(intervals_us), dtype=bool)
for int_i, dotcount in offdots.items():
off_mask[int_i:int_i + dotcount] = True
times_s = masked_array(times_s, mask=off_mask).compressed()
freqs_hz = masked_array(freqs_hz, mask=off_mask).compressed()
min_, max_ = min(freqs_hz) if any(freqs_hz) else 0, max(freqs_hz) if any(
freqs_hz) else 0
log.debug('%s\tDots: %d\tMinF: %.1f\tMaxF: %.1f', basename(fname),
len(freqs_hz), min_ / 1000.0, max_ / 1000.0)
times_s, freqs_hz, amplitudes = hpf_zc(times_s, freqs_hz, amplitudes,
hpfilter_khz * 1000)
assert (len(times_s) == len(freqs_hz) == len(amplitudes or freqs_hz))
return times_s, freqs_hz, amplitudes, metadata
def dump(fname):
print()
print(fname)
gfile = GuanoFile(fname, strict=False)
print(gfile.to_string())
def test_sb42_bad_timestamp(self):
"""SonoBat 4.2 blank timestamp"""
md = '''GUANO|Version: 1.0
Timestamp:
'''
GuanoFile.from_string(md)
def test_sb41_bad_key(self):
"""SonoBat 4.1 disembodied colon"""
md = '''GUANO|Version: 1.0
:
'''
self.assertEqual(1, len(list(GuanoFile.from_string(md).items())))
def test_sb42_bad_encoding(self):
"""SonoBat 4.2 doesn't actually encode as UTF-8. At least try not to blow up when reading."""
# SonoBat *probably* uses mac-roman on OS X and windows-1252 on Windows... in the US at least.
md = b'GUANO|Version: 1.0\nNote: Mobile transect with mic 4\xd5 above roof.\n\x00\x00'
GuanoFile.from_string(md)
def dump(fname):
print
print fname
gfile = GuanoFile(fname)
print gfile._as_string()
def test_sb42_bad_guano_version(self):
"""Some version of SonoBat 4.2 writes a GUANO|Version of "1.0:" by accident."""
md = b'GUANO|Version: 1.0:\n1.0:\n'
GuanoFile.from_string(md)
def test_from_string(self):
"""Parse a GUANO metadata block containing Unicode data"""
g = GuanoFile.from_string(self.MD)
self.assertEqual(self.NOTE, g['Note'])
from guano import GuanoFile, base64decode, base64encode
from zcant import print_timing
import logging
log = logging.getLogger(__name__)
Byte = struct.Struct('< B')
ANABAT_129_HEAD_FMT = '< H x B 2x 8s 8s 40s 50s 16s 73s 80s' # 0x0: data_info_pointer, file_type, tape, date, loc, species, spec, note1, note2
ANABAT_129_DATA_INFO_FMT = '< H H B B' # 0x11a: data_pointer, res1, divratio, vres
ANABAT_132_ADDL_DATA_INFO_FMT = '< H B B B B B B H 6s 32s' # 0x120: year, month, day, hour, minute, second, second_hundredths, microseconds, id_code, gps_data
GuanoFile.register('ZCANT', 'Amplitudes',
lambda b64data: np.frombuffer(base64decode(b64data)),
lambda data: base64encode(data.tobytes()))
class DotStatus:
"""Enumeration of dot status types"""
OUT_OF_RANGE = 0
OFF = 1
NORMAL = 2
MAIN = 3
def _s(s):
"""Strip whitespace and null bytes from string"""
return s.strip('\00\t ')
def get_row_from_guano(fname):
row = get_empty_row()
try:
g = GuanoFile(fname)
for i, keyvalue in row_lookup.iterrows():
value = g.get(keyvalue.guano_tag2, '')
if value.lower() == 'nan':
value = ''
row[keyvalue.df_columns] = value
if g.get('NABat|Site coordinates'):
# print('.')
lat, long = g.get('NABat|Site coordinates').split()
row['latitude'] = lat
row['longitude'] = long
# parse the software type from vendor namespaces
if 'SB' in g.get_namespaces():
software = 'Sonobat '
if g.get('SB|Version').startswith('4.2'):
software += '4.2'
elif g.get('SB|Version').startswith('4.'):
software += '4.x'
elif g.get('SB|Version').startswith('3.'):
software += '3.x'
row['software_type'] = software
else:
# TODO: add logic for Kaleidoscope
pass
# Make sure we're using a auto/manual ID if available
if row['auto_id'] == '':
row['auto_id'] = g.get('GUANO|Species Auto ID', '')
if row['manual_id'] == '':
row['manual_id'] = g.get('GUANO|Species Manual ID', '')
# convert nan to empty string
for which in ['auto', 'manual']:
if row[f'{which}_id'].lower() == 'nan':
row[f'{which}_id'] = ''
except:
# Something went dreadfully wrong. We'll populate with what we have
parts = parse_nabat_fname(fname)
row['grts_cell_id'] = parts['GrtsId']
row['location_name'] = parts['SiteName']
row['detector'] = "Problem extracting row from Guano"
row['audio_recording_name'] = Path(fname).name
return row
def test_sb41_bad_te(self):
"""SonoBat 4.1 "optional" TE value"""
md = '''GUANO|Version: 1.0
TE:
'''
GuanoFile.from_string(md)