def main(p_port):
if p_port == 0:
print 'port could not be set as 0'
log.e('port could not be set as 0')
exit(1)
log.c('www listening on port : %s' % p_port)
app = Application()
app.listen(p_port)
log.i(app.settings['template_path'])
log.i(app.settings['static_path'])
tornado.ioloop.IOLoop.instance().start()
def make_response(data, success=True):
try:
if success:
data['response'] = 'success'
else:
data['response'] = 'error'
return json.dumps(data, ensure_ascii=False, encoding='iso-8859-1')
except:
log.e(('Response Error!', data), discard=True)
data = {}
data['response'] = 'error'
return json.dumps(data)
def parseDataset():
CLASSES = [
c for c in sorted(os.listdir(os.path.join(cfg.TRAINSET_PATH, 'train')))
]
for c in CLASSES:
log.i(c)
index = []
afiles = [
f for f in sorted(
os.listdir(os.path.join(cfg.TRAINSET_PATH, 'train', c)))
]
max_specs = cfg.MAX_SPECS_PER_CLASS // len(afiles) + 1
for i in range(len(afiles)):
spec_cnt = 0
try:
print(i + 1, '/', len(afiles), c, afiles[i])
specs, noise = getSpecs(
os.path.join(cfg.TRAINSET_PATH, 'train', c, afiles[i]))
for s in range(len(specs)):
if np.isnan(noise[s]):
noise[s] = 0.0
if noise[s] >= cfg.SPEC_SIGNAL_THRESHOLD:
filepath = os.path.join(cfg.DATASET_PATH, c)
if not os.path.exists(filepath):
os.makedirs(filepath)
elif noise[s] <= cfg.NOISE_THRESHOLD:
if RANDOM.choice([True, False], p=[0.60, 0.40]):
filepath = os.path.join(cfg.NOISE_PATH)
if not os.path.exists(filepath):
os.makedirs(filepath)
else:
filepath = None
else:
filepath = None
if filepath:
filename = str(
int(noise[s] *
10000)).zfill(4) + '_' + afiles[i].split(
'.')[0] + '_' + str(s).zfill(3)
# Write to HDD
cv2.imwrite(os.path.join(filepath, filename + '.png'),
specs[s] * 255.0)
if spec_cnt >= max_specs:
break
except:
log.e((spec_cnt, 'specs', 'ERROR DURING SPEC EXTRACT'))
continue
def parseDataset():
# List of classes, subfolders as class names
CLASSES = [
c for c in sorted(os.listdir(os.path.join(cfg.TRAINSET_PATH, 'train')))
]
# Parse every class
for c in CLASSES:
# List all audio files
afiles = [
f for f in sorted(
os.listdir(os.path.join(cfg.TRAINSET_PATH, 'train', c)))
]
# Calculate maximum specs per file
max_specs = cfg.MAX_SPECS_PER_CLASS // len(afiles) + 1
# Get specs for every audio file
for i in range(len(afiles)):
spec_cnt = 0
try:
# Stats
print i + 1, '/', len(afiles), c, afiles[i],
# Get specs and signal to noise ratios
specs, noise = getSpecs(
os.path.join(cfg.TRAINSET_PATH, 'train', c, afiles[i]))
# Save specs if it contains signal
for s in range(len(specs)):
# NaN?
if np.isnan(noise[s]):
noise[s] = 0.0
# Above SIGNAL_THRESHOLD?
if noise[s] >= cfg.SPEC_SIGNAL_THRESHOLD:
# Create target path for accepted specs
filepath = os.path.join(cfg.DATASET_PATH, c)
if not os.path.exists(filepath):
os.makedirs(filepath)
# Count specs
spec_cnt += 1
else:
# Create target path for rejected specs -
# but we don't want to save every sample (only 10%)
if RANDOM.choice([True, False], p=[0.1, 0.90]):
filepath = os.path.join(cfg.NOISE_PATH)
if not os.path.exists(filepath):
os.makedirs(filepath)
else:
filepath = None
if filepath:
# Filename contains s2n-ratio
filename = str(
int(noise[s] *
10000)).zfill(4) + '_' + afiles[i].split(
'.')[0] + '_' + str(s).zfill(3)
# Write to HDD
cv2.imwrite(os.path.join(filepath, filename + '.png'),
specs[s] * 255.0)
# Do we have enough specs already?
if spec_cnt >= max_specs:
break
# Stats
log.i((spec_cnt, 'specs'))
except:
log.e((spec_cnt, 'specs', 'ERROR DURING SPEC EXTRACT'))
continue
def parseDataset():
spec_all = []
# List of classes, subfolders as class names
CLASSES = [
c for c in sorted(os.listdir(os.path.join(cfg.TRAINSET_PATH, 'audio')))
]
stop = 0
#switch = 0
# Parse every class
for c in CLASSES:
#if stop==1:
#break
#if c == 'norfli':
# switch = 1
#if switch == 1:
# List all audio files
afiles = [
f for f in sorted(
os.listdir(os.path.join(cfg.TRAINSET_PATH, 'audio', c)))
]
# Calculate maximum specs per file
max_specs = cfg.MAX_SPECS_PER_CLASS // len(afiles) + 1
# Get specs for every audio file
for i in range(len(afiles)):
spec_cnt = 0
try:
# Stats
print(i + 1, '/', len(afiles), c, afiles[i])
# Get specs and signal to noise ratios
specs, noise = getSpecs(
os.path.join(cfg.TRAINSET_PATH, 'audio', c, afiles[i]))
# print('number of specs = ' + str(specs))
threshold = 0
max_index = 0
# Save specs if it contains signal
for s in range(len(specs)):
# NaN?
if np.isnan(noise[s]):
noise[s] = 0.0
# Above SIGNAL_THRESHOLD?
if noise[s] >= threshold:
threshold = noise[s]
max_index = s
#print(threshold)
# Create target path for accepted specs
spec_filepath = os.path.join(cfg.DATASET_PATH, c)
if not os.path.exists(spec_filepath):
os.makedirs(spec_filepath)
else:
# Create target path for rejected specs -
# but we don't want to save every sample (only 10%)
if RANDOM.choice([True, False], p=[0.1, 0.90]):
filepath = os.path.join(cfg.NOISE_PATH)
if not os.path.exists(filepath):
os.makedirs(filepath)
if filepath:
# Filename contains s2n-ratio
filename = str(int(
noise[s] *
10000)).zfill(4) + '_' + afiles[i].split(
'.')[0] + '_' + str(s).zfill(3)
# Write to HDD
cv2.imwrite(
os.path.join(filepath, filename + '.png'),
specs[s] * 255.0)
else:
filepath = None
if s == (len(specs) - 1):
# Filename contains s2n-ratio
filename = str(
int(noise[max_index] *
10000)).zfill(4) + '_' + afiles[i].split(
'.')[0] + '_' + str(s).zfill(3)
# Write to HDD
cv2.imwrite(
os.path.join(spec_filepath, filename + '.png'),
specs[max_index] * 255.0)
# Save all spec into a list
spec_all.append(specs[max_index])
# Count specs
spec_cnt += 1
# Do we have enough specs already?
if spec_cnt >= max_specs:
break
# Stats
log.i((spec_cnt, 'specs'))
except:
log.e((spec_cnt, 'specs', 'ERROR DURING SPEC EXTRACT'))
continue
stop += 1
print(spec_all)
spec_all = np.array(spec_all)
spec_mean = np.mean(spec_all)
spec_std = np.std(spec_all)
print(spec_mean)
print(spec_std)
def parseDataset():
filedata = {}
with open(r"flockfiles.json", 'r') as f:
filedata = json.load(f)
# List of classes, subfolders as class names
CLASSES = [c for c in sorted(os.listdir(r""))]
CLASSES = CLASSES[28:]
# Parse every class
for c in CLASSES:
log.i(c)
index = []
# List all audio files
print(c)
afiles = [f for f in sorted(os.listdir(os.path.join(r"", c)))]
print(len(afiles))
# Calculate maximum specs per file
for i in range(len(afiles)):
if (afiles[i][:-4] + '.WAV' in filedata[c]):
index.append(i)
print("del one")
index.reverse()
for i in index:
del afiles[i]
print(len(afiles))
max_specs = cfg.MAX_SPECS_PER_CLASS // len(afiles) + 1
# Get specs for every audio file
for i in range(len(afiles)):
spec_cnt = 0
try:
# Stats
print(i + 1, '/', len(afiles), c, afiles[i])
# Get specs and signal to noise ratios
specs, noise = getSpecs(os.path.join(r"", c, afiles[i]))
# Save specs if it contains signal
for s in range(len(specs)):
# NaN?
if np.isnan(noise[s]):
noise[s] = 0.0
# Above SIGNAL_THRESHOLD?
if noise[s] >= cfg.SPEC_SIGNAL_THRESHOLD:
# Create target path for accepted specs
filepath = os.path.join(cfg.DATASET_PATH, c)
if not os.path.exists(filepath):
os.makedirs(filepath)
elif noise[s] <= cfg.NOISE_THRESHOLD:
if RANDOM.choice([True, False], p=[0.60, 0.40]):
filepath = os.path.join(cfg.NOISE_PATH)
if not os.path.exists(filepath):
os.makedirs(filepath)
else:
filepath = None
else:
filepath = None
if filepath:
# Filename contains s2n-ratio
filename = str(
int(noise[s] *
10000)).zfill(4) + '_' + afiles[i].split(
'.')[0] + '_' + str(s).zfill(3)
# Write to HDD
cv2.imwrite(os.path.join(filepath, filename + '.png'),
specs[s] * 255.0)
# Do we have enough specs already?
if spec_cnt >= max_specs:
break
except:
log.e((spec_cnt, 'specs', 'ERROR DURING SPEC EXTRACT'))
continue
def test(SNAPSHOTS):
# Do we have more than one snapshot?
if not isinstance(SNAPSHOTS, (list, tuple)):
SNAPSHOTS = [SNAPSHOTS]
# Load snapshots
test_functions = []
for s in SNAPSHOTS:
# Settings
NET = s['net']
cfg.CLASSES = s['classes']
cfg.IM_DIM = s['im_dim']
cfg.IM_SIZE = s['im_size']
# Compile test function
test_net = birdnet.test_function(NET, hasTargets=False, layer_index=-1)
test_functions.append(test_net)
# Parse Testset
TEST = parseTestSet()
# Status
log.i('START TESTING...')
stats.clearStats()
stats.tic('test_time')
# Make predictions
for spec_batch, labels, filename in bg.threadedGenerator(
getSpecBatches(TEST)):
try:
# Status
stats.tic('pred_time')
# Prediction
prediction_batch = []
for test_func in test_functions:
if len(prediction_batch) == 0:
prediction_batch = test_func(spec_batch)
else:
prediction_batch += test_func(spec_batch)
prediction_batch /= len(test_functions)
# Eliminate the scores for 'Noise'
if 'Noise' in cfg.CLASSES:
prediction_batch[:, cfg.CLASSES.index('Noise')] = np.min(
prediction_batch)
# Prediction pooling
p_pool = predictionPooling(prediction_batch)
# Get class labels
p_labels = {}
for i in range(p_pool.shape[0]):
p_labels[cfg.CLASSES[i]] = p_pool[i]
# Sort by score
p_sorted = sorted(p_labels.items(),
key=operator.itemgetter(1),
reverse=True)
# Calculate MLRAP (MRR for single labels)
targets = np.zeros(p_pool.shape[0], dtype='float32')
for label in labels:
if label in cfg.CLASSES:
targets[cfg.CLASSES.index(label)] = 1.0
lrap = metrics.lrap(np.expand_dims(p_pool, 0),
np.expand_dims(targets, 0))
stats.setValue('lrap', lrap, mode='append')
# Show sample stats
log.i((filename), new_line=True)
log.i(('\tLABELS:', labels), new_line=True)
log.i(('\tTOP PREDICTION:', p_sorted[0][0],
int(p_sorted[0][1] * 1000) / 10.0, '%'),
new_line=True)
log.i(('\tLRAP:', int(lrap * 1000) / 1000.0), new_line=False)
log.i(('\tMLRAP:',
int(np.mean(stats.getValue('lrap')) * 1000) / 1000.0),
new_line=True)
# Save some stats
if p_sorted[0][0] == labels[0]:
stats.setValue('top1_correct', 1, 'add')
stats.setValue('top1_confidence', p_sorted[0][1], 'append')
else:
stats.setValue('top1_incorrect', 1, 'add')
stats.toc('pred_time')
stats.setValue('time_per_batch', stats.getValue('pred_time'),
'append')
except KeyboardInterrupt:
cfg.DO_BREAK = True
break
except:
log.e('ERROR WHILE TRAINING!')
continue
# Stats
stats.toc('test_time')
log.i(('TESTING DONE!', 'TIME:', stats.getValue('test_time'), 's'))
log.r(('FINAL MLRAP:', np.mean(stats.getValue('lrap'))))
log.r(('TOP 1 ACCURACY:',
max(
0,
float(stats.getValue('top1_correct')) /
(stats.getValue('top1_correct') +
stats.getValue('top1_incorrect')))))
log.r(('TOP 1 MEAN CONFIDENCE:',
max(0, np.mean(stats.getValue('top1_confidence')))))
log.r(('TIME PER BATCH:',
int(np.mean(stats.getValue('time_per_batch')) * 1000), 'ms'))
return np.mean(stats.getValue('lrap')), int(
np.mean(stats.getValue('time_per_file')) * 1000)
help="Target script name",
action="store",
default=None)
parser.add_argument("-c",
"--target-count",
help="Target instance count",
action="store",
type=int,
default=1)
parser.add_argument("--shell",
help="Run command as shell mode",
action="store_true",
default=False)
parser.add_argument("--socket",
help="Unix domain socket path",
action="store",
default=None)
args = parser.parse_args()
log.init(args.name)
log.i(__name__, "Start QB Incubator")
if args.script is None and args.target is None:
log.e(__name__, "--script and --target are not set.")
exit(1)
if args.script is not None:
script_main(args.shell, args.script, args.socket)
else:
target_main(args.shell, args.target, args.target_count, args.socket)
def __exit__(self, exc_type, exc_val, exc_tb):
try:
self.session.rollback()
except:
log.e('session rollback error')
self.session.close()