def clean_and_test(directory, model_file, classifierType, birds, verbose,
skip_clean, no_sanitize):
if not len(birds):
raise Exception("Must specify at least one folder/category to test!")
start_time = time.clock()
# birds = ['bluejay_all', 'cardinal_all', 'cardinal_call', 'cardinal_song', 'chickadee_all', 'chickadee_call',
# 'chickadee_song', 'crow_all', 'goldfinch_all', 'goldfinch_call', 'goldfinch_song', 'robin_all',
# 'robin_call', 'robin_song', 'sparrow_all', 'sparrow_call',
# 'sparrow_song', 'titmouse_all', 'titmouse_call', 'titmouse_song']
test_dirs = test_params(directory, birds)
try:
if not no_sanitize:
sanatize_filenames(directory, verbose=verbose)
if not skip_clean:
for dir in test_dirs:
rootdir, subdir = os.path.split(dir)
cleaner = noiseCleaner(verbose=verbose)
cleaner.noise_removal_dir(rootdir)
model_dir, model_name = os.path.split(model_file)
if not len(model_dir):
model_dir = os.getcwd()
test_model(test_dirs,
modelName=model_name,
model_dir=model_dir,
classifierType=classifierType,
store_to_mySQL=True,
verbose=verbose)
except Exception:
send_notification("Clean and test process failed.")
raise
else:
send_notification("Clean and test finished successfully.")
print "Total time elapsed: ", time.clock() - start_time, " seconds."
from noise_removal import noiseCleaner import os from sanitize_filenames import sanatize_filenames #directory = os.getcwd() #train_dir = os.path.join(directory,'Testing') directory = '/home/anusha/Share-VBOX/Data' cleaner = noiseCleaner() cleaner.noise_removal_dir(directory)
def clean_and_test(directory,
model_file,
classifierType,
birds,
verbose,
skip_clean,
no_sanitize,
show_graphs=False):
if not len(birds):
raise Exception("Must specify at least one folder/category to test!")
start_time = time.clock()
# birds = ['bluejay_all', 'cardinal_song',
# 'chickadee_song', 'crow_all', 'goldfinch_song', 'robin_song',
# 'sparrow_song', 'titmouse_song']
test_dirs = test_params(directory, birds)
if not os.path.exists(roc_save_dir):
os.mkdir(roc_save_dir)
try:
if not no_sanitize:
sanatize_filenames(directory, verbose=verbose)
if not skip_clean:
for dir in test_dirs:
rootdir, subdir = os.path.split(dir)
cleaner = noiseCleaner(verbose=verbose, num_threads=0)
cleaner.noise_removal_dir(rootdir)
model_dir, model_name = os.path.split(model_file)
print ''
thresholds = [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]
t1 = tester(test_dirs=test_dirs,
model_dir=model_dir,
modelName=model_name,
verbose=verbose,
classifierType=classifierType)
tests = []
for t in thresholds:
tests.append(t1.test_model(t))
num_classes = len(birds)
per_class_fpr = [[] for a in xrange(num_classes)]
per_class_tpr = [[] for a in xrange(num_classes)]
micro_average_fpr = []
micro_average_tpr = []
for v in tests:
micro_average_fpr.append(
mean([1 - v[f].spec for f in xrange(num_classes)]))
micro_average_tpr.append(
mean([v[f].sens for f in xrange(num_classes)]))
for q in xrange(0, num_classes):
per_class_fpr[q].append(1 - v[q].spec)
per_class_tpr[q].append(v[q].sens)
auc_scores = []
for g in xrange(num_classes):
auc_scores.append(
basic_roc_plot(per_class_fpr[g],
per_class_tpr[g],
birds[g],
show_graph=show_graphs,
save_graph=True))
macro_average_auc = mean(auc_scores)
micro_average_auc = basic_roc_plot(micro_average_fpr,
micro_average_tpr,
"Micro-average",
show_graph=show_graphs,
save_graph=True,
filename=model_file)
plt.clf()
print "AUC for %s is %s" % ("Macro-average", macro_average_auc)
except Exception:
raise
else:
print "Total time elapsed: ", time.clock() - start_time, " seconds."
return [micro_average_fpr, micro_average_tpr, micro_average_auc, tests]
from noise_removal import noiseCleaner rootdir = "/home/zach/Documents/cleaner_test" new_cleaner = noiseCleaner(smoothingWindow=0.1, weight=0.3, sensitivity=0.3, num_threads=1) new_cleaner.noise_removal_dir(rootdir)
def classFile(self, file):
model_file = self.model_file
classifierType = self.classifierType
verbose = self.verbose
added = os.path.getmtime(file)
added = time.gmtime(added)
added = time.strftime(
'\'' + '-'.join(['%Y', '%m', '%d']) + ' ' +
':'.join(['%H', '%M', '%S']) + '\'', added)
cleaner = noiseCleaner(verbose=verbose)
clean_wav = cleaner.noise_removal(file)
Result, P, classNames = aT.fileClassification(clean_wav, model_file,
classifierType)
if verbose:
print file
print Result
print classNames
print P, '\n'
result_dict = {}
for i in xrange(0, len(classNames)):
result_dict[classNames[i]] = P[i]
result_dict = sorted(result_dict.items(),
key=lambda x: x[1],
reverse=True)
with open(file, 'rb') as file_contents:
sample_id = crc32(file_contents.read())
device_id = -1 # tbi
latitude = -1 # tbi
longitute = -1 # tbi
file_metadata = MediaInfo.parse(file)
file_metadata = file_metadata.tracks[0]
assert file_metadata.track_type == 'General'
humidity = file_metadata.humi
temp = file_metadata.temp
if humidity == None:
humidity = -1
else:
humidity = float(humidity)
if temp == None:
temp = -1
else:
temp = float(temp)
light = -1 # tbi
type1 = '\'' + result_dict[0][0] + '\''
type2 = '\'' + result_dict[1][0] + '\''
type3 = '\'' + result_dict[2][0] + '\''
per1 = result_dict[0][1]
per2 = result_dict[1][1]
per3 = result_dict[2][1]
values = [
sample_id, device_id, added, latitude, longitute, humidity, temp,
light, type1, per1, type2, per2, type3, per3
]
values = [str(x) for x in values]
with MySQLdb.connect(
host=host, user=user, passwd=passwd,
db=database) as cur: # config is in config.py: see above
query_text = "INSERT INTO sampleInfo (sampleid, deviceid, added, latitude, longitude, humidity, temp, light, type1, per1, type2, per2, type3, per3) values(" + ','.join(
values) + ");"
try:
cur.execute(query_text)
except _mysql_exceptions.ProgrammingError, e:
if e[0] != 1146:
raise
else:
tbl_create()
cur.execute(query_text)
except _mysql_exceptions.IntegrityError, e:
if e[0] != 1062:
raise
else:
sys.stderr.write("Warning: Duplicate key entry.\n")