def clean_and_test(directory, model_file, classifierType, birds, verbose, skip_clean, no_sanitize): if not len(birds): raise Exception("Must specify at least one folder/category to test!") start_time = time.clock() # birds = ['bluejay_all', 'cardinal_all', 'cardinal_call', 'cardinal_song', 'chickadee_all', 'chickadee_call', # 'chickadee_song', 'crow_all', 'goldfinch_all', 'goldfinch_call', 'goldfinch_song', 'robin_all', # 'robin_call', 'robin_song', 'sparrow_all', 'sparrow_call', # 'sparrow_song', 'titmouse_all', 'titmouse_call', 'titmouse_song'] test_dirs = test_params(directory, birds) try: if not no_sanitize: sanatize_filenames(directory, verbose=verbose) if not skip_clean: for dir in test_dirs: rootdir, subdir = os.path.split(dir) cleaner = noiseCleaner(verbose=verbose) cleaner.noise_removal_dir(rootdir) model_dir, model_name = os.path.split(model_file) if not len(model_dir): model_dir = os.getcwd() test_model(test_dirs, modelName=model_name, model_dir=model_dir, classifierType=classifierType, store_to_mySQL=True, verbose=verbose) except Exception: send_notification("Clean and test process failed.") raise else: send_notification("Clean and test finished successfully.") print "Total time elapsed: ", time.clock() - start_time, " seconds."
from noise_removal import noiseCleaner import os from sanitize_filenames import sanatize_filenames #directory = os.getcwd() #train_dir = os.path.join(directory,'Testing') directory = '/home/anusha/Share-VBOX/Data' cleaner = noiseCleaner() cleaner.noise_removal_dir(directory)
def clean_and_test(directory, model_file, classifierType, birds, verbose, skip_clean, no_sanitize, show_graphs=False): if not len(birds): raise Exception("Must specify at least one folder/category to test!") start_time = time.clock() # birds = ['bluejay_all', 'cardinal_song', # 'chickadee_song', 'crow_all', 'goldfinch_song', 'robin_song', # 'sparrow_song', 'titmouse_song'] test_dirs = test_params(directory, birds) if not os.path.exists(roc_save_dir): os.mkdir(roc_save_dir) try: if not no_sanitize: sanatize_filenames(directory, verbose=verbose) if not skip_clean: for dir in test_dirs: rootdir, subdir = os.path.split(dir) cleaner = noiseCleaner(verbose=verbose, num_threads=0) cleaner.noise_removal_dir(rootdir) model_dir, model_name = os.path.split(model_file) print '' thresholds = [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9] t1 = tester(test_dirs=test_dirs, model_dir=model_dir, modelName=model_name, verbose=verbose, classifierType=classifierType) tests = [] for t in thresholds: tests.append(t1.test_model(t)) num_classes = len(birds) per_class_fpr = [[] for a in xrange(num_classes)] per_class_tpr = [[] for a in xrange(num_classes)] micro_average_fpr = [] micro_average_tpr = [] for v in tests: micro_average_fpr.append( mean([1 - v[f].spec for f in xrange(num_classes)])) micro_average_tpr.append( mean([v[f].sens for f in xrange(num_classes)])) for q in xrange(0, num_classes): per_class_fpr[q].append(1 - v[q].spec) per_class_tpr[q].append(v[q].sens) auc_scores = [] for g in xrange(num_classes): auc_scores.append( basic_roc_plot(per_class_fpr[g], per_class_tpr[g], birds[g], show_graph=show_graphs, save_graph=True)) macro_average_auc = mean(auc_scores) micro_average_auc = basic_roc_plot(micro_average_fpr, micro_average_tpr, "Micro-average", show_graph=show_graphs, save_graph=True, filename=model_file) plt.clf() print "AUC for %s is %s" % ("Macro-average", macro_average_auc) except Exception: raise else: print "Total time elapsed: ", time.clock() - start_time, " seconds." return [micro_average_fpr, micro_average_tpr, micro_average_auc, tests]
from noise_removal import noiseCleaner rootdir = "/home/zach/Documents/cleaner_test" new_cleaner = noiseCleaner(smoothingWindow=0.1, weight=0.3, sensitivity=0.3, num_threads=1) new_cleaner.noise_removal_dir(rootdir)
def classFile(self, file): model_file = self.model_file classifierType = self.classifierType verbose = self.verbose added = os.path.getmtime(file) added = time.gmtime(added) added = time.strftime( '\'' + '-'.join(['%Y', '%m', '%d']) + ' ' + ':'.join(['%H', '%M', '%S']) + '\'', added) cleaner = noiseCleaner(verbose=verbose) clean_wav = cleaner.noise_removal(file) Result, P, classNames = aT.fileClassification(clean_wav, model_file, classifierType) if verbose: print file print Result print classNames print P, '\n' result_dict = {} for i in xrange(0, len(classNames)): result_dict[classNames[i]] = P[i] result_dict = sorted(result_dict.items(), key=lambda x: x[1], reverse=True) with open(file, 'rb') as file_contents: sample_id = crc32(file_contents.read()) device_id = -1 # tbi latitude = -1 # tbi longitute = -1 # tbi file_metadata = MediaInfo.parse(file) file_metadata = file_metadata.tracks[0] assert file_metadata.track_type == 'General' humidity = file_metadata.humi temp = file_metadata.temp if humidity == None: humidity = -1 else: humidity = float(humidity) if temp == None: temp = -1 else: temp = float(temp) light = -1 # tbi type1 = '\'' + result_dict[0][0] + '\'' type2 = '\'' + result_dict[1][0] + '\'' type3 = '\'' + result_dict[2][0] + '\'' per1 = result_dict[0][1] per2 = result_dict[1][1] per3 = result_dict[2][1] values = [ sample_id, device_id, added, latitude, longitute, humidity, temp, light, type1, per1, type2, per2, type3, per3 ] values = [str(x) for x in values] with MySQLdb.connect( host=host, user=user, passwd=passwd, db=database) as cur: # config is in config.py: see above query_text = "INSERT INTO sampleInfo (sampleid, deviceid, added, latitude, longitude, humidity, temp, light, type1, per1, type2, per2, type3, per3) values(" + ','.join( values) + ");" try: cur.execute(query_text) except _mysql_exceptions.ProgrammingError, e: if e[0] != 1146: raise else: tbl_create() cur.execute(query_text) except _mysql_exceptions.IntegrityError, e: if e[0] != 1062: raise else: sys.stderr.write("Warning: Duplicate key entry.\n")