def get_model_path(system_name, logging=True): """ Parameters ---------- args_name : list of string list of name for parsed argument, taken into account for creating model name Return ------ exp_dir, model_path, log_path """ args_name = [] if system_name == 'xvec': args_name += ['utt', 'seq'] elif system_name == 'ivec': args_name += ['nmix', 'tdim'] else: raise ValueError("No support for system with name: %s" % system_name) args_name += ['mindur', 'minutt'] # ====== base system and feature identity ====== # name = str(system_name).lower() name += '_' + FEATURE_RECIPE.replace('_', '') name += '.' + FEATURE_NAME # ====== concat the attributes ====== # attributes = [] for i in [str(i) for i in args_name]: attributes.append(str(getattr(_args, i))) attributes = '_'.join(attributes) name += '.' + attributes # ====== check the exclude dataset ====== # excluded_dataset = str(_args.exclude).strip() if len(excluded_dataset) > 0: dataset_str = [] for excluded in sorted(set(excluded_dataset.split(','))): assert excluded in sre_file_list or excluded == 'noise', \ "Unknown excluded dataset with name: '%s'" % excluded dataset_str.append(excluded) dataset_str = '_'.join(dataset_str) name += '.' + dataset_str # ====== check save_path ====== # save_path = os.path.join(EXP_DIR, name) if os.path.exists(save_path) and IS_OVERRIDE: print("Override path:", ctext(save_path, 'yellow')) shutil.rmtree(save_path) if not os.path.exists(save_path): os.mkdir(save_path) # ====== return path ====== # log_path = get_logpath(name='log.txt', increasing=True, odin_base=False, root=save_path) model_path = os.path.join(save_path, 'model.ai') if bool(logging): print("Model path:", ctext(model_path, 'cyan')) print("Log path:", ctext(log_path, 'cyan')) return save_path, model_path, log_path
def get_model_path(system_name, logging=True): """ Parameters ---------- args_name : list of string list of name for parsed argument, taken into account for creating model name Return ------ exp_dir, model_path, log_path """ args_name = [] if system_name == 'xvec': args_name += ['utt', 'seq'] elif system_name == 'ivec': args_name += ['nmix', 'tdim'] else: raise ValueError("No support for system with name: %s" % system_name) args_name += ['mindur', 'minutt'] # ====== base system and feature identity ====== # name = str(system_name).lower() name += '_' + FEATURE_RECIPE.replace('_', '') name += '.' + FEATURE_NAME # ====== concat the attributes ====== # attributes = [] for i in [str(i) for i in args_name]: attributes.append(str(getattr(_args, i))) attributes = '_'.join(attributes) name += '.' + attributes # ====== check the exclude dataset ====== # excluded_dataset = str(_args.exclude).strip() if len(excluded_dataset) > 0: dataset_str = [] for excluded in sorted(set(excluded_dataset.split(','))): assert excluded in sre_file_list or excluded == 'noise', \ "Unknown excluded dataset with name: '%s'" % excluded dataset_str.append(excluded) dataset_str = '_'.join(dataset_str) name += '.' + dataset_str # ====== check save_path ====== # save_path = os.path.join(EXP_DIR, name) if os.path.exists(save_path) and IS_OVERRIDE: print("Override path:", ctext(save_path, 'yellow')) shutil.rmtree(save_path) if not os.path.exists(save_path): os.mkdir(save_path) # ====== return path ====== # log_path = get_logpath(name='log.txt', increasing=True, odin_base=False, root=save_path) model_path = os.path.join(save_path, 'model.ai') if bool(logging): print("Model path:", ctext(model_path, 'cyan')) print("Log path:", ctext(log_path, 'cyan')) return save_path, model_path, log_path
# pp.validate_features(processor, # nb_samples=12, # path=os.path.join(EXP_DIR, 'feature_validation'), # override=True) ds = F.Dataset(PATH_ACOUSTIC_FEATURES, read_only=True) print(ds) indices = list(ds['indices_%s' % args.feat].items()) print("Utterances length:") print(" ", describe([end - start for name, (start, end) in indices], shorten=True)) # =========================================================================== # Basic path for GMM, T-matrix and I-vector # =========================================================================== EXP_DIR = os.path.join(EXP_DIR, '%s_%d_%d' % (FEAT, NMIX, TV_DIM)) LOG_PATH = get_logpath(name='log.txt', override=False, root=EXP_DIR, odin_base=False) stdio(LOG_PATH) print("Exp-dir:", ctext(EXP_DIR, 'cyan')) print("Log path:", ctext(LOG_PATH, 'cyan')) # ====== ivec path ====== # GMM_PATH = os.path.join(EXP_DIR, 'gmm') TMAT_PATH = os.path.join(EXP_DIR, 'tmat') # zero order statistics Z_PATH = (os.path.join(EXP_DIR, 'Z_train'), os.path.join(EXP_DIR, 'Z_test')) # first order statistics F_PATH = (os.path.join(EXP_DIR, 'F_train'), os.path.join(EXP_DIR, 'F_test')) # i-vector path I_PATH = (os.path.join(EXP_DIR, 'I_train'), os.path.join(EXP_DIR, 'I_test')) # labels L_PATH = ( # labels
from __future__ import print_function, division, absolute_import import os import cPickle import requests import webbrowser from twython import Twython from odin import utils # =========================================================================== # Constants # =========================================================================== CONSUMER_KEY = os.environ['TWITTER_KEY'] CONSUMER_SECRET = os.environ['TWITTER_SECRET'] SAVE_PATH = utils.get_logpath('twitter.log', override=False) # =========================================================================== # First authentication # =========================================================================== if not os.path.exists(SAVE_PATH): twitter = Twython(app_key=CONSUMER_KEY, app_secret=CONSUMER_SECRET) user_timeline = twitter.get_user_timeline(screen_name="NgoTrongTrung") auth = twitter.get_authentication_tokens() OAUTH_TOKEN = auth['oauth_token'] OAUTH_TOKEN_SECRET = auth['oauth_token_secret'] # ====== Getting the PIN using verifier URL ====== # webbrowser.open(auth['auth_url']) PIN = raw_input('We will open browser, copy the PIN code here:') if len(PIN) != 7: raise ValueError('PIN must be 7 numbers.')
#!/usr/bin/env python from __future__ import print_function, division, absolute_import import numpy as np from odin.utils import get_modelpath, ArgController, stdio, get_logpath stdio(get_logpath('tmp.log', override=True)) arg = ArgController(version=0.12).add( '-backend', 'theano or tensorflow', 'tensorflow').add('-ds', 'dataset cifar10, or mnist', 'mnist').add('-epoch', 'number of epoch', 3).add('-lr', 'learning rate', 0.01).parse() import os os.environ['ODIN'] = 'float32,gpu,%s,seed=12' % arg['backend'] from odin import backend as K from odin import nnet as N from odin import fuel, training from six.moves import cPickle # =========================================================================== # Load data # =========================================================================== USE_MNIST_DATA = True if 'mnist' in arg['ds'].lower() else False if USE_MNIST_DATA: ds = fuel.load_mnist()
def config_path(self): return os.path.join(get_logpath(), '%s%s.cfg' % (self.__class__.__name__, self._id))
def _transform(self, X): # ====== file input file ====== # raw = None path = None if isinstance(X, Mapping): if 'path' in X: path = X['path'] if 'sr' in X: if self.sr is None: self.sr = X['sr'] self._update_config() self._first_config_generated = True elif self.sr != X['sr']: raise ValueError("Given sample rate: %d, but the audio file has " "sample rate: %d" % (self.sr, X['sr'])) if 'raw' in X: raw = X['raw'] elif is_string(X): path = X elif isinstance(X, np.ndarray): raw = X else: raise ValueError("openSMILE extractor require path to audio file.") # no sample rate specified, cannot generate appropriate config if self.sr is None: raise RuntimeError("Cannot acquire sample rate for the input.") # ====== first time generate config ====== # if not self._first_config_generated: self._first_config_generated = True self._update_config() # ====== extract SAD ====== # unique_id = os.getpid() + random.randint(0, 10e8) inpath = os.path.join( get_logpath(), '%s%d.wav' % (self.__class__.__name__, unique_id)) outpath = os.path.join( get_logpath(), '%s%d.csv' % (self.__class__.__name__, unique_id)) try: if path is None or not os.path.exists(path): if raw is None: raise RuntimeError("openSMILE require input audio file, since " "we cannot find any audio file, it is required to provide " "raw array and sample rate, so the audio file will be cached.") from soundfile import write write(inpath, data=raw, samplerate=self.sr) path = inpath # if in debug mode or not command = 'SMILExtract -loglevel %d -C %s -I %s -O %s' % \ (self._log_level, self.config_path, path, outpath) os.system(command) results = np.genfromtxt(outpath, dtype='float32', delimiter=',', skip_header=0) except Exception as e: import traceback; traceback.print_exc() raise e finally: if os.path.exists(inpath): os.remove(inpath) if os.path.exists(outpath): os.remove(outpath) # ====== post-processing ====== # X_update = self._post_processing(results) if not isinstance(X_update, dict): raise ValueError("_post_processing must return a dictionary.") return X_update
# => Gaussian normalized is better, and float16 is no different from float32 # =========================================================================== from __future__ import print_function, division, absolute_import import numpy as np import os os.environ['ODIN'] = 'float32,gpu,theano,seed=12,cnmem=0.4' from odin import backend as K from odin import nnet as N from odin import fuel, training from odin.utils import get_modelpath, ArgController, stdio, get_logpath from six.moves import cPickle stdio(get_logpath('tmp.log')) # =========================================================================== # Load data # =========================================================================== ds = fuel.load_cifar10() print(ds) X_train = K.placeholder(shape=(None,) + ds['X_train'].shape[1:], name='X_train') X_score = K.placeholder(shape=(None,) + ds['X_train'].shape[1:], name='X_score') y = K.placeholder(shape=(None,), name='y', dtype='int32') # =========================================================================== # Build network # =========================================================================== ops = N.Sequence([
from __future__ import print_function, division, absolute_import import os import requests import webbrowser from twython import Twython from odin import utils CONSUMER_KEY = os.environ['TWITTER_KEY'] CONSUMER_SECRET = os.environ['TWITTER_SECRET'] SAVE_PATH = utils.get_logpath('twitter') # print(auth) twitter = Twython(app_key=CONSUMER_KEY, app_secret=CONSUMER_SECRET, oauth_version=2) ACCESS_TOKEN = twitter.obtain_access_token() twitter = Twython(CONSUMER_KEY, access_token=ACCESS_TOKEN) search_results = twitter.search(q='WebsDotCom', count=50) print(search_results)
from odin import preprocessing as pp from odin import fuel as F, nnet as N, backend as K from odin.utils import (get_module_from_path, get_script_path, ctext, Progbar, stdio, get_logpath, get_formatted_datetime) from odin.stats import describe from helpers import (SCORING_DATASETS, BACKEND_DATASETS, SCORE_SYSTEM_NAME, SCORE_SYSTEM_ID, N_PLDA, N_LDA, PLDA_MAXIMUM_LIKELIHOOD, PLDA_SHOW_LLK, PATH_ACOUSTIC_FEATURES, FEATURE_RECIPE, FEATURE_NAME, get_model_path, NCPU, get_logpath, prepare_dnn_feeder_recipe, sre_file_list, Config, EXP_DIR, VECTORS_DIR, RESULT_DIR, filter_utterances) # ====== scoring log ====== # stdio( get_logpath(name='make_score.log', increasing=True, odin_base=False, root=EXP_DIR)) print('=' * 48) print(get_formatted_datetime(only_number=False)) print("System name :", SCORE_SYSTEM_NAME) print("System id :", SCORE_SYSTEM_ID) print("Feature recipe :", FEATURE_RECIPE) print("Feature name :", FEATURE_NAME) print("Backend dataset:", ','.join(BACKEND_DATASETS.keys())) print("Scoring dataset:", ','.join(SCORING_DATASETS.keys())) print('=' * 48) # =========================================================================== # Some helper # ===========================================================================
# =========================================================================== from __future__ import print_function, division, absolute_import import numpy as np import os os.environ['ODIN'] = 'float32,gpu,theano,seed=12,cnmem=0.4' from odin import backend as K from odin import nnet as N from odin import fuel, training from odin.utils import get_modelpath, ArgController, stdio, get_logpath from six.moves import cPickle stdio(get_logpath('tmp.log')) # =========================================================================== # Load data # =========================================================================== ds = fuel.load_cifar10() print(ds) X_train = K.placeholder(shape=(None, ) + ds['X_train'].shape[1:], name='X_train') X_score = K.placeholder(shape=(None, ) + ds['X_train'].shape[1:], name='X_score') y = K.placeholder(shape=(None, ), name='y', dtype='int32') # =========================================================================== # Build network
from sklearn.metrics import accuracy_score, log_loss, f1_score from odin import fuel as F from odin import nnet as N, backend as K from odin import visual as V from odin.utils import (ctext, mpi, Progbar, catch_warnings_ignore, stdio, get_logpath, catch_warnings_ignore) from helpers import (FEATURE_RECIPE, FEATURE_NAME, PATH_ACOUSTIC_FEATURES, MINIMUM_UTT_DURATION, ANALYSIS_DIR, Config, filter_utterances, prepare_dnn_data) # ====== prepare log ====== # stdio(get_logpath(name="analyze_data.log", increasing=True, odin_base=False, root=ANALYSIS_DIR)) print(ctext(FEATURE_RECIPE, 'lightyellow')) print(ctext(FEATURE_NAME, 'lightyellow')) assert os.path.isdir(os.path.join(PATH_ACOUSTIC_FEATURES, FEATURE_RECIPE)) # ====== essential path ====== # figure_path = os.path.join(ANALYSIS_DIR, '%s_%s.pdf' % (FEATURE_RECIPE.replace('_', ''), FEATURE_NAME)) print(ctext(figure_path, 'lightyellow')) # =========================================================================== # Load the data # =========================================================================== ds = F.Dataset(os.path.join(PATH_ACOUSTIC_FEATURES, FEATURE_RECIPE), read_only=True) X = ds[FEATURE_NAME] # remove all noise data indices = {name: (start, end)
# ====== import ====== # import os os.environ['ODIN'] = 'float32,%s,%s' % (args['dev'], args['bk']) import numpy as np np.random.seed(1208) from odin import nnet as N, backend as K, fuel as F, stats from odin.utils import get_modelpath, stdio, get_logpath, get_datasetpath from odin.basic import has_roles, BIAS, WEIGHT from odin import training # set log path stdio(path=get_logpath('digit_audio.log', override=True)) # =========================================================================== # Get wav and process new dataset configuration # =========================================================================== # ====== process new features ====== # if False: datapath = F.load_digit_wav() output_path = get_datasetpath(name='digit', override=True) feat = F.SpeechProcessor(datapath, output_path, audio_ext='wav', sr_new=8000, win=0.025, shift=0.01, nb_melfilters=40,
def cache_path(self): return utils.get_logpath(self.__class__.__name__, override=False)
args = ArgController( ).add('-model', 'model name, specified in models_cifar.py', 'cnn' ).parse() import os os.environ['ODIN'] = 'float32,gpu,seed=87654321,log' import numpy as np import tensorflow as tf from odin import fuel as F, nnet as N, backend as K, training, utils from odin.stats import train_valid_test_split MODEL_NAME = args.model MODEL_PATH = utils.get_modelpath(name='cifar10_%s' % MODEL_NAME, override=True) LOG_PATH = utils.get_logpath(name='cifar10_%s.log' % MODEL_NAME, override=True) stdio(LOG_PATH) # =========================================================================== # Some handmade constants # =========================================================================== NB_EPOCH = 10 LEARNING_RATE = 0.001 # =========================================================================== # Load dataset # =========================================================================== ds = F.CIFAR10.get_dataset() nb_labels = 10 print(ds) X_train = ds['X_train'][:].astype('float32') / 255. y_train = one_hot(ds['y_train'][:], nb_classes=nb_labels) X_test = ds['X_test'][:].astype('float32') / 255.
from odin import training from odin import preprocessing as pp from odin.ml import evaluate, fast_tsne from odin.visual import (print_dist, print_confusion, print_hist, plot_scatter, plot_figure, plot_spectrogram, plot_save, plot_confusion_matrix, generate_random_colors, generate_random_marker) from odin.utils import (get_logpath, get_modelpath, get_datasetpath, get_figpath, Progbar, unique_labels, chain, as_tuple_of_shape, stdio, ctext, ArgController) # =========================================================================== # Const # =========================================================================== FEAT = ['mspec', 'sad'] MODEL_PATH = get_modelpath(name='DIGITS', override=True) LOG_PATH = get_logpath(name='digits.log', override=True) FIG_PATH = get_figpath(name='DIGITS', override=True) stdio(LOG_PATH) DEBUG = False # ====== trainign ====== # BATCH_SIZE = 32 NB_EPOCH = 20 NB_SAMPLES = 8 VALID_PERCENTAGE = 0.4 # =========================================================================== # Load dataset # =========================================================================== path = get_datasetpath(name='TIDIGITS_feats', override=False) assert os.path.isdir(path), \ "Cannot find preprocessed feature at: %s, try to run 'odin/examples/features.py'" % path
from sklearn.metrics import accuracy_score, log_loss, f1_score from odin import fuel as F from odin import nnet as N, backend as K from odin import visual as V from odin.utils import (ctext, mpi, Progbar, catch_warnings_ignore, stdio, get_logpath, catch_warnings_ignore) from helpers import (FEATURE_RECIPE, FEATURE_NAME, PATH_ACOUSTIC_FEATURES, MINIMUM_UTT_DURATION, ANALYSIS_DIR, Config, filter_utterances, prepare_dnn_data) # ====== prepare log ====== # stdio( get_logpath(name="analyze_data.log", increasing=True, odin_base=False, root=ANALYSIS_DIR)) print(ctext(FEATURE_RECIPE, 'lightyellow')) print(ctext(FEATURE_NAME, 'lightyellow')) assert os.path.isdir(os.path.join(PATH_ACOUSTIC_FEATURES, FEATURE_RECIPE)) # ====== essential path ====== # figure_path = os.path.join( ANALYSIS_DIR, '%s_%s.pdf' % (FEATURE_RECIPE.replace('_', ''), FEATURE_NAME)) print(ctext(figure_path, 'lightyellow')) # =========================================================================== # Load the data # =========================================================================== ds = F.Dataset(os.path.join(PATH_ACOUSTIC_FEATURES, FEATURE_RECIPE), read_only=True) X = ds[FEATURE_NAME]
stop_on_failure=True) processor.run() # pp.validate_features(processor, # nb_samples=12, # path=os.path.join(EXP_DIR, 'feature_validation'), # override=True) ds = F.Dataset(PATH_ACOUSTIC_FEATURES, read_only=True) print(ds) indices = list(ds['indices_%s' % args.feat].items()) print("Utterances length:") print(" ", describe([end - start for name, (start, end) in indices], shorten=True)) # =========================================================================== # Basic path for GMM, T-matrix and I-vector # =========================================================================== EXP_DIR = os.path.join(EXP_DIR, '%s_%d_%d' % (FEAT, NMIX, TV_DIM)) LOG_PATH = get_logpath(name='log.txt', override=False, root=EXP_DIR, odin_base=False) stdio(LOG_PATH) print("Exp-dir:", ctext(EXP_DIR, 'cyan')) print("Log path:", ctext(LOG_PATH, 'cyan')) # ====== ivec path ====== # GMM_PATH = os.path.join(EXP_DIR, 'gmm') TMAT_PATH = os.path.join(EXP_DIR, 'tmat') # zero order statistics Z_PATH = ( os.path.join(EXP_DIR, 'Z_train'), os.path.join(EXP_DIR, 'Z_test')) # first order statistics F_PATH = ( os.path.join(EXP_DIR, 'F_train'), os.path.join(EXP_DIR, 'F_test')) # i-vector path