def main():
configs = cu.get_config(__file__)
cu.set_main_log(__file__)
logger = lu.get_logger(__file__)
selector = DefaultSelector()
connector = Connector(configs, selector)
logger.info('Starting the proxy')
logger.info('logger name: {}'.format(cu.MAIN_LOG))
Driver(connector.forward())
loop(selector)
def __init__(self, filename, delimiter):
self._logger = log_util.get_logger()
self._logger.info('filename [%s]' % filename)
self._logger.info('delimiter [%s]' % delimiter)
self._dict = dict()
fin = open(filename)
for line in fin:
key, value = line.strip().split(delimiter, 1)
if key not in self._dict:
self._dict[key] = value
fin.close()
def __init__(self, filename, delimiter):
self._logger = log_util.get_logger()
self._logger.info('data_file [%s]' % filename)
self._logger.info('delimiter [%s]' % delimiter)
self.int_id_2_str_item_dict = dict()
self.str_item_2_int_id_dict = dict()
index_id = 0
fin = open(filename)
for line in fin:
fields = line.strip().split(delimiter)
if not fields:
continue
content = fields[0]
if content not in self.str_item_2_int_id_dict:
self.str_item_2_int_id_dict[content] = index_id
self.int_id_2_str_item_dict[index_id] = content
index_id += 1
fin.close()
import re
import sys
from src.frontend import silence_util
from util import file_util, log_util
# from logplot.logging_plotting import LoggerPlotter #, MultipleTimeSeriesPlot, SingleWeightMatrixPlot
logger = log_util.get_logger("label modifier")
class HTSLabelModification(object):
"""This class is to modify HTS format labels with predicted duration.
Time alignments are expected in the HTS labels. Here is an example of the HTS labels:
3050000 3100000 xx~#-p+l=i:1_4/A/0_0_0/B/1-1-4:1-1&1-4#1-3$1-4>0-1<0-1|i/C/1+1+3/D/0_0/E/content+1:1+3&1+2#0+1/F/content_1/G/0_0/H/4=3:1=1&L-L%/I/0_0/J/4+3-1[2]
3100000 3150000 xx~#-p+l=i:1_4/A/0_0_0/B/1-1-4:1-1&1-4#1-3$1-4>0-1<0-1|i/C/1+1+3/D/0_0/E/content+1:1+3&1+2#0+1/F/content_1/G/0_0/H/4=3:1=1&L-L%/I/0_0/J/4+3-1[3]
3150000 3250000 xx~#-p+l=i:1_4/A/0_0_0/B/1-1-4:1-1&1-4#1-3$1-4>0-1<0-1|i/C/1+1+3/D/0_0/E/content+1:1+3&1+2#0+1/F/content_1/G/0_0/H/4=3:1=1&L-L%/I/0_0/J/4+3-1[4]
3250000 3350000 xx~#-p+l=i:1_4/A/0_0_0/B/1-1-4:1-1&1-4#1-3$1-4>0-1<0-1|i/C/1+1+3/D/0_0/E/content+1:1+3&1+2#0+1/F/content_1/G/0_0/H/4=3:1=1&L-L%/I/0_0/J/4+3-1[5]
3350000 3900000 xx~#-p+l=i:1_4/A/0_0_0/B/1-1-4:1-1&1-4#1-3$1-4>0-1<0-1|i/C/1+1+3/D/0_0/E/content+1:1+3&1+2#0+1/F/content_1/G/0_0/H/4=3:1=1&L-L%/I/0_0/J/4+3-1[6]
305000 310000 are the starting and ending time.
[2], [3], [4], [5], [6] mean the HMM state index.
"""
def __init__(self, silence_pattern=['*-#+*'], label_type="state_align"):
self.silence_pattern = silence_pattern
def setUp(self):
self._logger = log_util.get_logger()
self._strategy = CharacterSegmentStrategyV1()
import matchers
from util import log_util
from util.str_util import tojsonstr
_logger = log_util.get_logger(__name__)
def has_matcher_syntax(expected, matcher):
# TODO: not compatible with python 3 - FIX
if isinstance(expected, basestring) and expected.startswith(matcher):
return True
return False
def is_matcher(expected):
for matcher in matchers.matcher_dict.keys():
if has_matcher_syntax(expected, matcher):
return True
return False
def verify_dict(expected, actual, **match_options):
for key in expected.keys():
_logger.debug("Checking '%s'..." % key)
verify(expected.get(key), actual.get(key), **match_options)
_logger.debug("Success.")
# TODO: we should support strict or non-strict types of comparisons of lists
def verify_list(expected, actual, **match_options):
match_subsets = match_options.get("match_subsets", False)
import os
import shutil
import numpy as np
from util import log_util
log = log_util.get_logger("file process")
def create_blank_file(file_name):
'''
create a blank file
:param file_name:
:return:
'''
with open(file_name, 'w') as wt:
wt.write("")
log.debug("blank file %s created.." % file_name)
def read_file_list_from_path(path, file_type=None, if_recursive=False):
'''
get all file list from path
:param path:
:param file_type:
:return:
'''
file_list = []
for file in os.listdir(path):
tmp_file = os.path.join(path, file)
import numpy as np
from util import log_util, file_util
log = log_util.get_logger("statistic function")
'''
### for htk
io_funcs = HTKFeat_read()
io_funcs.getall(in_file_list[i])
htk_writer = HTKFeat_write(veclen=io_funcs.veclen, sampPeriod=io_funcs.sampPeriod, paramKind=9)
htk_writer.writeall(norm_features, out_file_list[i])
## for normal
io_funcs = BinaryIOCollection()
io_funcs.load_binary_file_frame(in_file_list[i], self.feature_dimension)
io_funcs.array_to_binary_file(norm_features, out_file_list[i])
'''
class Statis(object):
def __init__(self, feature_dimension, read_func=None, writer_func=None, min_value=0.01, max_value=0.99,
min_vector=0.0, max_vector=0.0,
exclude_columns=[]):
self.target_min_value = min_value
"""
__Date__ : 2017/7/8
__Author__ : linyu
说明：该脚本用来获取西刺免费代理并且经过百度进行测试其有效性，同时将有效ip存入redis集合中，键为ip_list
"""
from lxml import etree
import requests
from util.connect_redis import get_redis
from util.log_util import get_logger
logger = get_logger('api.' + __file__.split('/')[-1][:-3])
headers = {
'User-Agent':
'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.104 Safari/537.36 Core/1.53.3103.400 QQBrowser/9.6.11372.400'
}
r = get_redis()
def is_valid_ip(is_http, ip, port):
proxies = {is_http: is_http + '://' + ip + ':' + port}
sess = requests.session()
sess.proxies = proxies
try:
r = sess.get('http://weibo.com/login.php', headers=headers)
if r.status_code == 200:
return True
else:
return False
except:
def setUp(self):
self._logger = log_util.get_logger()
self._reader = sequential_encoder.SequentialEncoder('test_data/test_data_for_sequential_encoder', '\t')
def setUp(self):
self._logger = log_util.get_logger()
self._reader = key_value_file_reader.KeyValueFileReader('test_data/test_data_for_key_value_file_reader', '\t')
import multiprocessing as mp
import os
import shutil
import time
import fastdtw
import numpy as np
from util import file_util, system_cmd_util, log_util
log = log_util.get_logger("dtw align")
# path to tools
tools_dir = ""
sptk = os.path.join(tools_dir, "bin/SPTK-3.9")
speech_tools = os.path.join(tools_dir, "speech_tools/bin")
festvox = os.path.join(tools_dir, "festvox")
tools_dir = ""
# Source features directory
src_feat_dir = ""
# Target features directory
tgt_feat_dir = ""
# Source-aligned features directory
src_aligned_feat_dir = ""
# bap dimension
bap_dim = ""
temp_dir = os.path.join(src_aligned_feat_dir, "../temp")
if not os.path.exists(src_aligned_feat_dir):
os.makedirs(src_aligned_feat_dir)
alignments_dir = os.path.join(src_aligned_feat_dir, "../dtw_alignments")
import multiprocessing as mp
import os
import shutil
import sys
import time
import numpy as np
# from tool_packages.magphase import libutils as lu
# from tool_packages.magphase import magphase as mp
from util import file_util, log_util, system_cmd_util
log = log_util.get_logger("extract vocoder features")
fs_nFFT_dict = {16000: 1024,
22050: 1024,
44100: 2048,
48000: 2048}
fs_alpha_dict = {16000: 0.58,
22050: 0.65,
44100: 0.76,
48000: 0.77}
raw_dir = "/home/top/workspace/tts/data/CarNum/raw"
sp_dir = "/home/top/workspace/tts/data/CarNum/sp"
ap_dir = "/home/top/workspace/tts/data/CarNum/ap"
f0_dir = "/home/top/workspace/tts/data/CarNum/f0"
# output feature dir
lf0_dir = "/home/top/workspace/tts/data/CarNum/lf0"
mgc_dir = "/home/top/workspace/tts/data/CarNum/mgc"
bap_dir = "/home/top/workspace/tts/data/CarNum/bap"
import logging
from functools import partial
from urlparse import urljoin
from docopt import docopt
import matchers
from util.str_util import tojsonstr, diff_strings
from skivvy_config import read_config
from util import file_util, http_util, dict_util
from util import log_util
from verify import verify
STATUS_OK = "OK"
STATUS_FAILED = "FAILED"
_logger = log_util.get_logger(__name__, level=logging.DEBUG)
def configure_testcase(test_dict, conf_dict):
testcase = dict(conf_dict)
testcase.update(test_dict)
return testcase
def configure_logging(testcase):
log_level = testcase.get("log_level", None)
if log_level:
_logger.setLevel(log_level)
def override_default_headers(default_headers, more_headers):
import re
from util import log_util
logger = log_util.get_logger("HTK Question")
def load_question_set_continous(qs_file_name):
fid = open(qs_file_name)
binary_qs_index = 0
continuous_qs_index = 0
binary_dict = {}
continuous_dict = {}
LL = re.compile(re.escape('LL-'))
for line in fid.readlines():
line = line.replace('\n', '').replace('\t', ' ')
if len(line) > 5:
temp_list = line.split('{')
temp_line = temp_list[1]
temp_list = temp_line.split('}')
temp_line = temp_list[0]
temp_line = temp_line.strip()
question_list = temp_line.split(',')
temp_list = line.split(' ')
question_key = temp_list[1]
# print line
if temp_list[0] == 'CQS':
assert len(question_list) == 1
processed_question = wildcards2regex(question_list[0], convert_number_pattern=True)
def setUp(self):
self._logger = log_util.get_logger()
import logging
import re
import sys
import matplotlib.mlab as mlab
import numpy
from src.frontend.linguistic_base import LinguisticBase
from src.frontend.question_util import load_question_set_continous
from util import file_util, log_util
# from logplot.logging_plotting import LoggerPlotter
# #, MultipleTimeSeriesPlot, SingleWeightMatrixPlot
logger = log_util.get_logger("label normalisation")
class LabelNormalisation(LinguisticBase):
# this class only knows how to deal with a single style of labels (XML or HTS)
# (to deal with composite labels, use LabelComposer instead)
def __init__(self, question_file_name=None, xpath_file_name=None):
pass
def extract_linguistic_features(self,
in_file_name,
out_file_name=None,
label_type="state_align",
dur_file_name=None):
if label_type == "phone_align":
A = self.load_labels_with_phone_alignment(in_file_name,
dur_file_name)
import os
from util import log_util
log = log_util.get_logger("system command")
def sptk_x2x_xargs(sptk_path, src_file, file_dim, target):
x2x_cmd1 = "%s +fa %s | xargs -n%d > %s" % (sptk_path, src_file, file_dim,
target)
log.debug("execuate system command %s" + x2x_cmd1)
os.system(x2x_cmd1)
def speech_tools_chtrack(speech_tools, src, target):
'''
:param speech_tools:
:param src:
:param target:
:return:
'''
chtrack_cmd1 = "%s -s 0.005 -otype est_binary %s -o %s" % (os.path.join(
speech_tools, "ch_track"), src, target)
log.debug("execuate system command %s" + chtrack_cmd1)
os.system(chtrack_cmd1)
def festvox_phone_align_cmd(festvox_path, src_mgc, taret_mgc, in_lab, out_lab,
dtw_alignment_file):
'''
def __init__(self,
numpy_rng,
theano_rng=None,
n_ins=784,
n_outs=10,
l1_reg=None,
l2_reg=None,
hidden_layers_sizes=[500, 500],
hidden_activation='tanh',
output_activation='linear',
use_rprop=0,
rprop_init_update=0.001):
logger = log_util.get_logger("DNN initialization")
self.sigmoid_layers = []
self.params = []
self.delta_params = []
self.n_layers = len(hidden_layers_sizes)
self.output_activation = output_activation
self.use_rprop = use_rprop
self.rprop_init_update = rprop_init_update
self.l1_reg = l1_reg
self.l2_reg = l2_reg
assert self.n_layers > 0
# allocate symbolic variables for the data
self.x = tf.placeholder(shape=[None, n_ins],
name="x",
dtype=tf.float32)
self.y = tf.placeholder(shape=[None, n_outs],
name="y",
dtype=tf.float32)
for i in range(self.n_layers):
if i == 0:
input_size = n_ins
else:
input_size = hidden_layers_sizes[i - 1]
if i == 0:
layer_input = self.x
else:
layer_input = self.sigmoid_layers[-1].output
sigmoid_layer = HiddenLayer(
rng=numpy_rng,
input=layer_input,
n_in=input_size,
n_out=hidden_layers_sizes[i],
activation=T.tanh) ##T.nnet.sigmoid) #
self.sigmoid_layers.append(sigmoid_layer)
self.params.extend(sigmoid_layer.params)
self.delta_params.extend(sigmoid_layer.delta_params)
# add final layer
if self.output_activation == 'linear':
self.final_layer = LinearLayer(
rng=numpy_rng,
input=self.sigmoid_layers[-1].output,
n_in=hidden_layers_sizes[-1],
n_out=n_outs)
elif self.output_activation == 'sigmoid':
self.final_layer = SigmoidLayer(
rng=numpy_rng,
input=self.sigmoid_layers[-1].output,
n_in=hidden_layers_sizes[-1],
n_out=n_outs,
activation=T.nnet.sigmoid)
else:
logger.critical(
"This output activation function: %s is not supported right now!"
% (self.output_activation))
sys.exit(1)
self.params.extend(self.final_layer.params)
self.delta_params.extend(self.final_layer.delta_params)
### MSE
self.finetune_cost = T.mean(
T.sum((self.final_layer.output - self.y) *
(self.final_layer.output - self.y),
axis=1))
self.errors = T.mean(
T.sum((self.final_layer.output - self.y) *
(self.final_layer.output - self.y),
axis=1))
### L1-norm
if self.l1_reg is not None:
for i in range(self.n_layers):
W = self.params[i * 2]
self.finetune_cost += self.l1_reg * (abs(W).sum())
### L2-norm
if self.l2_reg is not None:
for i in range(self.n_layers):
W = self.params[i * 2]
self.finetune_cost += self.l2_reg * T.sqr(W).sum()