def get_file(dbo, name, path, saveto):
"""
Gets DBFS file contents and saves them to the
filename given. Returns True for success
"""
utils.write_binary_file(saveto, get_string(dbo, name, path))
return True
def put(self, dbfsid, filename, filedata):
""" Stores the file data (clearing the Content column) and returns the URL """
try:
path = "%s/%s" % (DBFS_FILESTORAGE_FOLDER, self.dbo.database)
os.mkdir(path)
except OSError:
pass # Directory already exists - ignore
extension = self._extension_from_filename(filename)
filepath = "%s/%s/%s%s" % (DBFS_FILESTORAGE_FOLDER, self.dbo.database,
dbfsid, extension)
url = "file:%s%s" % (dbfsid, extension)
utils.write_binary_file(filepath, filedata)
os.chmod(filepath, 0o666) # Make the file world read/write
self.dbo.execute("UPDATE dbfs SET URL = ?, Content = '' WHERE ID = ?",
(url, dbfsid))
return url
def get_files(dbo, name, path, saveto):
"""
Gets DBFS files for the pattern given in name (use % like db)
and belonging to path (blank for all paths). saveto is
the folder to save all the files to. Returns True for success
"""
if path != "":
rows = dbo.query(
"SELECT ID, URL FROM dbfs WHERE LOWER(Name) LIKE ? AND Path = ?",
[name, path])
else:
rows = dbo.query("SELECT ID, URL FROM dbfs WHERE LOWER(Name) LIKE ?",
[name])
if len(rows) > 0:
for r in rows:
o = DBFSStorage(dbo, r.url)
utils.write_binary_file(saveto, o.get(r.id, r.url))
return True
return False
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--cmp_dir', default='')
parser.add_argument('--output',
default='./splited_cmp/',
type=str,
help='path to output cmp')
parser.add_argument('--model_type', default='')
args = parser.parse_args()
logging.basicConfig(
format='%(asctime)s %(filename)s %(levelname)s %(message)s',
datefmt='%a, %d %b %Y %H:%M:%S',
level=logging.DEBUG,
stream=sys.stdout)
if not os.path.exists(args.output):
os.mkdir(args.output)
cmp_file = os.listdir(args.cmp_dir)
if args.model_type == 'acoustic':
for cmp_filename in cmp_file:
cmp = read_binary_file(os.path.join(args.cmp_dir, cmp_filename),
dimension=hparams['target_channels'],
dtype=np.float64)
sp = np.zeros(cmp.shape)
sp[:, :hparams['spec_units']] = cmp[:, :hparams['spec_units']]
sp[:, -hparams['energy_units']] = cmp[:, -hparams['energy_units']]
lf0 = cmp[:, hparams['spec_units']:hparams['spec_units'] +
hparams['lf0_units']]
uv = cmp[:, hparams['spec_units'] +
hparams['lf0_units']:hparams['spec_units'] +
hparams['lf0_units'] + hparams['uv_units']]
cap = cmp[:, hparams['spec_units'] + hparams['lf0_units'] +
hparams['uv_units']:hparams['cap_units'] +
hparams['spec_units'] + hparams['lf0_units'] +
hparams['uv_units']]
lf0[uv == 0] = 0
write_binary_file(sp,
os.path.join(
args.output,
os.path.splitext(cmp_filename)[0] + '.sp'),
dtype=np.float64)
write_binary_file(sp,
os.path.join(
args.output,
os.path.splitext(cmp_filename)[0] + '.lf0'),
dtype=np.float64)
write_binary_file(sp,
os.path.join(
args.output,
os.path.splitext(cmp_filename)[0] + '.ap'),
dtype=np.float64)
def read_data(args, raw):
label_dir = os.path.join(cur_file, raw, 'prepared_label')
cmp_dir = os.path.join(cur_file, raw, 'prepared_cmp')
if os.path.exists(label_dir) and os.path.exists(cmp_dir):
logger.info('Raw data has been prepared.')
return
if not os.path.exists(label_dir):
os.mkdir(label_dir)
if not os.path.exists(cmp_dir):
os.mkdir(cmp_dir)
label_files = os.listdir(args.label_dir)
cmp_files = os.listdir(args.cmp_dir)
# Do frame alignment
for line in label_files:
filename, _ = os.path.splitext(line.strip())
logger.info('processing ' + filename)
sys.stdout.flush()
label_mat = np.loadtxt(os.path.join(args.label_dir, filename + '.lab'))
if args.model_type == 'acoustic':
cmp_mat = read_binary_file(os.path.join(args.cmp_dir,
filename + ".cmp"),
dimension=hparams['target_channels'],
dtype=np.float64)
elif args.model_type == 'acoustic_mgc':
cmp_mat = read_binary_file(
os.path.join(args.cmp_dir, filename + ".cmp"),
dimension=hparams['mgc_target_channels'],
dtype=np.float32)
if label_mat.shape[0] <= cmp_mat.shape[0]:
cmp_mat = cmp_mat[:label_mat.shape[0], :]
else:
frame_diff = label_mat.shape[0] - cmp_mat.shape[0]
rep = np.repeat(cmp_mat[-1:, :], frame_diff, axis=0)
cmp_mat = np.concatenate([cmp_mat, rep], axis=0)
write_binary_file(label_mat, os.path.join(label_dir,
filename + '.lab'))
if args.model_type == 'acoustic':
write_binary_file(cmp_mat,
os.path.join(cmp_dir, filename + '.cmp'),
dtype=np.float64)
elif args.model_type == 'acoustic_mgc':
write_binary_file(cmp_mat,
os.path.join(cmp_dir, filename + '.cmp'),
dtype=np.float32)
def write_binary_file(self, file_subpath, data):
return write_binary_file(self.to_full_path(file_subpath), data)
def write_binary_file(self, file_subpath, data):
return write_binary_file(self.to_full_path(file_subpath), data)
def decode(args, model, device):
model.eval()
data_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)),
'data_' + args.name)
config_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)),
'config_' + args.name)
data_list = open(os.path.join(config_dir, 'test.lst'), 'r').readlines()
cmvn = np.load(os.path.join(data_dir, "train_cmvn.npz"))
if not os.path.exists(args.output):
os.mkdir(args.output)
if args.model_type == 'acoustic':
for input_name in data_list:
input_name = input_name.split(' ')[0] + '.lab'
logging.info(f'decode {input_name} ...')
input = read_binary_file(os.path.join(
os.path.join(data_dir, 'test', 'label'), input_name),
dimension=hparams['in_channels'])
input = torch.from_numpy(input).to(device)
input = input.unsqueeze(0)
output, uv_output = model(input)
output = output.squeeze()
uv_output = F.softmax(uv_output, dim=-1)[:, :, 0]
uv_output = uv_output.squeeze()
uv = torch.ones(uv_output.shape).to(device)
uv[uv_output > 0.5] = 0.0
uv = uv.unsqueeze(-1)
output = torch.cat((uv, output), -1)
output = output.cpu().squeeze().detach().numpy()
uv = uv.cpu().squeeze().detach().numpy()
output = output * cmvn['stddev_labels'] + cmvn["mean_labels"]
cap = output[:, 1:hparams['cap_units']]
sp = np.concatenate(
(output[:, hparams['cap_units'] + hparams['energy_units'] +
1:hparams['cap_units'] + hparams['energy_units'] +
hparams['spec_units'] + 1],
output[:, hparams['cap_units'] + 1:hparams['cap_units'] +
hparams['energy_units'] + 1]),
axis=-1)
lf0 = output[:, hparams['cap_units'] + hparams['energy_units'] +
hparams['spec_units'] + 1:hparams['cap_units'] +
hparams['energy_units'] + hparams['spec_units'] +
hparams['lf0_units'] + 1]
lf0[uv == 0] = -1.0e+10
write_binary_file(sp,
os.path.join(
args.output,
os.path.splitext(input_name)[0] + '.sp'),
dtype=np.float64)
write_binary_file(lf0,
os.path.join(
args.output,
os.path.splitext(input_name)[0] + '.lf0'),
dtype=np.float32)
write_binary_file(cap,
os.path.join(
args.output,
os.path.splitext(input_name)[0] + '.ap'),
dtype=np.float64)
elif args.model_type == 'acoustic_mgc':
for input_name in data_list:
input_name = input_name.split(' ')[0] + '.lab'
logging.info(f'decode {input_name} ...')
input = read_binary_file(os.path.join(
os.path.join(data_dir, 'test', 'label'), input_name),
dimension=hparams['in_channels'])
input = torch.from_numpy(input).to(device)
input = input.unsqueeze(0)
output, uv_output = model(input)
output = output.squeeze()
uv_output = F.softmax(uv_output, dim=-1)[:, :, 0]
uv_output = uv_output.squeeze()
uv = torch.ones(uv_output.shape).to(device)
uv[uv_output > 0.5] = 0.0
uv = uv.unsqueeze(-1)
output = torch.cat(
(output[:, :hparams['mgc_units']], uv,
output[:, -(hparams['bap_units'] + hparams['lf0_units']):]),
-1)
output = output.cpu().squeeze().detach().numpy()
uv = uv.cpu().squeeze().detach().numpy()
output = output * cmvn['stddev_labels'] + cmvn["mean_labels"]
mgc = output[:, :hparams['mgc_units']]
lf0 = output[:, hparams['mgc_units'] + 1:hparams['mgc_units'] +
hparams['lf0_units'] + 1]
bap = output[:, -(hparams['bap_units']):]
write_binary_file(
mgc,
os.path.join(args.output,
os.path.splitext(input_name)[0] + '.mgc'))
write_binary_file(
lf0,
os.path.join(args.output,
os.path.splitext(input_name)[0] + '.lf0'))
write_binary_file(
bap,
os.path.join(args.output,
os.path.splitext(input_name)[0] + '.bap'))