def __init__(self,
username: str,
password: str,
watch_folder: str,
destination_folder: str,
host: str,
debug: bool = False,
scanning_interval: int = 1,
template: str = "{artist}/{album}/{title}.mp3"):
super().__init__(watch_folder=watch_folder,
target_folder=destination_folder,
debug=debug,
recursive=True,
thread_name="Main thread of owncloud watcher",
scanning_interval=scanning_interval,
template=template)
self._username = username
self._host = host
self._oc = owncloud.Client(host)
_wait = 5
while True:
try:
self._oc.login(user_id=username, password=password)
break
except owncloud.owncloud.HTTPResponseError as e:
logging.error(
f"Got Connection Error while connection to owncloud. Waiting {_wait}min for retry. "
f"HTTP-Error: {e.status_code}")
time.sleep(_wait * 60)
logging.info(
f"Connected to owncloud {host}. Host runs version {self._oc.get_version()}"
)
def evaluate(model, device):
# Turn on evaluation mode which disables dropout.
model.eval()
mix_scp = os.path.join(FLAGS.data_dir, 'tt', 'mix.scp')
s1_scp = os.path.join(FLAGS.data_dir, 'tt', 's1.scp')
s2_scp = os.path.join(FLAGS.data_dir, 'tt', 's2.scp')
dataset = DataReader(mix_scp, s1_scp, s2_scp)
total_num = len(dataset)
save_path = os.path.join(FLAGS.model_dir, 'wav')
if not os.path.exists(save_path):
os.makedirs(save_path)
print('=> Decoding ...')
sys.stdout.flush()
start_time = datetime.datetime.now()
output_spk1 = np.zeros(0)
output_spk2 = np.zeros(0)
index = 0
with torch.no_grad():
for idx, data in enumerate(dataset.read()):
start = datetime.datetime.now()
key = data['key']
mix = data['mix'].to(device)
s1 = data['s1']
s2 = data['s2']
length = mix.size(-1)
output = model(mix, length)
output1 = np.squeeze(output[:, 0, :].cpu().numpy())
output2 = np.squeeze(output[:, 1, :].cpu().numpy())
mix = np.squeeze(mix.cpu().numpy())
s1 = np.squeeze(s1.numpy())
s2 = np.squeeze(s2.numpy())
clean_s1_path = FLAGS.test_wav_dir + '/s1/' + key + '.wav'
s1_clean = np.squeeze(wavread(clean_s1_path)[0])
sys.stdout.flush()
if np.sum(s1 - s1_clean) != 0:
print('[*]:', key, s1, s1_clean)
sys.exit(0)
save_prefix = os.path.join(save_path, key)
output_spk1 = output1 / np.max(np.abs(output1)) * 0.7
output_spk2 = output2 / np.max(np.abs(output2)) * 0.7
wavwrite(output_spk1, SAMPLE_RATE, save_prefix + '_1.wav')
wavwrite(output_spk2, SAMPLE_RATE, save_prefix + '_2.wav')
index += 1
elapsed = (datetime.datetime.now() - start).total_seconds()
logger.info('{:04d}/{:04d} | time = {:.3f} s'.format(
index, total_num, elapsed))
# logger.info('total_length = {} | cur_lenght = {}'.format(
# total_length, output_spk1.size))
# Reset buffer
output_spk1 = np.zeros(0)
output_spk2 = np.zeros(0)
elapsed = (datetime.datetime.now() - start_time).total_seconds()
print('=> Decode done. Total time is {:.2f} mins'.format(elapsed /
60.0))
def load_config(config_path):
config = {}
try:
with open("config.json") as f:
config = json.load(f)
logging.info(
f"Config Loaded. JSON Contents: \n{json.dumps(config, indent=4)}")
except Exception as e:
logging.error(
f"Failed to Load Config, using default config Params. Error: {e}")
return config
def __init__(self, watch_folder: str, target_folder: str, thread_name="Main thread of folder watcher",
recursive: bool = True, debug: bool = False, scanning_interval: int = 1,
template: str = "{artist}/{album}/{title}.mp3"):
super().__init__(daemon=True, name=thread_name, time_out=scanning_interval)
self._watch_folder = watch_folder
self._target_folder = target_folder
self._recursive = recursive
self._debug = debug
template = template.strip("/")
if not template.endswith(".mp3"):
template = f"{template}.mp3"
self._template = template
logging.info(f"Starting watch on \"{watch_folder}\" as {self.__class__.__name__}")
def download_weights(self):
folder = self.config.get("weights_folder", "weights")
# Hardcoded URLs
bin_url = "https://download.01.org/opencv/2020/openvinotoolkit/2020.3/open_model_zoo/models_bin/1/face-detection-0104/FP16/face-detection-0104.bin"
xml_url = "https://download.01.org/opencv/2020/openvinotoolkit/2020.3/open_model_zoo/models_bin/1/face-detection-0104/FP16/face-detection-0104.xml"
# Get Filenames
bin_filename = os.path.basename(bin_url)
xml_filename = os.path.basename(xml_url)
# Generate Path
bin_path = os.path.join(folder, bin_filename)
xml_path = os.path.join(folder, xml_filename)
# Check if weights already exist
bin_exists = os.path.exists(bin_path)
xml_exists = os.path.exists(xml_path)
if not bin_exists or not xml_exists:
# If weights does not exist
try:
bin_success, bin_message, bin_path = download_file(
source_url=bin_url, filename=bin_filename, folder=folder)
xml_success, xml_message, xml_path = download_file(
source_url=xml_url, filename=xml_filename, folder=folder)
except Exception as e:
logging.error(f"Failed to Download Weights. Error: {e}")
logging.info(f"The Program will exit.")
exit(1)
else:
# If weights exists
return (
xml_path,
f"Found existing weights. Skipping download.",
)
# If downloading fails, exit the program
if not bin_success or not xml_success:
logging.error(
f"Failed to Download Weights. Messages: bin_message: {bin_message} | xml_message: {xml_message}"
)
logging.info(f"The Program will exit.")
exit(1)
return (
xml_path,
f"Weights Downloaded Successfully. Messages: bin_message: {bin_message} | xml_message: {xml_message}",
)
def simple_conversion(audiobook: AudioBook,
target_path: str,
output_format: str = "mp3"):
try:
logging.info(f"Converting {audiobook}")
temporary_out_name = f"{target_path}.conv"
ffmpeg_result, (stdout, stderr) = ffmpeg(
input_file=audiobook.get_input_file(),
output_file=temporary_out_name,
quality_level=2,
output_format=output_format,
activation_bytes=audiobook.get_activation_bytes())
logging.info(f"Job for book {audiobook} exited good: {ffmpeg_result}")
if ffmpeg_result:
if audiobook.get_cover() is not None:
logging.debug(
f"Adding cover to {audiobook} -> {audiobook.get_cover()}")
add_cover(temporary_out_name, audiobook.get_cover())
rename(temporary_out_name, target_path)
return target_path
else:
error_name = f"/tmp/error_{uuid.uuid4()}.json"
with open(error_name, "w") as f_err:
if isinstance(stdout, bytes):
stdout = stdout.decode("utf-8")
else:
stdout = str(stdout)
if isinstance(stderr, bytes):
stderr = stderr.decode("utf-8")
else:
stderr = str(stderr)
json.dump({"stdout": stdout, "stderr": stderr}, f_err)
logging.notify("FFMPEG had an error during conversion",
extra={
"notification_title": "FFMPEG-ERROR",
"attach": error_name
})
remove(temporary_out_name)
remove(error_name)
except:
pass
return None
def finalize_file(self, final_file: str, audiobook: AudioBook,
target_folder: str) -> bool:
target_file = self.audiobook_already_exists(
audiobook=audiobook, target_folder=target_folder)
if final_file is None or target_file is None:
return False
self.__mkdir_recursive(os.path.dirname(target_file).split("/"))
logging.info(
f"Starting upload of {audiobook} to \"{target_file}\" on {self._host}"
)
self._oc.put_file(remote_path=target_file,
local_source_file=final_file)
if audiobook.get_cover() is not None:
self._oc.put_file(remote_path=f"{target_file}.jpg",
local_source_file=audiobook.get_cover())
logging.info(
f"Finished upload of {audiobook} to \"{target_file}\" on {self._host}"
)
return True
def one_run(self) -> bool:
all_files = self.get_file_list(current_element=self._watch_folder, recursive=self._recursive)
all_files = [x for x in all_files if
x.lower().endswith(".aax") or x.lower().endswith(".mp3") or x.lower().endswith(".aaxc")]
if len(all_files) <= 0:
return False
work_file = self.prepare_file(selected_file=all_files[0])
audiobook = get_audiobook_by_file(work_file)
if audiobook is None:
self.broken_input(selected_file=all_files[0], work_file=work_file)
logging.notify(f"Audiobook conversion of {os.path.basename(all_files[0])} failed. "
f"Has been awarded 'broken' :(")
return True
if self.audiobook_already_exists(audiobook=audiobook, target_folder=self._target_folder) is None:
logging.info(f"Audiobook {audiobook} already exists. Deleting and carrying on")
self.cleanup(audiobook=audiobook, final_file=None, selected_file=all_files[0])
return True
cover = audiobook.get_cover()
_extra = {"notification_title": "Conversion started"}
if cover is not None:
_extra["attach"] = cover
logging.notify(message=f"Conversion of {audiobook} initiated", extra=_extra)
try:
_ = audiobook.get_activation_bytes(create_if_missing=True)
except TypeError:
pass
converted_file = simple_conversion(audiobook=audiobook, target_path=f"{work_file}.mp3")
logging.debug(message=f"Conversion of {audiobook} finished. Saved to \"{converted_file}\"")
if not self.finalize_file(final_file=converted_file, audiobook=audiobook, target_folder=self._target_folder):
logging.info(f"Finalization of {audiobook} failed")
logging.notify(message=f"Conversion of {audiobook} failed",
extra={"notification_title": "Conversion failed"})
self.stale_input(selected_input=all_files[0])
else:
logging.info(f"Finalization of {audiobook} succeeded. Going to clean up")
logging.success(message=f"Conversion and upload of {audiobook} secceded",
extra={"notification_title": "Conversion successful"})
self.cleanup(audiobook=audiobook, final_file=converted_file, selected_file=all_files[0])
return True
def load_model(
self,
model,
device,
input_size,
output_size,
num_requests,
cpu_extension=None,
plugin=None,
):
"""
Loads a network and an image to the Inference Engine plugin.
:param model: .xml file of pre trained model
:param cpu_extension: extension for the CPU device
:param device: Target device
:param input_size: Number of input layers
:param output_size: Number of output layers
:param num_requests: Index of Infer request value. Limited to device capabilities.
:param plugin: Plugin for specified device
:return: Shape of input layer
"""
model_xml = model
model_bin = os.path.splitext(model_xml)[0] + ".bin"
# Plugin initialization for specified device
# and load extensions library if specified
if not plugin:
log.info("Initializing plugin for {} device...".format(device))
self.plugin = IECore()
# print("========== {} ==========".format(self.plugin.get_config("CPU", "CPU_THREADS_NUM")))
# self.plugin.set_config({"CPU_THREADS_NUM": "4"}, "CPU")
# print("========== {} ==========".format(self.plugin.get_config("CPU", "CPU_THREADS_NUM")))
# print("=="*10)
# print("========== {} ==========".format(self.plugin.get_config("CPU", "CPU_BIND_THREAD")))
else:
self.plugin = plugin
if cpu_extension and "CPU" in device:
self.plugin.add_extension(cpu_extension, "CPU")
# Read IR
log.info("Reading IR...")
self.net = IENetwork(model=model_xml, weights=model_bin)
log.info("Loading IR to the plugin...")
if device == "CPU":
supported_layers = self.plugin.query_network(self.net, "CPU")
not_supported_layers = [
l for l in self.net.layers.keys() if l not in supported_layers
]
if len(not_supported_layers) != 0:
log.error(
"Following layers are not supported by "
"the plugin for specified device {}:\n {}".format(
device, ", ".join(not_supported_layers)
)
)
log.error(
"Please try to specify cpu extensions library path"
" in command line parameters using -l "
"or --cpu_extension command line argument"
)
sys.exit(1)
if num_requests == 0:
# Loads network read from IR to theplugin
self.net_plugin = self.plugin.load_network(
network=self.net, device_name=device
)
else:
self.net_plugin = self.plugin.load_network(
network=self.net, num_requests=num_requests, device_name=device
)
self.input_blob = next(iter(self.net.inputs))
self.out_blob = next(iter(self.net.outputs))
return self.plugin, self.get_input_shape()