def validateTask(self, task, *, interactive=False):
"""Check if task is properly defined.
Parameters
----------
task: SyncTask
Task to validate.
interactive: bool, optional
For interactive synchronization `out` will not be vaildated.
Raises
------
Error
Invalid task.
KeyError or Exception
Invalid `task`.out.path pattern.
"""
sub, ref, out = task.sub, task.ref, task.out
if sub is None or not sub.path or sub.no is None:
raise Error('subtitles not set', task=task)
if ref is None or not ref.path or ref.no is None:
raise Error('reference file not set', task=task)
if not interactive and (out is None or not out.path):
raise Error('output path not set', task=task)
if not interactive:
out.validateOutputPattern()
def validateAssets(parent, tasks, updateAssets=True, askForLang=True):
for task in tasks:
sub = task.sub
ref = task.ref
if sub == None or sub.path == None or sub.no == None:
raise raiseTaskError(task, _('Subtitles not set'))
if ref == None or ref.path == None or ref.no == None:
raise raiseTaskError(task, _('Reference file not set'))
if sub.path == ref.path and sub.no == ref.no:
raise raiseTaskError(
task, _('Subtitles can\'t be the same as reference'))
if ref.type == 'audio' and not ref.lang:
raise raiseTaskError(task, _('Select reference language first'))
if task.out and task.out.path:
task.out.validateOutputPattern()
if askForLang and settings().showLanguageNotSelectedPopup:
if not askForLangSelection(parent, tasks):
return False
assetListNotReady = assetListUpdater.isRunning()
needed = set()
for task in tasks:
needed |= assetManager.getAssetsForTask(task)
missing = [asset for asset in needed if asset.isMissing()]
if assetListNotReady and missing:
with BusyDlg(self, _('Downloading assets list...')) as dlg:
dlg.ShowModalWhile(assetListUpdater.isRunning)
missing = [asset for asset in needed if asset.isMissing()]
if missing:
msg = [_('Following assets are missing on server:'), '']
msg += [' - ' + asset.getPrettyName() for asset in missing]
raise Error('\n'.join(msg))
nonLocal = [asset for asset in needed if not asset.isLocal()]
if nonLocal:
if not askForDownloadAssets(parent, nonLocal):
return False
if updateAssets:
update = [asset for asset in needed if asset.isUpgradable()]
if update:
askForUpdateAssets(parent, update)
missing = [asset for asset in needed if not asset.isLocal()]
if missing:
msg = [_('Following assets are missing:')]
msg += [' - ' + asset.getPrettyName() for asset in missing]
raise Error('\n'.join(msg))
return True
def validateSelection(self):
subs = self.m_panelSub.stream
refs = self.m_panelRef.stream
if subs.path == None or subs.no == None:
raise Error(_('Subtitles not set'))
if refs.path == None or refs.no == None:
raise Error(_('Reference file not set'))
if subs.path == refs.path and subs.no == refs.no:
raise Error(_('Subtitles can\'t be the same as reference'))
if refs.type == 'audio' and not refs.lang:
raise Error(_('Select reference language first'))
def formatPattern(pattern, formatter):
try:
return pattern.format(
**formatter, **{
'if': ConditionalFormatter(formatter),
'if_not': ConditionalFormatter(formatter, inverted=True)
})
except KeyError as e:
raise Error(_('Invalid output pattern, invalid keyword: {}').format(e),
pattern=pattern)
except Exception as e:
raise Error(_('Invalid output pattern, {}').format(e), pattern=pattern)
def loadDictionary(langKey, langVal, minLen=0):
langKeyInfo = languages.get(code3=langKey)
langValInfo = languages.get(code3=langVal)
minKeyLen = langKeyInfo.ngrams or minLen
minValLen = langValInfo.ngrams or minLen
dictionary = gizmo.Dictionary()
def addEntry(key, val):
if len(key) >= minKeyLen and len(val) >= minValLen:
if langKeyInfo.rightToLeft: key = key[::-1]
if langValInfo.rightToLeft: val = val[::-1]
for k in splitNgrams(key, langKeyInfo.ngrams):
for v in splitNgrams(val, langValInfo.ngrams):
dictionary.add(k.lower(), v)
asset = assets.getAsset('dict', (langKey, langVal))
if asset.localVersion():
for key, val in asset.readDictionary():
addEntry(key, val)
else:
asset = assets.getAsset('dict', (langVal, langKey))
if asset.localVersion():
for key, val in asset.readDictionary():
addEntry(val, key)
if not asset.localVersion():
raise Error(_('There is no dictionary for transaltion from {} to {}') \
.format(langKey, langVal)) \
.add('language1', langKey) \
.add('language2', langVal)
logger.info('dictionary ready with %u entries', dictionary.size())
return dictionary
def hasList(self):
if self.error:
e = self.error[1]
self.error = None
msg = '{}\n{}'.format(_('Communication with server failed'), str(e))
raise Error(msg) from e
return self.isListReady
def createProducerPipeline(stream):
if stream.type == 'subtitle/text':
return SubtitlePipeline(stream)
elif stream.type == 'audio':
return SpeechPipeline(stream)
else:
raise Error(_('Not supported stream type'), type=stream.type)
def openStream(self, file=None, path=None):
if path:
file = readStream(self, self.file, path)
self.file = file
self.m_textPath.SetValue(self.file.path)
self.m_textPath.SetInsertionPoint(self.m_textPath.GetLastPosition())
self.m_choiceEncoding.Enable(False)
self.m_buttonOk.Enable(False)
lang = validateLang(file.lang)
enc = file.enc
channels = file.channels
self.m_listStreams.setStreams(file.streams, file.types)
if self.m_listStreams.GetItemCount() == 0:
raise Error(_('There are no usable streams'),
path=file.path,
types=self.file.types)
if file.no is not None:
self.selectStream(file.stream())
if not lang:
lang = self.defaultLang
file.lang = lang
self.m_choiceLang.SetValue(lang)
self.m_choiceEncoding.SetValue(enc)
if channels:
self.selectAudioChannels(channels)
def detectEncoding(path, lang, probeSize=32 * 1024):
dlang, denc = locale.getdefaultlocale()
if not lang:
lang2 = dlang.split('_', 1)[0]
lang = languages2to3.get(lang2)
encs = ['UTF-8'] + languages.get(lang, (None, []))[1]
if denc not in encs:
encs.append(denc)
try:
for enc in encs:
with open(path, 'r', encoding=enc) as fp:
try:
fp.read(32 * 1024)
logger.info('detected encoding %s for file "%s"', enc,
path)
return enc
except UnicodeError:
pass
except FileNotFoundError:
raise Error('File not found').add('path', path)
logger.info('couldn\'t detect encoding for file "%s", tried %s', path,
encs)
def openStream(self, stream=None, path=None):
if path:
return readStream(self, path, self.stream.types)
self.stream = stream
self.m_textPath.SetValue(self.stream.path)
self.m_textPath.SetInsertionPoint(self.m_textPath.GetLastPosition())
self.m_choiceEncoding.Enable(False)
self.m_buttonOk.Enable(False)
if not stream.lang and len(stream.streams) == 1:
stream.lang = validateLang(getLangFromPath(stream.path))
lang = validateLang(stream.lang)
enc = stream.enc
channels = stream.channels
self.m_listStreams.setStreams(stream.streams, stream.types)
if self.m_listStreams.GetItemCount() == 0:
raise Error(_('There are no usable streams'),
path=stream.path,
types=self.stream.types)
if stream.no != None:
self.selectStream(stream.stream())
stream.lang = lang
self.m_choiceLang.SetValue(lang)
self.m_choiceEncoding.SetValue(enc)
if channels:
self.selectAudioChannels(channels)
def detectEncoding(path, lang, probeSize=32 * 1024):
try:
dlang, denc = locale.getdefaultlocale()
except Exception as e:
logger.warn('getdefaultlocale failed, %r', e)
dlang, denc = None, None
if not lang and dlang:
lang = dlang.split('_', 1)[0]
encs = ['UTF-8'] + languages.get(lang).encodings
if denc and denc not in encs:
encs.append(denc)
try:
for enc in encs:
with open(path, 'r', encoding=enc) as fp:
try:
fp.read(32 * 1024)
logger.info('detected encoding %s for file "%s"', enc,
path)
return enc
except UnicodeError:
pass
except FileNotFoundError:
raise Error('File not found').add('path', path)
logger.info('couldn\'t detect encoding for file "%s", tried %s', path,
encs)
def __init__(self, parent):
title = _('Application upgrade')
asset = assetManager.getSelfUpdaterAsset()
updater = asset and asset.getUpdater()
if not updater:
raise Error('Application upgrade is not available')
super().__init__(parent, title, updater)
def checkResponseCode(url, response):
if response.status < 200 or response.status >= 300:
code = response.status
reason = response.reason
raise Error(_('Got response {}: {}').format(code, reason),
code=code,
reason=reason,
url=url)
async def verify(self, hash):
logger.info('downloading signature')
sig = await async_utils.downloadRaw(self.asset.getRemote('sig'))
logger.info('verifying signature')
if not pubkey.getVerifier().verify(hash, sig):
raise Error(_('Signature verification failed'),
url=self.asset.getRemote('url'))
logger.info('signature is valid')
def load(self):
try:
if os.path.isfile(config.configpath):
with open(config.configpath, encoding='utf8') as fp:
cfg = json.load(fp)
logger.info('configuration loaded from %s',
config.configpath)
logger.debug('configuration: %r', cfg)
self.set(**cfg)
except Exception as err:
raise Error(_('Cannot load settings file, {}').format(err),
path=config.configpath)
def installUpdate(self):
try:
instPath = os.path.join(self.localDir, self.getLocal('install'))
logger.info('executing installer %s', instPath)
mode = os.stat(instPath).st_mode
if (mode & stat.S_IEXEC) == 0:
os.chmod(instPath, mode | stat.S_IEXEC)
subprocess.Popen(instPath, cwd=self.localDir)
except Exception as e:
logger.error('cannot install update %s: %r', self.path, e, exc_info=True)
raise Error(_('Update instalation failed miserably'))
def _run(self, timeout):
try:
remote = self._asset._getRemoteData()
url = remote.get('url')
for key in ['url', 'sig', 'type']:
if not isinstance(remote.get(key), str):
logger.warning('invalid asset remote data %r', remote)
return
with tempfile.TemporaryFile() as fp:
hash = self._download(fp, url, remote.get('size'), timeout)
if not self._terminated:
try:
self._verify(fp, remote.get('sig'), hash)
except:
raise Error(_('Signature verification failed'),
asset=self._asset.getId(),
url=url)
if not self._terminated:
try:
self._asset._removeLocalData()
self._install(fp, remote.get('type'))
except Exception:
self._asset._removeLocalData()
raise Error(_('Asset installation failed'),
asset=self._asset.getId(),
url=url)
except:
e = sys.exc_info()
self._exception = e
logger.error('updater failed', exc_info=True)
finally:
with self._lock:
for onEnd in self._onEnd:
onEnd(self._asset, self._terminated, self._exception)
def __init__(self, parent, asset):
super().__init__(parent)
self.m_textName.SetLabel(asset.getPrettyName())
self.lastPos = 0
self.startTime = time.monotonic()
self.downloader = asset.downloader()
if not self.downloader:
raise Error(_('Update not available'))
self.downloader.registerCallbacks(self)
self.downloader.run(timeout=0.5)
def raiseTaskError(task, msg):
msgs = [msg, '']
if task.sub and task.sub.path:
msgs.append(_('subtitles: ') + task.sub.path)
if task.ref and task.ref.path:
msgs.append(_('references: ') + task.ref.path)
if task.out and task.out.path:
msgs.append(_('output: ') + task.out.path)
raise Error('\n'.join(msgs)) \
.addn('sub', task.sub) \
.addn('ref', task.ref) \
.addn('out', task.out)
def __init__(self, asset):
self.asset = asset
self.lock = threading.Lock()
self.done = False
self.progress = 0
self.error = None
super().__init__(self.job, name='Download')
for key in ['type', 'url', 'sig']:
if key not in asset.getRemote():
raise Error('Invalid asset data, missing parameter', key=key)
async def install(self, fp):
assetType = self.asset.getRemote('type')
if assetType == 'zip':
dstdir = config.assetdir
logger.info('extracting zip asset to %s', dstdir)
os.makedirs(dstdir, exist_ok=True)
zipf = zipfile.ZipFile(fp)
zipf.extractall(dstdir)
logger.info('extraction completed')
else:
raise Error('Invalid asset type',
type=assetType,
url=self.asset.getRemote('url'))
def selectBy(self, type=None, lang=None):
"""Select stream by type and language (or only one of them)."""
for s in self.streams.values():
if self.types and s.type not in self.types:
continue
if type and not s.type.startswith(type):
continue
if lang and lang != s.lang.lower():
continue
return self.select(s.no)
raise Error(_('There is no matching stream in {}').format(self.path)) \
.addn('path', self.path) \
.addn('type', type) \
.addn('language', lang)
def _install(self, fp, type):
with self._asset._lock:
self._asset._local = None
if type == 'zip':
dstdir = config.assetdir
logger.info('extracting zip asset to %s', dstdir)
os.makedirs(dstdir, exist_ok=True)
zipf = zipfile.ZipFile(fp)
zipf.extractall(dstdir)
logger.info('extraction completed')
else:
raise Error('Invalid asset type',
asset=self._asset.getId(),
type=type,
url=self._asset._getRemoteData().get('url'))
def install(self):
"""Run local installer.
Application must be terminated immediately to let installer work.
"""
with self._lock:
try:
self.installerVersion()
instPath = os.path.join(self.localDir, self._getLocalData().get('install'))
logger.info('executing installer %s', instPath)
mode = os.stat(instPath).st_mode
if (mode & stat.S_IEXEC) == 0:
os.chmod(instPath, mode | stat.S_IEXEC)
subprocess.Popen(instPath, cwd=self.localDir)
except:
logger.error('cannot install update %s', self.path, exc_info=True)
self._removeLocalData()
raise Error(_('Update instalation failed miserably'))
def selectBy(self, type=None, lang=None):
for s in self.streams.values():
if self.types and s.type not in self.types:
continue
if type and not s.type.startswith(type):
continue
if lang and lang != s.lang.lower():
continue
return self.select(s.no)
err = Error(_('There is no matching stream in ') + self.path).add(
'path', self.path)
if type: err.add('type', type)
if lang: err.add('language', lang)
raise err
def raiseNotSupportedAssets(assets):
msg = []
speech = [asset for asset in assets if asset.type == 'speech']
dicts = [asset for asset in assets if asset.type == 'dict']
if speech:
langs = ', '.join([languages.getName(a.params[0]) for a in speech])
msg += [ _('Synchronization with {} audio is currently not supported.') \
.format(langs) ]
if dicts:
langs = [
' - '.join([languages.getName(p) for p in a.params]) for a in dicts
]
msg += [ _('Synchronization between languages {} is currently not supported.') \
.format(', '.join(langs)) ]
msg += ['', _('missing assets:')]
msg += [' - ' + asset.getPrettyName() for asset in assets]
raise Error('\n'.join(msg))
def validateTask(task, outputRequired=False):
sub, ref, out = task.sub, task.ref, task.out
if sub is None or not sub.path or sub.no is None:
raise Error(_('Subtitles not set'), task=task)
if ref is None or not ref.path or ref.no is None:
raise Error(_('Reference file not set'), task=task)
if outputRequired and (not out or not out.path):
raise Error(_('Output file not set'), task=task)
if sub.path == ref.path and sub.no == ref.no:
raise Error(_('Subtitles can\'t be the same as reference'), task=task)
if ref.type == 'audio' and not ref.lang:
raise Error(_('Select reference language first'), task=task)
if out and out.path:
try:
out.validateOutputPattern()
except:
raise Error(_('Invalid output pattern'), task=task)
def loadDictionary(lang1, lang2, minLen=0):
dictionary = gizmo.Dictionary()
asset = assets.getAsset('dict', (lang1, lang2))
if asset.isLocal():
for key, val in loadDictionaryFromFile(asset.path):
if len(key) >= minLen and len(val) >= minLen:
dictionary.add(key, val)
else:
asset = assets.getAsset('dict', (lang2, lang1))
if asset.isLocal():
for key, val in loadDictionaryFromFile(asset.path):
if len(key) >= minLen and len(val) >= minLen:
dictionary.add(val, key)
if not asset.isLocal():
raise Error(_('There is no dictionary for transaltion from {} to {}')
.format(lang1, lang2)) \
.add('language1', lang1) \
.add('language2', lang2)
logger.info('dictionary ready with %u entries', dictionary.size())
return dictionary
def raiseMissingAssets(assets):
msg = [_('Following assets are missing:')]
msg += [' - ' + asset.getPrettyName() for asset in assets]
raise Error('\n'.join(msg))
