def compile_content_server_translations(self):
self.info('Compiling content-server translations')
from calibre.utils.rapydscript import msgfmt
from calibre.utils.zipfile import ZipFile, ZIP_DEFLATED, ZipInfo, ZIP_STORED
with ZipFile(self.j(self.RESOURCES, 'content-server', 'locales.zip'), 'w', ZIP_DEFLATED) as zf:
for src in glob.glob(os.path.join(self.TRANSLATIONS, 'content-server', '*.po')):
data, h = self.hash_and_data(src)
current_hash = h.digest()
saved_hash, saved_data = self.read_cache(src)
if current_hash == saved_hash:
raw = saved_data
else:
# self.info('\tParsing ' + os.path.basename(src))
raw = None
po_data = data.decode('utf-8')
data = json.loads(msgfmt(po_data))
translated_entries = {k:v for k, v in iteritems(data['entries']) if v and sum(map(len, v))}
data[u'entries'] = translated_entries
data[u'hash'] = h.hexdigest()
cdata = b'{}'
if translated_entries:
raw = json.dumps(data, ensure_ascii=False, sort_keys=True)
if isinstance(raw, type(u'')):
raw = raw.encode('utf-8')
cdata = raw
self.write_cache(cdata, current_hash, src)
if raw:
zi = ZipInfo(os.path.basename(src).rpartition('.')[0])
zi.compress_type = ZIP_STORED if is_ci else ZIP_DEFLATED
zf.writestr(zi, raw)
def safe_replace(self,
name,
datastream,
extra_replacements={},
add_missing=False):
from calibre.utils.zipfile import ZipFile, ZipInfo
replacements = {name: datastream}
replacements.update(extra_replacements)
names = frozenset(replacements.keys())
found = set([])
with SpooledTemporaryFile(max_size=100 * 1024 * 1024) as temp:
ztemp = ZipFile(temp, 'w')
for offset, header in self.file_info.itervalues():
if header.filename in names:
zi = ZipInfo(header.filename)
zi.compress_type = header.compression_method
ztemp.writestr(zi, replacements[header.filename].read())
found.add(header.filename)
else:
ztemp.writestr(header.filename,
self.read(header.filename, spool_size=0))
if add_missing:
for name in names - found:
ztemp.writestr(name, replacements[name].read())
ztemp.close()
zipstream = self.stream
temp.seek(0)
zipstream.seek(0)
zipstream.truncate()
shutil.copyfileobj(temp, zipstream)
zipstream.flush()
def compile_content_server_translations(self):
self.info('Compiling content-server translations')
from calibre.utils.rapydscript import msgfmt
from calibre.utils.zipfile import ZipFile, ZIP_DEFLATED, ZipInfo, ZIP_STORED
with ZipFile(self.j(self.RESOURCES, 'content-server', 'locales.zip'), 'w', ZIP_DEFLATED) as zf:
for src in glob.glob(os.path.join(self.TRANSLATIONS, 'content-server', '*.po')):
data, h = self.hash_and_data(src)
current_hash = h.digest()
saved_hash, saved_data = self.read_cache(src)
if current_hash == saved_hash:
raw = saved_data
else:
self.info('\tParsing ' + os.path.basename(src))
raw = None
po_data = data.decode('utf-8')
data = json.loads(msgfmt(po_data))
translated_entries = {k:v for k, v in iteritems(data['entries']) if v and sum(map(len, v))}
data[u'entries'] = translated_entries
data[u'hash'] = h.hexdigest()
cdata = b'{}'
if translated_entries:
raw = json.dumps(data, ensure_ascii=False, sort_keys=True)
if isinstance(raw, type(u'')):
raw = raw.encode('utf-8')
cdata = raw
self.write_cache(cdata, current_hash, src)
if raw:
zi = ZipInfo(os.path.basename(src).rpartition('.')[0])
zi.compress_type = ZIP_STORED if is_ci else ZIP_DEFLATED
zf.writestr(zi, raw)
def safe_replace(self, name, datastream, extra_replacements={},
add_missing=False):
from calibre.utils.zipfile import ZipFile, ZipInfo
replacements = {name:datastream}
replacements.update(extra_replacements)
names = frozenset(replacements.keys())
found = set([])
with SpooledTemporaryFile(max_size=100*1024*1024) as temp:
ztemp = ZipFile(temp, 'w')
for offset, header in self.file_info.itervalues():
if header.filename in names:
zi = ZipInfo(header.filename)
zi.compress_type = header.compression_method
ztemp.writestr(zi, replacements[header.filename].read())
found.add(header.filename)
else:
ztemp.writestr(header.filename, self.read(header.filename,
spool_size=0))
if add_missing:
for name in names - found:
ztemp.writestr(name, replacements[name].read())
ztemp.close()
zipstream = self.stream
temp.seek(0)
zipstream.seek(0)
zipstream.truncate()
shutil.copyfileobj(temp, zipstream)
zipstream.flush()
def compile_website_translations(self):
from calibre.utils.zipfile import ZipFile, ZipInfo, ZIP_STORED
from calibre.ptempfile import TemporaryDirectory
from calibre.utils.localization import get_iso639_translator, get_language, get_iso_language
self.info('Compiling website translations...')
srcbase = self.j(self.d(self.SRC), 'translations', 'website')
fmap = {}
files = []
stats = {}
done = []
def handle_stats(src, nums):
locale = fmap[src]
trans = nums[0]
total = trans if len(nums) == 1 else (trans + nums[1])
stats[locale] = int(round(100 * trans / total))
with TemporaryDirectory() as tdir, ZipFile(self.j(srcbase, 'locales.zip'), 'w', ZIP_STORED) as zf:
for f in os.listdir(srcbase):
if f.endswith('.po'):
l = f.partition('.')[0]
pf = l.split('_')[0]
if pf in {'en'}:
continue
d = os.path.join(tdir, l + '.mo')
f = os.path.join(srcbase, f)
fmap[f] = l
files.append((f, d))
self.compile_group(files, handle_stats=handle_stats)
for locale, translated in iteritems(stats):
if translated >= 20:
with open(os.path.join(tdir, locale + '.mo'), 'rb') as f:
raw = f.read()
zi = ZipInfo(os.path.basename(f.name))
zi.compress_type = ZIP_STORED
zf.writestr(zi, raw)
done.append(locale)
dl = done + ['en']
lang_names = {}
for l in dl:
if l == 'en':
t = get_language
else:
t = getattr(get_iso639_translator(l), 'gettext' if ispy3 else 'ugettext')
t = partial(get_iso_language, t)
lang_names[l] = {x: t(x) for x in dl}
zi = ZipInfo('lang-names.json')
zi.compress_type = ZIP_STORED
zf.writestr(zi, json.dumps(lang_names, ensure_ascii=False).encode('utf-8'))
dest = self.j(self.d(self.stats), 'website-languages.txt')
data = ' '.join(sorted(done))
if not isinstance(data, bytes):
data = data.encode('utf-8')
with open(dest, 'wb') as f:
f.write(data)
def compile_content_server_translations(self):
self.info('\nCompiling content-server translations')
from calibre.utils.rapydscript import msgfmt
from calibre.utils.zipfile import ZipFile, ZIP_DEFLATED, ZipInfo
with ZipFile(self.j(self.RESOURCES, 'content-server', 'locales.zip'), 'w', ZIP_DEFLATED) as zf:
for src in glob.glob(os.path.join(self.TRANSLATIONS, 'content-server', '*.po')):
with open(src, 'rb') as f:
po_data = f.read().decode('utf-8')
data = json.loads(msgfmt(po_data))
translated_entries = {k:v for k, v in data['entries'].iteritems() if v and sum(map(len, v))}
data['entries'] = translated_entries
if translated_entries:
raw = json.dumps(data, ensure_ascii=False, sort_keys=True)
if isinstance(raw, type(u'')):
raw = raw.encode('utf-8')
zi = ZipInfo(os.path.basename(src).rpartition('.')[0])
zi.compress_type = ZIP_DEFLATED
zf.writestr(zi, raw)
def _compile_website_translations(self, name='website', threshold=50):
from calibre.utils.zipfile import ZipFile, ZipInfo, ZIP_STORED
from calibre.ptempfile import TemporaryDirectory
from calibre.utils.localization import get_iso639_translator, get_language, get_iso_language
self.info('Compiling', name, 'translations...')
srcbase = self.j(self.d(self.SRC), 'translations', name)
if not os.path.exists(srcbase):
os.makedirs(srcbase)
fmap = {}
files = []
stats = {}
done = []
def handle_stats(src, nums):
locale = fmap[src]
trans = nums[0]
total = trans if len(nums) == 1 else (trans + nums[1])
stats[locale] = int(round(100 * trans / total))
with TemporaryDirectory() as tdir, ZipFile(
self.j(srcbase, 'locales.zip'), 'w', ZIP_STORED) as zf:
for f in os.listdir(srcbase):
if f.endswith('.po'):
l = f.partition('.')[0]
pf = l.split('_')[0]
if pf in {'en'}:
continue
d = os.path.join(tdir, l + '.mo')
f = os.path.join(srcbase, f)
fmap[f] = l
files.append((f, d))
self.compile_group(files, handle_stats=handle_stats)
for locale, translated in iteritems(stats):
if translated >= 50:
with open(os.path.join(tdir, locale + '.mo'), 'rb') as f:
raw = f.read()
zi = ZipInfo(os.path.basename(f.name))
zi.compress_type = ZIP_STORED
zf.writestr(zi, raw)
done.append(locale)
dl = done + ['en']
lang_names = {}
for l in dl:
if l == 'en':
t = get_language
else:
t = getattr(get_iso639_translator(l),
'gettext' if ispy3 else 'ugettext')
t = partial(get_iso_language, t)
lang_names[l] = {x: t(x) for x in dl}
zi = ZipInfo('lang-names.json')
zi.compress_type = ZIP_STORED
zf.writestr(
zi,
json.dumps(lang_names, ensure_ascii=False).encode('utf-8'))
return done
def compile_content_server_translations(self):
self.info('\nCompiling content-server translations')
from calibre.utils.rapydscript import msgfmt
from calibre.utils.zipfile import ZipFile, ZIP_DEFLATED, ZipInfo
with ZipFile(self.j(self.RESOURCES, 'content-server', 'locales.zip'),
'w', ZIP_DEFLATED) as zf:
for src in glob.glob(
os.path.join(self.TRANSLATIONS, 'content-server', '*.po')):
with open(src, 'rb') as f:
po_data = f.read().decode('utf-8')
data = json.loads(msgfmt(po_data))
translated_entries = {
k: v
for k, v in data['entries'].iteritems()
if v and sum(map(len, v))
}
data['entries'] = translated_entries
if translated_entries:
raw = json.dumps(data, ensure_ascii=False, sort_keys=True)
if isinstance(raw, type(u'')):
raw = raw.encode('utf-8')
zi = ZipInfo(os.path.basename(src).rpartition('.')[0])
zi.compress_type = ZIP_DEFLATED
zf.writestr(zi, raw)