def write_ext(glos, filename, sort=True, dictZip=True):
if sort:
g = glos.copy()
g.data.sort()
else:
g = glos
try:
import _stardictbuilder
except ImportError:
printAsError('Binary module "_stardictbuilder" can not be imported! '+\
'Using internal StarDict builder')
return g.writeStardict(filename, sort=False)
db = _stardictbuilder.new_StarDictBuilder(filename)
_stardictbuilder.StarDictBuilder_swigregister(db)
for item in g.data:
_stardictbuilder.StarDictBuilder_addHeadword(db,item[0],item[1], '')
_stardictbuilder.StarDictBuilder_setTitle(db, g.getInfo('name'))
_stardictbuilder.StarDictBuilder_setAuthor(db, g.getInfo('author'))
_stardictbuilder.StarDictBuilder_setLicense(db, g.getInfo('license'))
_stardictbuilder.StarDictBuilder_setOrigLang(db, g.getInfo('origLang'))
_stardictbuilder.StarDictBuilder_setDestLang(db, g.getInfo('destLang'))
_stardictbuilder.StarDictBuilder_setDescription(db, g.getInfo('description'))
_stardictbuilder.StarDictBuilder_setComments(db, g.getInfo('comments'))
_stardictbuilder.StarDictBuilder_setEmail(db, g.getInfo('email'))
_stardictbuilder.StarDictBuilder_setWebsite(db, g.getInfo('website'))
_stardictbuilder.StarDictBuilder_setVersion(db, g.getInfo('version'))
_stardictbuilder.StarDictBuilder_setcreationTime(db, '')
_stardictbuilder.StarDictBuilder_setLastUpdate(db, '')
_stardictbuilder.StarDictBuilder_finish(db)
if dictZip:
if filename[-4:]=='.ifo':
filename = filename[:-4]
runDictzip(filename)
def write_ext(glos, filename, sort=True, dictZip=True):
if sort:
g = glos.copy()
g.data.sort()
else:
g = glos
try:
import _stardictbuilder
except ImportError:
printAsError('Binary module "_stardictbuilder" can not be imported! '+\
'Using internal StarDict builder')
return g.writeStardict(filename, sort=False)
db = _stardictbuilder.new_StarDictBuilder(filename)
_stardictbuilder.StarDictBuilder_swigregister(db)
for item in g.data:
_stardictbuilder.StarDictBuilder_addHeadword(db, item[0], item[1], '')
_stardictbuilder.StarDictBuilder_setTitle(db, g.getInfo('name'))
_stardictbuilder.StarDictBuilder_setAuthor(db, g.getInfo('author'))
_stardictbuilder.StarDictBuilder_setLicense(db, g.getInfo('license'))
_stardictbuilder.StarDictBuilder_setOrigLang(db, g.getInfo('origLang'))
_stardictbuilder.StarDictBuilder_setDestLang(db, g.getInfo('destLang'))
_stardictbuilder.StarDictBuilder_setDescription(db,
g.getInfo('description'))
_stardictbuilder.StarDictBuilder_setComments(db, g.getInfo('comments'))
_stardictbuilder.StarDictBuilder_setEmail(db, g.getInfo('email'))
_stardictbuilder.StarDictBuilder_setWebsite(db, g.getInfo('website'))
_stardictbuilder.StarDictBuilder_setVersion(db, g.getInfo('version'))
_stardictbuilder.StarDictBuilder_setcreationTime(db, '')
_stardictbuilder.StarDictBuilder_setLastUpdate(db, '')
_stardictbuilder.StarDictBuilder_finish(db)
if dictZip:
if filename[-4:] == '.ifo':
filename = filename[:-4]
runDictzip(filename)
def utf8ReplaceErrors(self):
errors = 0
for i in xrange(len(self.data)):
(w, m) = self.data[i][:2]
w = w.replace('\x00', '')
m = m.replace('\x00', '')
try:
m.decode('utf-8')
except UnicodeDecodeError:
m = m.decode('utf-8', 'replace').encode('utf-8')
errors += 1
try:
w.decode('utf-8')
except UnicodeDecodeError:
w = w.decode('utf-8', 'replace').encode('utf-8')
errors += 1
if len(self.data[i]) >= 3:
d = self.data[i][2]
if 'alts' in d:
a = d['alts']
for j in xrange(len(a)):
a[j] = a[j].replace('\x00', '')
try:
a[j].decode('utf-8')
except UnicodeDecodeError:
a[j] = a[j].decode('utf-8',
'replace').encode('utf-8')
errors += 1
d = [d]
else:
d = []
a = [w, m]
a.extend(d)
a.extend(self.data[i][3:])
self.data[i] = a
for i in xrange(len(self.info)):
(w, m) = self.info[i]
w = w.replace('\x00', '')
m = m.replace('\x00', '')
try:
m.decode('utf-8')
except UnicodeDecodeError:
m = m.decode('utf-8', 'replace').encode('utf-8')
errors += 1
try:
w.decode('utf-8')
except UnicodeDecodeError:
w = w.decode('utf-8', 'replace').encode('utf-8')
errors += 1
self.info[i] = (w, m)
if errors:
printAsError(
'There was %s number of invalid utf8 strings, invalid characters are replaced with "�"'
% errors)
def utf8ReplaceErrors(self):
errors = 0
for i in xrange(len(self.data)):
(w, m) = self.data[i][:2]
w = w.replace('\x00', '')
m = m.replace('\x00', '')
try:
m.decode('utf-8')
except UnicodeDecodeError:
m = m.decode('utf-8', 'replace').encode('utf-8')
errors += 1
try:
w.decode('utf-8')
except UnicodeDecodeError:
w = w.decode('utf-8', 'replace').encode('utf-8')
errors += 1
if len(self.data[i]) >= 3:
d = self.data[i][2]
if 'alts' in d:
a = d['alts']
for j in xrange(len(a)):
a[j] = a[j].replace('\x00', '')
try:
a[j].decode('utf-8')
except UnicodeDecodeError:
a[j] = a[j].decode('utf-8', 'replace').encode('utf-8')
errors += 1
d = [d]
else:
d = []
a = [w, m]
a.extend(d)
a.extend(self.data[i][3:])
self.data[i] = a
for i in xrange(len(self.info)):
(w, m) = self.info[i]
w = w.replace('\x00', '')
m = m.replace('\x00', '')
try:
m.decode('utf-8')
except UnicodeDecodeError:
m = m.decode('utf-8', 'replace').encode('utf-8')
errors += 1
try:
w.decode('utf-8')
except UnicodeDecodeError:
w = w.decode('utf-8', 'replace').encode('utf-8')
errors += 1
self.info[i] = (w, m)
if errors:
printAsError('There was %s number of invalid utf8 strings, invalid characters are replaced with "�"'%errors)
def writeTxt(self,
sep,
filename='',
writeInfo=True,
rplList=[],
ext='.txt',
head=''):
if not filename:
filename = self.filename + ext
txt = head
if writeInfo:
for t in self.info:
#??????????????????????????
inf = t[1]
for rpl in rplList:
inf = inf.replace(rpl[0], rpl[1])
txt += ('##' + t[0] + sep[0] + inf + sep[1])
#inf = self.getInfo(t[0])
#if inf!='':
# try:
# txt+=('##' + t[0] + sep[0] + inf + sep[1])
# except:
# myRaise(__file__)
# printAsError('Error on writing info line for "%s"'%t[0])
for item in self.data:
(word, defi) = item[:2]
if word.startswith('#'):
continue
if self.getPref('enable_alts', True):
try:
alts = item[2]['alts']
except:
pass
else:
if alts:
word = '|'.join([word] + alts)
for rpl in rplList:
defi = defi.replace(rpl[0], rpl[1])
try:
line = word + sep[0] + defi + sep[1]
txt += line
except:
myRaise(__file__)
printAsError('Error on writing line for word "%s"' % word)
continue
if filename == None:
return txt
with open(filename, 'wb') as fp:
fp.write(txt)
return True
def writeTxt(self, sep, filename='', writeInfo=True, rplList=[], ext='.txt', head=''):
if not filename:
filename = self.filename + ext
txt = head
if writeInfo:
for t in self.info:
#??????????????????????????
inf = t[1]
for rpl in rplList:
inf = inf.replace(rpl[0], rpl[1])
txt += ('##' + t[0] + sep[0] + inf + sep[1])
#inf = self.getInfo(t[0])
#if inf!='':
# try:
# txt+=('##' + t[0] + sep[0] + inf + sep[1])
# except:
# myRaise(__file__)
# printAsError('Error on writing info line for "%s"'%t[0])
for item in self.data:
(word, defi) = item[:2]
if word.startswith('#'):
continue
if self.getPref('enable_alts', True):
try:
alts = item[2]['alts']
except:
pass
else:
if alts:
word = '|'.join([word] + alts)
for rpl in rplList:
defi = defi.replace(rpl[0], rpl[1])
try:
line = word + sep[0] + defi + sep[1]
txt += line
except:
myRaise(__file__)
printAsError('Error on writing line for word "%s"'%word)
continue
if filename==None:
return txt
with open(filename, 'wb') as fp:
fp.write(txt)
return True
def copy_resources(fromPath, toPath, overwrite):
"""Copy resource files from fromPath to toPath.
"""
if not fromPath:
return
fromPath = os.path.abspath(fromPath)
toPath = os.path.abspath(toPath)
if fromPath == toPath:
return
if not os.path.isdir(fromPath):
return
if len(os.listdir(fromPath)) == 0:
return
if overwrite and os.path.exists(toPath):
shutil.rmtree(toPath)
if os.path.exists(toPath):
if len(os.listdir(toPath)) > 0:
printAsError('Output resource directory is not empty: "{0}". Resources will not be copied! '
'Clean the output directory before running the converter or pass option: --write-options=res-overwrite=True.'\
.format(toPath))
return
os.rmdir(toPath)
shutil.copytree(fromPath, toPath)
def copy_resources(fromPath, toPath, overwrite):
"""Copy resource files from fromPath to toPath.
"""
if not fromPath:
return
fromPath = os.path.abspath(fromPath)
toPath = os.path.abspath(toPath)
if fromPath == toPath:
return
if not os.path.isdir(fromPath):
return
if len(os.listdir(fromPath))==0:
return
if overwrite and os.path.exists(toPath):
shutil.rmtree(toPath)
if os.path.exists(toPath):
if len(os.listdir(toPath)) > 0:
printAsError('Output resource directory is not empty: "{0}". Resources will not be copied! '
'Clean the output directory before running the converter or pass option: --write-options=res-overwrite=True.'\
.format(toPath))
return
os.rmdir(toPath)
shutil.copytree(fromPath, toPath)
def write(self, filename, format='', **options):
if not filename:
printAsError('Invalid filename %r'%filename)
return False
ext = ''
(filename_nox, fext) = splitext(filename)
fext = fext.lower()
if fext in ('.gz', '.bz2', '.zip'):
zipExt = fext
filename = filename_nox
fext = splitext(filename)[1].lower()
else:
zipExt = ''
del filename_nox
if format:
try:
ext = Glossary.formatsExt[format][0]
except KeyError:
myRaise()
format = '' ## ?????
if not format:
items = Glossary.formatsExt.items()
for (fmt, extList) in items:
for e in extList:
if format==e[1:] or format==e:
format = fmt
ext = e
break
if format:
break
if not format:
for (fmt, extList) in items:
if filename==fmt:
format = filename
ext = extList[0]
filename = self.filename + ext
break
for e in extList:
if filename==e[1:] or filename==e:
format = fmt
ext = e
filename = self.filename + ext
break
if format:
break
if not format:
for (fmt, extList) in items:
if fext in extList:
format = fmt
ext = fext
if not format:
printAsError('Unable to detect write format!')
return False
if isdir(filename):
#filename = join(filename, split(self.filename)[1]+ext)
filename = join(filename, self.filename+ext)
validOptionKeys = self.formatsWriteOptions[format]
for key in options.keys():
if not key in validOptionKeys:
printAsError('Invalid write option "%s" given for %s format'%(key, format))
del options[key]
print 'filename=%s'%filename
getattr(self, 'write%s'%format).__call__(filename, **options)
if zipExt:
try:
os.remove('%s%s'%(filename, zipExt))
except OSError:
pass
if zipExt=='.gz':
(output, error) = subprocess.Popen(
['gzip', filename],
stdout=subprocess.PIPE,
).communicate()
if error:
printAsError('%s\nfail to compress file "%s"'%(error, filename))
elif zipExt=='.bz2':
(output, error) = subprocess.Popen(
['bzip2', filename],
stdout=subprocess.PIPE,
).communicate()
if error:
printAsError('%s\nfail to compress file "%s"'%(error, filename))
elif zipExt=='.zip':
(dirn, name) = split(filename)
initCwd = os.getcwd()
os.chdir(dirn)
(output, error) = subprocess.Popen(
['zip', filename+'.zip', name, '-m'],
stdout=subprocess.PIPE,
).communicate()
if error:
printAsError('%s\nfail to compress file "%s"'%(error, filename))
os.chdir(initCwd)
def read(self, filename, format='', **options):
delFile=False
ext = splitext(filename)[1]
ext = ext.lower()
if ext in ('.gz', '.bz2', '.zip'):
if ext=='.bz2':
(output, error) = subprocess.Popen(
['bzip2', '-dk', filename],
stdout=subprocess.PIPE,
).communicate()
## -k ==> keep original bz2 file
## bunzip2 ~= bzip2 -d
if error:
printAsError('%s\nfail to decompress file "%s"'%(error, filename))
return False
else:
filename = filename[:-4]
ext = splitext(filename)[1]
delFile = True
elif ext=='.gz':
(output, error) = subprocess.Popen(
['gzip', '-dc', filename],
stdout=subprocess.PIPE,
).communicate()
## -c ==> write to stdout (because we want to keep original gz file)
## gunzip ~= gzip -d
if error:
printAsError('%s\nfail to decompress file "%s"'%(error, filename))
return False
else:
filename = filename[:-3]
open(filename, 'w').write(output)
ext = splitext(filename)[1]
delFile = True
elif ext=='.zip':
(output, error) = subprocess.Popen(
['unzip', filename, '-d', dirname(filename)],
stdout=subprocess.PIPE,
).communicate()
if error:
printAsError('%s\nfail to decompress file "%s"'%(error, filename))
return False
else:
filename = filename[:-4]
ext = splitext(filename)[1]
delFile = True
if not format:
for key in Glossary.formatsExt.keys():
if ext in Glossary.formatsExt[key]:
format = key
if not format:
#if delFile:
# os.remove(filename)
printAsError('Unknown extension "%s" for read support!'%ext)
return False
validOptionKeys = self.formatsReadOptions[format]
for key in options.keys():
if not key in validOptionKeys:
printAsError('Invalid read option "%s" given for %s format'%(key, format))
del options[key]
getattr(self, 'read%s'%format).__call__(filename, **options)
(filename_nox, ext) = splitext(filename)
if ext.lower() in self.formatsExt[format]:
filename = filename_nox
self.filename = filename
if self.getInfo('name') == '':
self.setInfo('name', split(filename)[1])
if delFile:
os.remove(filename)
return True
from text_utils import myRaise, printAsError, faEditStr, replacePostSpaceChar, removeTextTags,\
takeStrWords, findWords, findAll, addDefaultOptions
import warnings
warnings.resetwarnings() ## ??????
_myDir = dirname(__file__)
if not isdir(_myDir):
_myDir = dirname(dirname(_myDir))
plugDir = join(_myDir, 'plugins')
if isdir(plugDir):
sys.path.append(plugDir)
else:
printAsError('invalid plugin directory %r'%plugDir)
plugDir = ''
psys = platform.system()
if os.sep=='/': ## Operating system is Unix-Like
homeDir = os.getenv('HOME')
user = os.getenv('USER')
tmpDir = '/tmp'
## os.name == 'posix' ## ????
if psys=='Darwin':## MacOS X
confPath = homeDir + '/Library/Preferences/PyGlossary' ## OR '/Library/PyGlossary'
## os.environ['OSTYPE'] == 'darwin10.0'
## os.environ['MACHTYPE'] == 'x86_64-apple-darwin10.0'