def __init__(self, path, fields, read_only=False):
self.nb_transactions = 0
self.last_transaction_dtime = None
self.path = abspath(path) + '/'
self.fields = fields
self.read_only = read_only
# Open database
self.path_data = '%s/database/' % self.path
# Check if is a folder
self.path_data = '%s/database/' % self.path
if not lfs.is_folder(self.path_data):
error = '"{0}" should be a folder, but it is not'.format(self.path_data)
raise ValueError(error)
# New interface to Git
self.worktree = open_worktree(self.path_data)
# Initialize the database, but chrooted
self.fs = lfs.open(self.path_data)
# Static FS
database_static_path = '{0}/database_static'.format(path)
if not lfs.exists(database_static_path):
self.init_backend_static(path)
self.static_fs = lfs.open(database_static_path)
# Patchs backend
self.patchs_backend = PatchsBackend(path, self.fs, read_only)
# Catalog
self.catalog = self.get_catalog()
def __init__(self, path, size_min=4800, size_max=5200):
# 1. Keep the path
if not lfs.is_folder(path):
error = '"%s" should be a folder, but it is not' % path
raise ValueError, error
folder = lfs.open(path)
self.path = str(folder.path)
# 2. Keep the path to the data
self.path_data = '%s/database/' % self.path
if not lfs.is_folder(self.path_data):
error = '"%s" should be a folder, but it is not' % self.path_data
raise ValueError, error
# 3. Initialize the database, but chrooted
self.fs = lfs.open(self.path_data)
# 4. New interface to Git
self.worktree = open_worktree(self.path_data)
# 5. A mapping from key to handler
self.cache = LRUCache(size_min, size_max, automatic=False)
# 6. The git cache
self.git_cache = LRUCache(900, 1100)
def __init__(self, path, size_min=4800, size_max=5200):
# 1. Keep the path
if not lfs.is_folder(path):
error = '"%s" should be a folder, but it is not' % path
raise ValueError, error
folder = lfs.open(path)
self.path = str(folder.path)
# 2. Keep the path to the data
self.path_data = '%s/database/' % self.path
if not lfs.is_folder(self.path_data):
error = '"%s" should be a folder, but it is not' % self.path_data
raise ValueError, error
# 3. Initialize the database, but chrooted
self.fs = lfs.open(self.path_data)
# 4. New interface to Git
self.worktree = open_worktree(self.path_data)
# 5. A mapping from key to handler
self.cache = LRUCache(size_min, size_max, automatic=False)
# 6. The git cache
self.git_cache = LRUCache(900, 1100)
def action_test(self):
""" Test if ikaaro instances of this Python environment are alive"""
for ikaaro in config.get_sections_by_type('ikaaro'):
if ikaaro.options['pyenv'] == self.name:
uri = ikaaro.options['uri']
for i in range(1, 6):
try:
lfs.open('{}/;_ctrl'.format(uri))
except Exception:
log_error('[ERROR {}/5] {}'.format(i, uri))
sleep(0.5)
else:
log_info('[OK] {}'.format(uri))
break
def test_move_file(self):
lfs.copy('tests/hello.txt', 'tests/hello.txt.bak')
lfs.move('tests/hello.txt.bak', 'tests/hello.txt.old')
file = lfs.open('tests/hello.txt.old')
self.assertEqual(file.read(), 'hello world\n')
self.assertEqual(lfs.exists('tests/hello.txt.bak'), False)
lfs.remove('tests/hello.txt.old')
def _smtp_send(self):
nb_max_mails_to_send = 2
spool = lfs.open(self.spool)
def get_names():
# Find out emails to send
locks = set()
names = set()
for name in spool.get_names():
if name == 'failed':
# Skip "failed" special directory
continue
if name[-5:] == '.lock':
locks.add(name[:-5])
else:
names.add(name)
names.difference_update(locks)
return names
# Send emails
names = get_names()
smtp_host = self.smtp_host
for name in list(names)[:nb_max_mails_to_send]:
# 1. Open connection
try:
smtp = SMTP(smtp_host)
except gaierror, excp:
log_warning('%s: "%s"' % (excp[1], smtp_host))
return 60 # 1 minute
except Exception:
self.smtp_log_error()
return 60 # 1 minute
def test_copy_file(self):
lfs.copy('tests/hello.txt', 'tmp/hello.txt.bak')
file = lfs.open('tmp/hello.txt.bak')
try:
self.assertEqual(file.read(), 'hello world\n')
finally:
file.close()
def tearDown(self):
folder = lfs.open('tests')
for name in 'toto.txt', 'fofo.txt', 'fofo2.txt', 'empty':
if folder.exists(name):
folder.remove(name)
if lfs.exists("test_dir"):
lfs.remove("test_dir")
def get_namespace(self, resource, context):
namespace = {}
# Credits
credits = get_abspath('CREDITS.txt')
lines = lfs.open(credits).readlines()
names = [ x[2:].strip() for x in lines if x.startswith(' ') ]
namespace['hackers'] = names
# Installed software
root = context.root
is_admin = root.is_admin(context.user, resource)
namespace['is_admin'] = is_admin
if is_admin:
package2title = {
'gio': u'pygobject',
'lpod': u'lpOD',
'sys': u'Python',
'os': MSG(u'Operating System')}
packages = [
{'name': package2title.get(x, x),
'version': y or MSG('no version found')}
for x, y in root.get_version_of_packages(context).items() ]
location = (getuser(), gethostname(), context.server.target)
namespace['packages'] = packages
namespace['location'] = u'%s@%s:%s' % location
# Ok
return namespace
def test_copy_file(self):
lfs.copy('tests/hello.txt', 'tmp/hello.txt.bak')
file = lfs.open('tmp/hello.txt.bak')
try:
self.assertEqual(file.read(), 'hello world\n')
finally:
file.close()
def test_copy_folder_to_folder(self):
lfs.copy('tests', 'tmp')
file = lfs.open('tmp/tests/hello.txt')
try:
self.assertEqual(file.read(), 'hello world\n')
finally:
file.close()
def test_move_file(self):
lfs.copy('tests/hello.txt', 'tests/hello.txt.bak')
lfs.move('tests/hello.txt.bak', 'tests/hello.txt.old')
file = lfs.open('tests/hello.txt.old')
self.assertEqual(file.read(), 'hello world\n')
self.assertEqual(lfs.exists('tests/hello.txt.bak'), False)
lfs.remove('tests/hello.txt.old')
def test_copy_folder_to_folder(self):
lfs.copy('tests', 'tmp')
file = lfs.open('tmp/tests/hello.txt')
try:
self.assertEqual(file.read(), 'hello world\n')
finally:
file.close()
def get_namespace(self, resource, context):
namespace = {}
# Credits
credits = get_abspath('CREDITS.txt')
lines = lfs.open(credits).readlines()
names = [x[2:].strip() for x in lines if x.startswith(' ')]
namespace['hackers'] = names
# Installed software
root = context.root
is_admin = root.is_admin(context.user, resource)
namespace['is_admin'] = is_admin
if is_admin:
package2title = {
'gio': u'pygobject',
'lpod': u'lpOD',
'sys': u'Python',
'os': MSG(u'Operating System')
}
packages = [{
'name': package2title.get(x, x),
'version': y or MSG('no version found')
} for x, y in root.get_version_of_packages(context).items()]
location = (getuser(), gethostname(), context.server.target)
namespace['packages'] = packages
namespace['location'] = u'%s@%s:%s' % location
# Ok
return namespace
def get_namespace(self, resource, context):
namespace = {}
# Credits
credits = get_abspath("CREDITS.txt")
lines = lfs.open(credits).readlines()
names = [x[2:].strip() for x in lines if x.startswith(" ")]
namespace["hackers"] = names
# Installed software
root = context.root
is_admin = root.is_admin(context.user, resource)
namespace["is_admin"] = is_admin
if is_admin:
package2title = {"gio": u"pygobject", "lpod": u"lpOD", "sys": u"Python", "os": MSG(u"Operating System")}
packages = [
{"name": package2title.get(x, x), "version": y or MSG("no version found")}
for x, y in root.get_version_of_packages(context).items()
]
location = (getuser(), gethostname(), context.server.target)
namespace["packages"] = packages
namespace["location"] = u"%s@%s:%s" % location
# Ok
return namespace
def _make_resource(cls, folder, name, *args, **kw):
ShopFolder._make_resource(cls, folder, name, *args, **kw)
# Image
body = lfs.open(get_abspath(cls.img)).read()
img = Image._make_resource(Image, folder,
'%s/logo.png' % name, body=body,
**{'state': 'public'})
# Load zones
shop = get_context().resource.parent
zones = []
handler = shop.get_resource('countries-zones').handler
for record in handler.get_records():
zones.append(handler.get_record_value(record, 'title'))
# Create history
cls.shipping_history_cls._make_resource(cls.shipping_history_cls,
folder, '%s/history' % name)
# Import CSV with prices
ShippingPrices._make_resource(ShippingPrices, folder,
'%s/prices' % name)
if getattr(cls, 'csv', None):
table = ShippingPricesTable()
csv = ro_database.get_handler(get_abspath(cls.csv), ShippingPricesCSV)
for row in csv.get_rows():
table.add_record(
{'zone': str(zones.index(row.get_value('zone'))),
'max-weight': row.get_value('max-weight'),
'price': row.get_value('price')})
folder.set_handler('%s/prices' % name, table)
def _smtp_send(self):
nb_max_mails_to_send = 2
spool = lfs.open(self.spool)
def get_names():
# Find out emails to send
locks = set()
names = set()
for name in spool.get_names():
if name == 'failed':
# Skip "failed" special directory
continue
if name[-5:] == '.lock':
locks.add(name[:-5])
else:
names.add(name)
names.difference_update(locks)
return names
# Send emails
names = get_names()
smtp_host = self.smtp_host
for name in list(names)[:nb_max_mails_to_send]:
# 1. Open connection
try:
smtp = SMTP(smtp_host)
except Exception:
self.smtp_log_error()
spool.move(name, 'failed/%s' % name)
return 60 if get_names() else False
log_info('CONNECTED to %s' % smtp_host)
# 2. Login
if self.smtp_login and self.smtp_password:
smtp.login(self.smtp_login, self.smtp_password)
# 3. Send message
try:
message = spool.open(name).read()
headers = HeaderParser().parsestr(message)
subject = headers['subject']
from_addr = headers['from']
to_addr = headers['to']
smtp.sendmail(from_addr, to_addr, message)
# Remove
spool.remove(name)
# Log
log_msg = 'Email "%s" sent from "%s" to "%s"'
log_info(log_msg % (subject, from_addr, to_addr))
except SMTPRecipientsRefused:
# The recipient addresses has been refused
self.smtp_log_error()
spool.move(name, 'failed/%s' % name)
except SMTPResponseException, excp:
# The SMTP server returns an error code
self.smtp_log_error()
spool.move(name, 'failed/%s_%s' % (excp.smtp_code, name))
except Exception:
self.smtp_log_error()
def get_catalog_values(self):
data = lfs.open(self.abspath).read()
return {
'name': basename(self.abspath),
'title': data.splitlines()[0],
'data': data,
'size': len(data),
'about_wolf': re.search('wolf', data, re.I) is not None,
'is_long': len(data) > 1024,
}
def get_catalog_values(self):
data = lfs.open(self.abspath).read()
return {
'name': basename(self.abspath),
'title': data.splitlines()[0],
'data': data,
'size': len(data),
'about_wolf': re.search('wolf', data, re.I) is not None,
'is_long': len(data) > 1024,
}
def __init__(self, db_path, db_fs, read_only):
self.db_fs = db_fs
self.db_path = db_path
# Init patchs folder
self.patchs_path = '{0}/database/.git/patchs'.format(db_path)
if not lfs.exists(self.patchs_path):
lfs.make_folder(self.patchs_path)
self.patchs_fs = lfs.open(self.patchs_path)
# Launch rotate on patchs folder (only one time, so only on RW database)
if not read_only:
self.launch_rotate()
def setUp(self):
# Make the catalog
catalog = make_catalog('tests/catalog', Document.fields)
# Index
fables = lfs.open('fables')
for name in fables.get_names():
abspath = fables.get_absolute_path(name)
document = Document(abspath)
catalog.index_document(document)
# Save
catalog.save_changes()
def setUp(self):
# Make the catalog
catalog = make_catalog('tests/catalog', Document.fields)
# Index
fables = lfs.open('fables')
for name in fables.get_names():
abspath = fables.get_absolute_path(name)
document = Document(abspath)
catalog.index_document(document)
# Save
catalog.save_changes()
def __init__(self, path):
self.nb_transactions = 0
self.last_transaction_dtime = None
self.path = abspath(path) + '/'
# Open database
self.path_data = '%s/database/' % self.path
# Check if is a folder
self.path_data = '%s/database/' % self.path
if not lfs.is_folder(self.path_data):
error = '"{0}" should be a folder, but it is not'.format(
self.path_data)
raise ValueError(error)
# New interface to Git
self.worktree = open_worktree(self.path_data)
# Initialize the database, but chrooted
self.fs = lfs.open(self.path_data)
# Static FS
database_static_path = '{0}/database_static'.format(path)
if not lfs.exists(database_static_path):
self.init_backend_static(path)
self.static_fs = lfs.open(database_static_path)
def __init__(self, path, fields, read_only=False):
self.nb_transactions = 0
self.last_transaction_dtime = None
self.path = abspath(path) + '/'
self.fields = fields
self.read_only = read_only
# Open database
self.path_data = '%s/database/' % self.path
# Check if is a folder
self.path_data = '%s/database/' % self.path
if not lfs.is_folder(self.path_data):
error = '"{0}" should be a folder, but it is not'.format(self.path_data)
raise ValueError(error)
# New interface to Git
self.worktree = open_worktree(self.path_data)
# Initialize the database, but chrooted
self.fs = lfs.open(self.path_data)
# Static FS
database_static_path = '{0}/database_static'.format(path)
if not lfs.exists(database_static_path):
self.init_backend_static(path)
self.static_fs = lfs.open(database_static_path)
# Catalog
self.catalog = self.get_catalog()
def test_append(self):
# Initialize
file = lfs.make_file('tests/toto.txt')
try:
file.write('hello\n')
finally:
file.close()
# Test
file = lfs.open('tests/toto.txt', APPEND)
try:
file.write('bye\n')
finally:
file.close()
self.assertEqual(open('tests/toto.txt').read(), 'hello\nbye\n')
# Remove temporary file
lfs.remove('tests/toto.txt')
def test_append(self):
# Initialize
file = lfs.make_file('tests/toto.txt')
try:
file.write('hello\n')
finally:
file.close()
# Test
file = lfs.open('tests/toto.txt', APPEND)
try:
file.write('bye\n')
finally:
file.close()
self.assertEqual(open('tests/toto.txt').read(), 'hello\nbye\n')
# Remove temporary file
lfs.remove('tests/toto.txt')
def test_write_and_truncate(self):
# Initialize
file = lfs.make_file('tests/toto.txt')
try:
file.write('hello\n')
finally:
file.close()
# Test
file = lfs.open('tests/toto.txt', WRITE)
try:
file.write('bye\n')
finally:
file.close()
self.assertEqual(open('tests/toto.txt').read(), 'bye\n')
# Remove temporary file
lfs.remove('tests/toto.txt')
def test_write_and_truncate(self):
# Initialize
file = lfs.make_file('tests/toto.txt')
try:
file.write('hello\n')
finally:
file.close()
# Test
file = lfs.open('tests/toto.txt', WRITE)
try:
file.write('bye\n')
finally:
file.close()
self.assertEqual(open('tests/toto.txt').read(), 'bye\n')
# Remove temporary file
lfs.remove('tests/toto.txt')
def setUp(self):
# Make database
self.database = make_git_database("fables", 20, 20, Document.fields)
self.database.worktree.git_add(".")
self.database.worktree.git_commit("Initial commit")
self.root = self.database.get_handler(".")
# Index
catalog = self.database.catalog
fables = lfs.open("fables/database")
for name in fables.get_names():
_, ext, _ = FileName.decode(name)
if ext == "txt":
abspath = fables.get_absolute_path(name)
document = Document(abspath)
catalog.index_document(document)
# Save
catalog.save_changes()
def setUp(self):
# Make database
self.database = make_git_database('fables', 20, 20, Document.fields)
self.database.worktree.git_add('.')
self.database.worktree.git_commit('Initial commit')
self.root = self.database.get_handler('.')
# Index
catalog = self.database.catalog
fables = lfs.open('fables/database')
for name in fables.get_names():
_, ext, _ = FileName.decode(name)
if ext == 'txt':
abspath = fables.get_absolute_path(name)
document = Document(abspath)
catalog.index_document(document)
# Save
catalog.save_changes()
def make_figures(format):
source_base = 'figures-src'
target_base = 'figures'
docs = lfs.open('docs')
if not docs.exists(source_base):
return
if not docs.exists(target_base):
docs.make_folder(target_base)
for name in docs.get_names(source_base):
source = '%s/%s' % (source_base, name)
mtime = docs.get_mtime(source)
name, extension = name.rsplit('.')
target = '%s/%s.%s' % (target_base, name, format)
if docs.exists(target) and docs.get_mtime(target) > mtime:
continue
command = converters.get((extension, format))
if command:
command = command.format(source=source, target=target)
run(command)
def make_figures(format):
source_base = 'figures-src'
target_base = 'figures'
docs = lfs.open('docs')
if not docs.exists(source_base):
return
if not docs.exists(target_base):
docs.make_folder(target_base)
for name in docs.get_names(source_base):
source = '%s/%s' % (source_base, name)
mtime = docs.get_mtime(source)
name, extension = name.rsplit('.')
target = '%s/%s.%s' % (target_base, name, format)
if docs.exists(target) and docs.get_mtime(target) > mtime:
continue
command = converters.get((extension, format))
if command:
command = command.format(source=source, target=target)
run(command)
'-f',
'--forget',
action='store_true',
default=False,
help='forget the old copyright (a more aggresive approach)')
options, args = parser.parse_args()
if len(args) == 0:
parser.error('incorrect number of arguments')
# Load CREDITS
credits_names = {} # Mapping from canonical email to real full name
credits_mails = {} # Mapping from secondary email to canonical email
emails = []
if lfs.exists('CREDITS'):
for line in lfs.open('CREDITS').readlines():
line = line.strip()
if line.startswith('#'):
continue
if line:
key, value = line.split(':', 1)
value = value.lstrip()
if key == 'N':
name = value
elif key == 'E':
emails.append(value)
if len(emails) == 1:
credits_names[value] = name
else:
credits_mails[value] = emails[0]
else:
def test_paragraph_cjk(self):
handler = lfs.open('pml/paragraph_cjk.xml')
story, stylesheet = pmltopdf_test(handler)
self.assertEqual(len(story), 12)
def test_span(self):
handler = lfs.open('pml/span.xml')
story, stylesheet = pmltopdf_test(handler)
self.assertEqual(len(story), 9)
def setUp(self):
self.tests = lfs.open('tests/')
self.tests.make_folder('toto')
def update_locale(srx_handler, exclude_folders, no_wrap=False):
# Read configuration for languages
config = get_config()
src_language = config.get_value('source_language', default='en')
# Get local folder
package_root = config.get_value('package_root')
if lfs.exists(package_root):
locale_folder_path = Path('{0}/locale'.format(package_root))
else:
locale_folder_path = Path('locale/')
locale_folder = lfs.open(locale_folder_path)
# Initialize message catalog
po = POFile()
lines = []
for line in open('MANIFEST').readlines():
line = line.strip()
exclude_folder = False
for x in exclude_folders:
if line.startswith(x):
exclude_folder = True
break
if exclude_folder is False:
lines.append(line)
# Process Python and HTML files
write('* Extract text strings')
extensions = [
'.py', '.js',
'.xhtml.%s' % src_language,
'.xml.%s' % src_language,
'.html.%s' % src_language
]
for path in lines:
# Filter files
for extension in extensions:
if path.endswith(extension):
break
else:
continue
# Get the units
write('.')
try:
handler = ro_database.get_handler(path)
except Exception:
print
print '*'
print '* Error:', path
print '*'
raise
try:
units = handler.get_units(srx_handler=srx_handler)
units = list(units)
except Exception:
print
print '*'
print '* Error:', path
print '*'
raise
relative_path = locale_folder_path.get_pathto(path)
for source, context, line in units:
po.add_unit(relative_path, source, context, line)
print
write('* Update PO template ')
data = po.to_str()
# Write the po into the locale.pot
try:
locale_pot = locale_folder.open('locale.pot', WRITE)
except IOError:
# The locale.pot file does not exist create and open
locale_pot = locale_folder.make_file('locale.pot')
else:
with locale_pot:
locale_pot.write(data)
# Update PO files
filenames = set([x for x in locale_folder.get_names() if x[-3:] == '.po'])
filenames.add('%s.po' % src_language)
for language in config.get_value('target_languages'):
filenames.add('%s.po' % language)
filenames = list(filenames)
filenames.sort()
print '* Update PO files:'
locale_pot_path = locale_folder.get_absolute_path('locale.pot')
for filename in filenames:
if locale_folder.exists(filename):
write(' %s ' % filename)
file_path = locale_folder.get_absolute_path(filename)
if no_wrap:
call([
'msgmerge', '--no-wrap', '-U', '-s', file_path,
locale_pot_path
])
else:
call(['msgmerge', '-U', '-s', file_path, locale_pot_path])
else:
print ' %s (new)' % filename
file_path = locale_folder.get_absolute_path(filename)
lfs.copy(locale_pot_path, file_path)
print
def get_spool_size(self):
spool = lfs.open(self.spool)
# We have always a 'failed' directory => "-1"
return len(spool.get_names()) - 1
def test_open_file(self):
file = lfs.open('tests/hello.txt')
self.assertEqual(file.read(), 'hello world\n')
def test_open_file(self):
file = lfs.open('tests/hello.txt')
self.assertEqual(file.read(), 'hello world\n')
def get_test_filenames(test_path, force_download):
"""Return the test file names
If the test files does'nt exists, we download it
"""
uris = {'http://download.wikimedia.org/qualitywiki/latest':
[('qualitywiki-latest-stub-articles.xml', '.gz'), #~ 3.1 KB
('qualitywiki-latest-stub-meta-current.xml', '.gz'), #~ 11.0 KB
('qualitywiki-latest-stub-meta-history.xml', '.gz')], #~ 28.9 KB
'http://download.wikimedia.org/tawiki/latest':
[('tawiki-latest-stub-articles.xml', '.gz'), #~ 1.2 MB
('tawiki-latest-stub-meta-history.xml', '.gz')], #~ 7.3 MB
'http://www.w3.org/XML/Test/': [('xmlts20080205', '.tar.gz')]
}
compressed_dir_path = join(test_path, 'compressed_files')
if force_download is True:
if lfs.exists(compressed_dir_path):
print 'Remove compressed directory ', compressed_dir_path
lfs.remove(compressed_dir_path)
for names in uris.itervalues():
for (name, ext) in names:
path = join(test_path, name)
if lfs.exists(path):
print 'Remove %s file' % path
lfs.remove(path)
# test directory
if lfs.exists(test_path) is False:
lfs.make_folder(test_path)
# compressed directory
if lfs.exists(compressed_dir_path) is False:
lfs.make_folder(compressed_dir_path)
else:
lfs.open(compressed_dir_path)
test_dir_filenames = lfs.get_names(test_path)
for base_uri, names in uris.iteritems():
for (name, ext) in names:
if test_dir_filenames.count(name):
continue
compressed_dest = join(compressed_dir_path, '%s%s' % (name, ext))
# check if tarball already exists
if lfs.exists(compressed_dest) is False:
src = join(base_uri, '%s%s' % (name, ext))
print 'GET %s file' % src
dest = join(test_path, name)
if vfs.exists(src) is False:
print "%s uri does not exists" % src
continue
src_file = vfs.open(src)
# save Gzip file
compressed_dest_file = lfs.make_file(compressed_dest)
compressed_dest_file.write(src_file.read())
compressed_dest_file.close()
src_file.close()
print 'Extract file %s' % compressed_dest
# Uncompressed File Path
if name == 'xmlts20080205':
# uncompress only xmlconf.xml file
tar = open_tar(compressed_dest)
xmlconf_file = tar.extractfile('xmlconf/xmlconf.xml')
ucf_path = join(test_path, name)
ucf_file = lfs.make_file(ucf_path)
ucf_file.write(xmlconf_file.read())
ucf_file.close()
else:
# untar Gzip file
compressed_dest_file = lfs.open(compressed_dest)
gzip_file = GzipFile(compressed_dest)
ucf_path = join(test_path, name)
ucf_file = lfs.make_file(ucf_path)
ucf_file.write(gzip_file.read())
compressed_dest_file.close()
gzip_file.close()
ucf_file.close()
tests = []
# update test dir name
test_dir_filenames = lfs.get_names(test_path)
for filename in test_dir_filenames:
real_path = join(test_path, filename)
if lfs.is_file(real_path):
bytes = lfs.get_size(real_path)
tests.append((real_path, filename, bytes,
get_string_size(bytes)))
tests.sort(key=lambda x: x[2])
return tests
"Modifies the copyright notice in the given FILES. Uses 'git blame'"
" to figure out the authors.")
parser = OptionParser(usage, version=version, description=description)
parser.add_option('-f', '--forget', action='store_true', default=False,
help='forget the old copyright (a more aggresive approach)')
options, args = parser.parse_args()
if len(args) == 0:
parser.error('incorrect number of arguments')
# Load CREDITS.txt
credits_names = {} # Mapping from canonical email to real full name
credits_mails = {} # Mapping from secondary email to canonical email
emails = []
if lfs.exists('CREDITS.txt'):
for line in lfs.open('CREDITS.txt').readlines():
line = line.strip()
if line.startswith('#'):
continue
if line:
key, value = line.split(':', 1)
value = value.lstrip()
if key == 'N':
name = value
elif key == 'E':
emails.append(value)
if len(emails) == 1:
credits_names[value] = name
else:
credits_mails[value] = emails[0]
else:
def test_span(self):
handler = lfs.open('pml/span.xml')
story, stylesheet = pmltopdf_test(handler)
self.assertEqual(len(story), 9)
def test_paragraph_cjk(self):
handler = lfs.open('pml/paragraph_cjk.xml')
story, stylesheet = pmltopdf_test(handler)
self.assertEqual(len(story), 12)
def update_locale(srx_handler, exclude_folders, no_wrap=False):
# Read configuration for languages
config = get_config()
src_language = config.get_value('source_language', default='en')
# Get local folder
package_root = config.get_value('package_root')
if lfs.exists(package_root):
locale_folder_path = Path('{0}/locale'.format(package_root))
else:
locale_folder_path = Path('locale/')
locale_folder = lfs.open(locale_folder_path)
# Initialize message catalog
po = POFile()
lines = []
for line in open('MANIFEST').readlines():
line = line.strip()
exclude_folder = False
for x in exclude_folders:
if line.startswith(x):
exclude_folder = True
break
if exclude_folder is False:
lines.append(line)
# Process Python and HTML files
write('* Extract text strings')
extensions = [
'.py',
'.js',
'.xhtml.%s' % src_language,
'.xml.%s' % src_language,
'.html.%s' % src_language]
for path in lines:
# Filter files
for extension in extensions:
if path.endswith(extension):
break
else:
continue
# Get the units
write('.')
try:
handler = ro_database.get_handler(path)
except Exception:
print
print '*'
print '* Error:', path
print '*'
raise
try:
units = handler.get_units(srx_handler=srx_handler)
units = list(units)
except Exception:
print
print '*'
print '* Error:', path
print '*'
raise
relative_path = locale_folder_path.get_pathto(path)
for source, context, line in units:
po.add_unit(relative_path, source, context, line)
print
write('* Update PO template ')
data = po.to_str()
# Write the po into the locale.pot
try:
locale_pot = locale_folder.open('locale.pot', WRITE)
except IOError:
# The locale.pot file does not exist create and open
locale_pot = locale_folder.make_file('locale.pot')
else:
with locale_pot:
locale_pot.write(data)
# Update PO files
filenames = set([ x for x in locale_folder.get_names() if x[-3:] == '.po' ])
filenames.add('%s.po' % src_language)
for language in config.get_value('target_languages'):
filenames.add('%s.po' % language)
filenames = list(filenames)
filenames.sort()
print '* Update PO files:'
locale_pot_path = locale_folder.get_absolute_path('locale.pot')
for filename in filenames:
if locale_folder.exists(filename):
write(' %s ' % filename)
file_path = locale_folder.get_absolute_path(filename)
if no_wrap:
call(['msgmerge', '--no-wrap', '-U', '-s', file_path, locale_pot_path])
else:
call(['msgmerge', '-U', '-s', file_path, locale_pot_path])
else:
print ' %s (new)' % filename
file_path = locale_folder.get_absolute_path(filename)
lfs.copy(locale_pot_path, file_path)
print
def read_file(context, uri, file):
uri = get_reference(uri)
# Shortcuts
elements = context['elements']
references = context['references']
# XML stream
stream = XMLParser(file.read())
# Parse !
stack = []
for type, value, line in stream:
# Set the good encoding
if type == XML_DECL:
context['encoding'] = value[1]
elif type == START_ELEMENT:
tag_uri, tag_name, attrs = value
# Set your context variable
read_common_attrs(tag_uri, attrs, context)
# Only RNG !
if tag_uri == rng_uri:
# <element>
if tag_name == 'element':
# Create a new element
qnames = read_qnames(attrs, context, stream)
element = {
'qnames': qnames,
'attributes': [],
'data': None,
'is_empty': True,
'refs': []
}
# My father has at least a child, me!
if stack:
stack[-1]['is_empty'] = False
# And push it
elements.append(element)
stack.append(element)
# <attribute>
elif tag_name == 'attribute':
# Create a new attribute
qnames = read_qnames(attrs, context, stream)
attribute = {'qnames': qnames, 'data': None, 'refs': []}
# And push it
stack[-1]['attributes'].append(attribute)
stack.append(attribute)
# <data>
elif tag_name == 'data':
type = attrs[None, 'type']
last = stack[-1]
if last['data'] is not None:
last['data'] = ''
else:
last['data'] = type
# <ref>
elif tag_name == 'ref':
name = attrs[None, 'name']
if stack:
stack[-1]['refs'].append(name)
# <define>
elif tag_name == 'define':
name = attrs[None, 'name']
# New or merge ?
if name in references and (None, 'combine') in attrs:
ref = references[name]
else:
ref = {
'attributes': [],
'data': None,
'is_empty': True,
'refs': []
}
references[name] = ref
stack.append(ref)
# <text>
elif tag_name == 'text':
last = stack[-1]
if last['data'] is not None:
last['data'] = ''
else:
last['data'] = 'string'
# <value>
elif tag_name == 'value':
stack[-1]['data'] = ''
# <include>
elif tag_name == 'include':
href = attrs[None, 'href']
include_uri = uri.resolve(href)
include_uri = str(include_uri)
include_file = lfs.open(include_uri)
read_file(context, include_uri, include_file)
# Ignored tags
elif tag_name in [
'grammar', 'start', 'choice', 'optional', 'zeroOrMore',
'oneOrMore', 'group', 'empty', 'interleave', 'param',
'list', 'mixed'
]:
continue
# Tags not implemented
else:
raise NotImplementedError, ('relax NG: <%s> not '
'implemented') % tag_name
elif type == END_ELEMENT:
tag_uri, tag_name = value
if (tag_uri == rng_uri
and tag_name in ['element', 'attribute', 'define']):
stack.pop()
print '* Error:', path
print '*'
raise
relative_path = Path('..').resolve2(path)
for source, context, line in units:
po.add_unit(relative_path, source, context, line)
print
# Update locale.pot
if not lfs.exists('locale/locale.pot'):
lfs.make_file('locale/locale.pot')
write('* Update PO template ')
data = po.to_str()
file = lfs.open('locale/locale.pot', WRITE)
try:
file.write(data)
finally:
file.close()
print
# Update PO files
folder = lfs.open('locale')
filenames = set([ x for x in folder.get_names() if x[-3:] == '.po' ])
filenames.add('%s.po' % src_language)
for language in config.get_value('target_languages'):
filenames.add('%s.po' % language)
filenames = list(filenames)
filenames.sort()
options, args = parser.parse_args()
if len(args) != 0:
parser.error('incorrect number of arguments')
# Read configuration for languages
config = get_config()
src_language = config.get_value('source_language', default='en')
# Get local folder
package_root = config.get_value('package_root')
if lfs.exists(package_root):
locale_folder_path = Path('{0}/locale'.format(package_root))
else:
locale_folder_path = Path('locale/')
locale_folder = lfs.open(locale_folder_path)
# The SRX file
if options.srx is not None:
srx_handler = ro_database.get_handler(options.srx)
else:
srx_handler = None
# Initialize message catalog
po = POFile()
lines = []
for line in open('MANIFEST').readlines():
line = line.strip()
if line.split(sep)[0] not in ('archive', 'docs', 'skeleton', 'test'):
lines.append(line)
def greek(self):
"""Anonymize the ODF file.
"""
# Verify PIL is installed
if PILImage is None:
err = 'The greeking feature requires the Python Imaging Library'
raise ImportError, err
folder = lfs.open(get_abspath('.'))
err = 'Unexpected "%s" file will be omitted from the greeked document'
modified_files = {}
for filename in self.get_contents():
extension = splitext(filename)[1]
startswith = filename.startswith
# Files to keep as they are
# TODO the manifest.xml file should be properly updated
keep = ['mimetype', 'settings.xml', 'META-INF/manifest.xml']
if filename in keep:
pass
# Content, metadata and style
elif filename in ['content.xml', 'meta.xml', 'styles.xml']:
events = self.get_events(filename)
translation = translate(events, GreekCatalog)
modified_files[filename] = stream_to_str(translation)
# Thumbnails
elif startswith('Thumbnails'):
if extension == '.pdf':
modified_files[filename] = folder.open('thumb.pdf').read()
elif extension == '.png':
modified_files[filename] = folder.open('thumb.png').read()
else:
# Unexpected (TODO use the logging system)
modified_files[filename] = None
print err % filename
# SVM files (usually they are in the Pictures folder)
elif extension == '.svm':
modified_files[filename] = folder.open('square.svm').read()
# Pictures
elif startswith('Pictures') or startswith('media'):
# Try with PIL
file = self.get_file(filename)
file = StringIO(file)
image = PILImage.open(file)
format = image.format
image = image.convert('RGB')
image.filename = filename
draw = PILImageDraw.Draw(image)
# Make a cross
h, l = image.size
draw.rectangle((0, 0, h - 1, l - 1),
fill="grey",
outline="black")
draw.line((0, 0, h - 1, l - 1), fill="black")
draw.line((0, l - 1, h - 1, 0), fill="black")
# Save
data = StringIO()
image.save(data, format)
modified_files[filename] = data.getvalue()
# Unexpected (TODO use the logging system)
else:
modified_files[filename] = None
print err % filename
return zip_data(self.data, modified_files)
def __init__(self, path, fields, read_only=False):
if path:
self.fs = lfs.open('{0}/database'.format(path))
else:
self.fs = lfs
def greek(self):
"""Anonymize the ODF file.
"""
# Verify PIL is installed
if PILImage is None:
err = 'The greeking feature requires the Python Imaging Library'
raise ImportError, err
folder = lfs.open(get_abspath('.'))
err = 'Unexpected "%s" file will be omitted from the greeked document'
modified_files = {}
for filename in self.get_contents():
extension = splitext(filename)[1]
startswith = filename.startswith
# Files to keep as they are
# TODO the manifest.xml file should be properly updated
keep = ['mimetype', 'settings.xml', 'META-INF/manifest.xml']
if filename in keep:
pass
# Content, metadata and style
elif filename in ['content.xml', 'meta.xml', 'styles.xml']:
events = self.get_events(filename)
translation = translate(events, GreekCatalog)
modified_files[filename] = stream_to_str(translation)
# Thumbnails
elif startswith('Thumbnails'):
if extension == '.pdf':
modified_files[filename] = folder.open('thumb.pdf').read()
elif extension == '.png':
modified_files[filename] = folder.open('thumb.png').read()
else:
# Unexpected (TODO use the logging system)
modified_files[filename] = None
print err % filename
# SVM files (usually they are in the Pictures folder)
elif extension == '.svm':
modified_files[filename] = folder.open('square.svm').read()
# Pictures
elif startswith('Pictures') or startswith('media'):
# Try with PIL
file = self.get_file(filename)
file = StringIO(file)
image = PILImage.open(file)
format = image.format
image = image.convert('RGB')
image.filename = filename
draw = PILImageDraw.Draw(image)
# Make a cross
h, l = image.size
draw.rectangle((0, 0, h-1, l-1), fill="grey", outline="black")
draw.line((0, 0, h-1, l-1), fill="black")
draw.line((0, l-1, h-1, 0), fill="black")
# Save
data = StringIO()
image.save(data, format)
modified_files[filename] = data.getvalue()
# Unexpected (TODO use the logging system)
else:
modified_files[filename] = None
print err % filename
return zip_data(self.data, modified_files)
def __init__(self, path):
if path:
self.fs = lfs.open('{0}/database'.format(path))
else:
self.fs = lfs
def setUp(self):
self.tests = lfs.open('tests/')
self.tests.make_folder('toto')
def read_file(context, uri, file):
uri = get_reference(uri)
# Shortcuts
elements = context['elements']
references = context['references']
# XML stream
stream = XMLParser(file.read())
# Parse !
stack = []
for type, value, line in stream:
# Set the good encoding
if type == XML_DECL:
context['encoding'] = value[1]
elif type == START_ELEMENT:
tag_uri, tag_name, attrs = value
# Set your context variable
read_common_attrs(tag_uri, attrs, context)
# Only RNG !
if tag_uri == rng_uri:
# <element>
if tag_name == 'element':
# Create a new element
qnames = read_qnames(attrs, context, stream)
element = {'qnames': qnames,
'attributes': [],
'data': None,
'is_empty': True,
'refs': []}
# My father has at least a child, me!
if stack:
stack[-1]['is_empty'] = False
# And push it
elements.append(element)
stack.append(element)
# <attribute>
elif tag_name == 'attribute':
# Create a new attribute
qnames = read_qnames(attrs, context, stream)
attribute = {'qnames': qnames,
'data': None,
'refs': []}
# And push it
stack[-1]['attributes'].append(attribute)
stack.append(attribute)
# <data>
elif tag_name == 'data':
type = attrs[None, 'type']
last = stack[-1]
if last['data'] is not None:
last['data'] = ''
else:
last['data'] = type
# <ref>
elif tag_name == 'ref':
name = attrs[None, 'name']
if stack:
stack[-1]['refs'].append(name)
# <define>
elif tag_name == 'define':
name = attrs[None, 'name']
# New or merge ?
if name in references and (None, 'combine') in attrs:
ref = references[name]
else:
ref = {'attributes': [],
'data': None,
'is_empty': True,
'refs': []}
references[name] = ref
stack.append(ref)
# <text>
elif tag_name == 'text':
last = stack[-1]
if last['data'] is not None:
last['data'] = ''
else:
last['data'] = 'string'
# <value>
elif tag_name == 'value':
stack[-1]['data'] = ''
# <include>
elif tag_name == 'include':
href = attrs[None, 'href']
include_uri = uri.resolve(href)
include_uri = str(include_uri)
include_file = lfs.open(include_uri)
read_file(context, include_uri, include_file)
# Ignored tags
elif tag_name in ['grammar', 'start', 'choice',
'optional', 'zeroOrMore', 'oneOrMore',
'group', 'empty', 'interleave',
'param', 'list', 'mixed']:
continue
# Tags not implemented
else:
raise NotImplementedError, ('relax NG: <%s> not '
'implemented') % tag_name
elif type == END_ELEMENT:
tag_uri, tag_name = value
if (tag_uri == rng_uri and
tag_name in ['element', 'attribute', 'define']):
stack.pop()
def tearDown(self):
folder = lfs.open('tests')
for name in 'toto.txt', 'fofo.txt', 'fofo2.txt', 'empty':
if folder.exists(name):
folder.remove(name)