parser_names = ['expat', 'itools']
else:
parser_names = ['itools', 'expat']
# parser_names = ['itools_c', 'itools']
output_init(parser_names);
# Go
for real_path, filename, file_bytes, file_size in filenames:
nb_repeat = get_clock_nb_pass(file_bytes)
nb_repeat_float = float(nb_repeat)
nb_repeat = str(nb_repeat)
test_results = []
for parser_name in parser_names:
# Run
script = parser_scripts[parser_name]
v0 = vmsize()
t0 = get_time_spent(mode='both')
return_code = call([script, real_path, nb_repeat])
t1 = get_time_spent(mode='both', since=t0)
v1 = vmsize()
# Append
if return_code == 0:
time_spent = t1 - t0
memo = v1 - v0
time_spent = get_string_time(time_spent / nb_repeat_float)
test_results.append((parser_name, (time_spent, memo)))
else:
test_results.append((parser_name, None))
output_result(test_results, (filename, file_bytes))
print
def reindex_catalog(self, quiet=False, quick=False, as_test=False):
if self.is_running_in_rw_mode():
print 'Cannot proceed, the server is running in read-write mode.'
return
# Check for database consistency
if quick is False and check_database(self.target) is False:
return False
# Create a temporary new catalog
catalog_path = '%s/catalog.new' % self.target
if lfs.exists(catalog_path):
lfs.remove(catalog_path)
catalog = make_catalog(catalog_path, get_register_fields())
# Get the root
root = self.root
# Build a fake context
context = self.get_fake_context()
# Update
t0, v0 = time(), vmsize()
doc_n = 0
error_detected = False
if as_test:
log = open('%s/log/update-catalog' % self.target, 'w').write
for obj in root.traverse_resources():
if not isinstance(obj, Resource):
continue
if not quiet:
print doc_n, obj.abspath
doc_n += 1
context.resource = obj
# Index the document
try:
catalog.index_document(obj)
except Exception:
if as_test:
error_detected = True
log('*** Error detected ***\n')
log('Abspath of the resource: %r\n\n' % str(obj.abspath))
log(format_exc())
log('\n')
else:
raise
# Free Memory
del obj
self.database.make_room()
if not error_detected:
if as_test:
# Delete the empty log file
remove('%s/log/update-catalog' % self.target)
# Update / Report
t1, v1 = time(), vmsize()
v = (v1 - v0)/1024
print '[Update] Time: %.02f seconds. Memory: %s Kb' % (t1 - t0, v)
# Commit
print '[Commit]',
sys.stdout.flush()
catalog.save_changes()
# Commit / Replace
old_catalog_path = '%s/catalog' % self.target
if lfs.exists(old_catalog_path):
lfs.remove(old_catalog_path)
lfs.move(catalog_path, old_catalog_path)
# Commit / Report
t2, v2 = time(), vmsize()
v = (v2 - v1)/1024
print 'Time: %.02f seconds. Memory: %s Kb' % (t2 - t1, v)
return True
else:
print '[Update] Error(s) detected, the new catalog was NOT saved'
print ('[Update] You can find more infos in %r' %
join(self.target, 'log/update-catalog'))
return False
def reindex_catalog(self, quiet=False, quick=False, as_test=False):
# FIXME: should be moved into backend
from itools.database.backends.catalog import make_catalog
msg = 'reindex catalog %s %s %s' % (quiet, quick, as_test)
log_info(msg)
if self.is_running_in_rw_mode():
print 'Cannot proceed, the server is running in read-write mode.'
return
# Create a temporary new catalog
catalog_path = '%s/catalog.new' % self.target
if lfs.exists(catalog_path):
lfs.remove(catalog_path)
catalog = make_catalog(catalog_path, get_register_fields())
# Get the root
root = self.root
# Update
t0, v0 = time(), vmsize()
doc_n = 0
error_detected = False
if as_test:
log = open('%s/log/update-catalog' % self.target, 'w').write
with self.database.init_context() as context:
for obj in root.traverse_resources():
if not quiet or doc_n % 10000==0:
print('{0} {1}'.format(doc_n, obj.abspath))
doc_n += 1
context.resource = obj
values = obj.get_catalog_values()
# Index the document
try:
catalog.index_document(values)
except Exception:
if as_test:
error_detected = True
log('*** Error detected ***\n')
log('Abspath of the resource: %r\n\n' % str(obj.abspath))
log(format_exc())
log('\n')
else:
raise
# Free Memory
del obj
self.database.make_room()
if not error_detected:
if as_test:
# Delete the empty log file
remove('%s/log/update-catalog' % self.target)
# Update / Report
t1, v1 = time(), vmsize()
v = (v1 - v0)/1024
print '[Update] Time: %.02f seconds. Memory: %s Kb' % (t1 - t0, v)
# Commit
print '[Commit]',
sys.stdout.flush()
catalog.save_changes()
catalog.close()
# Commit / Replace
old_catalog_path = '%s/catalog' % self.target
if lfs.exists(old_catalog_path):
lfs.remove(old_catalog_path)
lfs.move(catalog_path, old_catalog_path)
# Commit / Report
t2, v2 = time(), vmsize()
v = (v2 - v1)/1024
print 'Time: %.02f seconds. Memory: %s Kb' % (t2 - t1, v)
return True
else:
print '[Update] Error(s) detected, the new catalog was NOT saved'
print ('[Update] You can find more infos in %r' %
join(self.target, 'log/update-catalog'))
return False
