parser_names = ['expat', 'itools'] else: parser_names = ['itools', 'expat'] # parser_names = ['itools_c', 'itools'] output_init(parser_names); # Go for real_path, filename, file_bytes, file_size in filenames: nb_repeat = get_clock_nb_pass(file_bytes) nb_repeat_float = float(nb_repeat) nb_repeat = str(nb_repeat) test_results = [] for parser_name in parser_names: # Run script = parser_scripts[parser_name] v0 = vmsize() t0 = get_time_spent(mode='both') return_code = call([script, real_path, nb_repeat]) t1 = get_time_spent(mode='both', since=t0) v1 = vmsize() # Append if return_code == 0: time_spent = t1 - t0 memo = v1 - v0 time_spent = get_string_time(time_spent / nb_repeat_float) test_results.append((parser_name, (time_spent, memo))) else: test_results.append((parser_name, None)) output_result(test_results, (filename, file_bytes)) print
def reindex_catalog(self, quiet=False, quick=False, as_test=False): if self.is_running_in_rw_mode(): print 'Cannot proceed, the server is running in read-write mode.' return # Check for database consistency if quick is False and check_database(self.target) is False: return False # Create a temporary new catalog catalog_path = '%s/catalog.new' % self.target if lfs.exists(catalog_path): lfs.remove(catalog_path) catalog = make_catalog(catalog_path, get_register_fields()) # Get the root root = self.root # Build a fake context context = self.get_fake_context() # Update t0, v0 = time(), vmsize() doc_n = 0 error_detected = False if as_test: log = open('%s/log/update-catalog' % self.target, 'w').write for obj in root.traverse_resources(): if not isinstance(obj, Resource): continue if not quiet: print doc_n, obj.abspath doc_n += 1 context.resource = obj # Index the document try: catalog.index_document(obj) except Exception: if as_test: error_detected = True log('*** Error detected ***\n') log('Abspath of the resource: %r\n\n' % str(obj.abspath)) log(format_exc()) log('\n') else: raise # Free Memory del obj self.database.make_room() if not error_detected: if as_test: # Delete the empty log file remove('%s/log/update-catalog' % self.target) # Update / Report t1, v1 = time(), vmsize() v = (v1 - v0)/1024 print '[Update] Time: %.02f seconds. Memory: %s Kb' % (t1 - t0, v) # Commit print '[Commit]', sys.stdout.flush() catalog.save_changes() # Commit / Replace old_catalog_path = '%s/catalog' % self.target if lfs.exists(old_catalog_path): lfs.remove(old_catalog_path) lfs.move(catalog_path, old_catalog_path) # Commit / Report t2, v2 = time(), vmsize() v = (v2 - v1)/1024 print 'Time: %.02f seconds. Memory: %s Kb' % (t2 - t1, v) return True else: print '[Update] Error(s) detected, the new catalog was NOT saved' print ('[Update] You can find more infos in %r' % join(self.target, 'log/update-catalog')) return False
def reindex_catalog(self, quiet=False, quick=False, as_test=False): # FIXME: should be moved into backend from itools.database.backends.catalog import make_catalog msg = 'reindex catalog %s %s %s' % (quiet, quick, as_test) log_info(msg) if self.is_running_in_rw_mode(): print 'Cannot proceed, the server is running in read-write mode.' return # Create a temporary new catalog catalog_path = '%s/catalog.new' % self.target if lfs.exists(catalog_path): lfs.remove(catalog_path) catalog = make_catalog(catalog_path, get_register_fields()) # Get the root root = self.root # Update t0, v0 = time(), vmsize() doc_n = 0 error_detected = False if as_test: log = open('%s/log/update-catalog' % self.target, 'w').write with self.database.init_context() as context: for obj in root.traverse_resources(): if not quiet or doc_n % 10000==0: print('{0} {1}'.format(doc_n, obj.abspath)) doc_n += 1 context.resource = obj values = obj.get_catalog_values() # Index the document try: catalog.index_document(values) except Exception: if as_test: error_detected = True log('*** Error detected ***\n') log('Abspath of the resource: %r\n\n' % str(obj.abspath)) log(format_exc()) log('\n') else: raise # Free Memory del obj self.database.make_room() if not error_detected: if as_test: # Delete the empty log file remove('%s/log/update-catalog' % self.target) # Update / Report t1, v1 = time(), vmsize() v = (v1 - v0)/1024 print '[Update] Time: %.02f seconds. Memory: %s Kb' % (t1 - t0, v) # Commit print '[Commit]', sys.stdout.flush() catalog.save_changes() catalog.close() # Commit / Replace old_catalog_path = '%s/catalog' % self.target if lfs.exists(old_catalog_path): lfs.remove(old_catalog_path) lfs.move(catalog_path, old_catalog_path) # Commit / Report t2, v2 = time(), vmsize() v = (v2 - v1)/1024 print 'Time: %.02f seconds. Memory: %s Kb' % (t2 - t1, v) return True else: print '[Update] Error(s) detected, the new catalog was NOT saved' print ('[Update] You can find more infos in %r' % join(self.target, 'log/update-catalog')) return False