def start(self, detach=False, profile=False, loop=True): msg = 'Start database %s %s %s' % (detach, profile, loop) log_info(msg) self.profile = '{0}/log/profile'.format(self.target) if profile else None # Daemon mode if detach: become_daemon() # Find out the IP to listen to address = self.config.get_value('listen-address').strip() if not address: raise ValueError, 'listen-address is missing from config.conf' if address == '*': address = None # Check port if self.port is None: raise ValueError('listen-port is missing from config.conf') # Save PID pid = getpid() with open(self.target + '/pid', 'w') as f: f.write(str(pid)) # Call method on root at start with self.database.init_context() as context: context.root.launch_at_start(context) # Listen & set context if not self.read_only: self.launch_cron() self.listen(address, self.port) # XXX The interpreter do not go here #self.server.root.launch_at_stop(context) ## Ok return True
def log_access(self, host, request_line, status_code, body_length): if host: host = host.split(',', 1)[0].strip() now = strftime('%d/%b/%Y:%H:%M:%S') message = '%s - - [%s] "%s" %d %d\n' % (host, now, request_line, status_code, body_length) log_info(message, domain='itools.web_access')
def load(self): path = expanduser('~/.usine') if lfs.is_file(path): log_fatal('ERROR: %s is a file, remove it first' % path) # Make the user configuration file if needed if not lfs.exists(path): log_info('Making the configuration folder: {}'.format(path)) lfs.make_folder(path) log_fatal('Now add the INI files within the folder') # Read the user configuration file ini = [ '%s/%s' % (path, x) for x in lfs.get_names(path) if x[-4:] == '.ini' ] if len(ini) == 0: log_fatal('ERROR: zero INI files found in {}/'.format(path)) # Read the ini file cfg = RawConfigParser() cfg.read(ini) # Get the data for section in cfg._sections: options = cfg._sections[section] type, name = section.split() module = modules[type] obj = module(options) # Keep the data unit self.by_type.setdefault(type, []).append(obj) self.by_type_and_name[(type, name)] = obj # Sort for type in self.by_type: self.by_type[type].sort(key=lambda x: x.name)
def log_access(self, host, request_line, status_code, body_length): if host: host = host.split(',', 1)[0].strip() now = strftime('%d/%b/%Y:%H:%M:%S') message = '%s - - [%s] "%s" %d %d\n' % (host, now, request_line, status_code, body_length) log_info(message, domain='itools.web_access')
def _smtp_send(self): nb_max_mails_to_send = 2 spool = lfs.open(self.spool) def get_names(): # Find out emails to send locks = set() names = set() for name in spool.get_names(): if name == 'failed': # Skip "failed" special directory continue if name[-5:] == '.lock': locks.add(name[:-5]) else: names.add(name) names.difference_update(locks) return names # Send emails names = get_names() smtp_host = self.smtp_host for name in list(names)[:nb_max_mails_to_send]: # 1. Open connection try: smtp = SMTP(smtp_host) except Exception: self.smtp_log_error() spool.move(name, 'failed/%s' % name) return 60 if get_names() else False log_info('CONNECTED to %s' % smtp_host) # 2. Login if self.smtp_login and self.smtp_password: smtp.login(self.smtp_login, self.smtp_password) # 3. Send message try: message = spool.open(name).read() headers = HeaderParser().parsestr(message) subject = headers['subject'] from_addr = headers['from'] to_addr = headers['to'] smtp.sendmail(from_addr, to_addr, message) # Remove spool.remove(name) # Log log_msg = 'Email "%s" sent from "%s" to "%s"' log_info(log_msg % (subject, from_addr, to_addr)) except SMTPRecipientsRefused: # The recipient addresses has been refused self.smtp_log_error() spool.move(name, 'failed/%s' % name) except SMTPResponseException, excp: # The SMTP server returns an error code self.smtp_log_error() spool.move(name, 'failed/%s_%s' % (excp.smtp_code, name)) except Exception: self.smtp_log_error()
def index_document(self, document): self.nb_changes += 1 abspath, term, xdoc = self.get_xdoc_from_document(document) self._db.replace_document(term, xdoc) if self.logger: log_info(abspath, domain='itools.catalog') if self.root: self.transaction_abspaths.append(abspath)
def stop(self, force=False): msg = 'Stoping server...' log_info(msg) print(msg) # Stop wsgi server if self.wsgi_server: self.wsgi_server.stop() # Close database self.close()
def unindex_document(self, abspath): """Remove the document that has value stored in its abspath. If the document does not exist => no error """ self.nb_changes += 1 data = _reduce_size(_encode(self._fields['abspath'], abspath)) self._db.delete_document('Q' + data) if self.logger: log_info(abspath, domain='itools.catalog')
def wrapper(*args, **kwargs): # Remove action in the function name func_name = func.__name__.split('action_')[1] start_dtime = datetime.now() log_info('Start {} ({})'.format(func_name, start_dtime)) # Function call ! func(*args, **kwargs) duration = datetime.now() - start_dtime log_info('End {} (duration : {})'.format(func_name, duration))
def unindex_document(self, abspath): """Remove the document that has value stored in its abspath. If the document does not exist => no error """ self.nb_changes += 1 data = _reduce_size(_encode(self._fields['abspath'], abspath)) self._db.delete_document('Q' + data) if self.logger: log_info(abspath, domain='itools.catalog')
def action_test(self): """ Test if ikaaro instances of this Python environment are alive""" for ikaaro in config.get_sections_by_type('ikaaro'): if ikaaro.options['pyenv'] == self.name: uri = ikaaro.options['uri'] for i in range(1, 6): try: lfs.open('{}/;_ctrl'.format(uri)) except Exception: log_error('[ERROR {}/5] {}'.format(i, uri)) sleep(0.5) else: log_info('[OK] {}'.format(uri)) break
def check_consistency(self, quick): log_info('Check database consistency') # Check the server is not running if self.read_only: pid = get_pid('%s/pid_ro' % self.target) else: pid = get_pid('%s/pid' % self.target) if pid is not None: msg = '[%s] The Web Server is already running.' % self.target log_warning(msg) print(msg) return False # Ok return True
def action_install(self): """Installs every required package (and dependencies) into the remote virtual environment. """ # Get host host = self.get_host() # Build commands install_command = '%s setup.py --quiet install --force' % self.bin_python pip_install_command = '%s install -r requirements.txt --upgrade' % self.bin_pip prefix = self.options.get('prefix') if prefix: pip_install_command += ' --prefix=%s' % prefix install_command += ' --prefix=%s' % prefix # Get package install paths paths = {} for name, version in self.get_packages(): source = self.get_source(name) if self.is_local: source_path = source.get_path() paths[name] = source_path else: # If remove we need to untar sources log_info('UNTAR sources for {}'.format(name)) source = self.get_source(name) pkgname = source.get_pkgname() host.run('tar xzf %s.tar.gz' % pkgname, '/tmp') source_path = '/tmp/%s' % pkgname paths[name] = source_path # Install for name, path in paths.iteritems(): try: log_info('INSTALL DEPENDENCIES for {}'.format(name)) host.run(pip_install_command, path) except EnvironmentError: # In case there is no requirements.txt log_info('No file requirements.txt found, ignore') pass # Install log_info('INSTALL package {}'.format(name)) host.run(install_command, path) if not self.is_local: # Clean untar sources log_info('DELETE untar sources {}'.format(path)) host.run('rm -rf %s' % path, '/tmp')
def cron_manager(self): database = self.database error = False # Build fake context with database.init_context() as context: context.is_cron = True context.git_message = u'[CRON]' # Go t0 = time() catalog = database.catalog query = RangeQuery('next_time_event', None, context.timestamp) search = database.search(query) if not search: return self.config.get_value('cron-interval') nb = len(search) msg = 'Cron launched for {nb} resources'.format(nb=nb) log_info(msg, domain='itools.cron') for brain in search.get_documents(): tcron0 = time() payload = brain.next_time_event_payload if payload: payload = pickle.loads(payload) resource = database.get_resource(brain.abspath) try: resource.time_event(payload) except Exception: # Log error log_error('Cron error\n' + format_exc()) context.root.alert_on_internal_server_error(context) # Abort changes database.abort_changes() # With error error = True break # Reindex resource without committing values = resource.get_catalog_values() catalog.index_document(values) catalog.save_changes() # Log tcron1 = time() msg = 'Done for %s in %s seconds' % (brain.abspath, tcron1-tcron0) log_info(msg, domain='itools.cron') # Save changes if not error: try: database.save_changes() except Exception: log_error('Cron error on save changes\n' + format_exc()) context.root.alert_on_internal_server_error(context) # Message t1 = time() if not error: msg = '[OK] Cron finished for {nb} resources in {s} seconds'.format(nb=nb, s=t1-t0) else: msg = '[ERROR] Cron finished for {nb} resources in {s} seconds'.format(nb=nb, s=t1-t0) log_info(msg, domain='itools.cron') # Again, and again return self.config.get_value('cron-interval')
def listen(self, address, port): # Language negotiation init_language_selector(select_language) # Say hello address = address if address is not None else '*' print 'Listen %s:%d' % (address, port) msg = 'Listing at port %s' % port log_info(msg) self.port = port # Say hello address = address if address is not None else '*' msg = 'Listen %s:%d' % (address, port) log_info(msg) self.port = port self.wsgi_server = WSGIServer( ('', port), application, handler_class=ServerHandler, log=self.access_log) gevent_signal(SIGTERM, self.stop) gevent_signal(SIGINT, self.stop) if self.profile: runctx("self.wsgi_server.serve_forever()", globals(), locals(), self.profile) else: self.wsgi_server.serve_forever()
class Server(WebServer): def __init__(self, target, read_only=False, cache_size=None, profile_space=False): target = lfs.get_absolute_path(target) self.target = target # Load the config config = get_config(target) self.config = config load_modules(config) # Contact Email self.smtp_from = config.get_value('smtp-from') # Full-text indexing self.index_text = config.get_value('index-text', type=Boolean, default=True) # Profile Memory if profile_space is True: import guppy.heapy.RM # The database if cache_size is None: cache_size = config.get_value('database-size') if ':' in cache_size: size_min, size_max = cache_size.split(':') else: size_min = size_max = cache_size size_min, size_max = int(size_min), int(size_max) read_only = read_only or config.get_value('database-readonly') database = get_database(target, size_min, size_max, read_only) self.database = database # Find out the root class root = get_root(database) # Initialize access_log = '%s/log/access' % target super(Server, self).__init__(root, access_log=access_log) # Email service self.spool = lfs.resolve2(self.target, 'spool') spool_failed = '%s/failed' % self.spool if not lfs.exists(spool_failed): lfs.make_folder(spool_failed) # Configuration variables get_value = config.get_value self.smtp_host = get_value('smtp-host') self.smtp_login = get_value('smtp-login', default='').strip() self.smtp_password = get_value('smtp-password', default='').strip() # Email is sent asynchronously self.flush_spool() # Logging log_file = '%s/log/events' % target log_level = config.get_value('log-level') if log_level not in log_levels: msg = 'configuraion error, unexpected "%s" value for log-level' raise ValueError, msg % log_level log_level = log_levels[log_level] logger = Logger(log_file, log_level, rotate=timedelta(weeks=3)) register_logger(logger, None) logger = WebLogger(log_file, log_level) register_logger(logger, 'itools.web') # Session timeout self.session_timeout = get_value('session-timeout') def get_pid(self): return get_pid('%s/pid' % self.target) def set_context(self, path, context): context = super(Server, self).set_context(path, context) context.database = self.database def listen(self, address, port): super(Server, self).listen(address, port) # Set ui context = StaticContext(local_path=get_abspath('ui')) self.set_context('/ui', context) for name in skin_registry: skin = skin_registry[name] context = StaticContext(local_path=skin.key) self.set_context('/ui/%s' % name, context) def is_running_in_rw_mode(self): address = self.config.get_value('listen-address').strip() if address == '*': address = '127.0.0.1' port = self.config.get_value('listen-port') url = 'http://%s:%s/;_ctrl' % (address, port) try: h = vfs.open(url) except GError: # The server is not running return False data = h.read() return json.loads(data)['read-only'] is False ####################################################################### # Mailing ####################################################################### def get_spool_size(self): spool = lfs.open(self.spool) # We have always a 'failed' directory => "-1" return len(spool.get_names()) - 1 def save_email(self, message): # Check the SMTP host is defined if not self.smtp_host: raise ValueError, '"smtp-host" is not set in config.conf' spool = lfs.resolve2(self.target, 'spool') tmp_file, tmp_path = mkstemp(dir=spool) file = fdopen(tmp_file, 'w') try: file.write(message.as_string()) finally: file.close() def flush_spool(self): cron(self._smtp_send, timedelta(seconds=1)) def send_email(self, message): self.save_email(message) self.flush_spool() def _smtp_send(self): nb_max_mails_to_send = 2 spool = lfs.open(self.spool) def get_names(): # Find out emails to send locks = set() names = set() for name in spool.get_names(): if name == 'failed': # Skip "failed" special directory continue if name[-5:] == '.lock': locks.add(name[:-5]) else: names.add(name) names.difference_update(locks) return names # Send emails names = get_names() smtp_host = self.smtp_host for name in list(names)[:nb_max_mails_to_send]: # 1. Open connection try: smtp = SMTP(smtp_host) except gaierror, excp: log_warning('%s: "%s"' % (excp[1], smtp_host)) return 60 # 1 minute except Exception: self.smtp_log_error() return 60 # 1 minute log_info('CONNECTED to %s' % smtp_host) # 2. Login if self.smtp_login and self.smtp_password: smtp.login(self.smtp_login, self.smtp_password) # 3. Send message try: message = spool.open(name).read() headers = HeaderParser().parsestr(message) subject = headers['subject'] from_addr = headers['from'] to_addr = headers['to'] smtp.sendmail(from_addr, to_addr, message) # Remove spool.remove(name) # Log log_msg = 'Email "%s" sent from "%s" to "%s"' log_info(log_msg % (subject, from_addr, to_addr)) except SMTPRecipientsRefused: # The recipient addresses has been refused self.smtp_log_error() spool.move(name, 'failed/%s' % name) except SMTPResponseException, excp: # The SMTP server returns an error code self.smtp_log_error() spool.move(name, 'failed/%s_%s' % (excp.smtp_code, name))
class Server(WebServer): timestamp = None port = None environment = {} modules = [] def __init__(self, target, read_only=False, cache_size=None, profile_space=False): target = lfs.get_absolute_path(target) self.target = target self.read_only = read_only # Set timestamp self.timestamp = str(int(time() / 2)) # Load the config config = get_config(target) self.config = config load_modules(config) self.modules = config.get_value('modules') # Contact Email self.smtp_from = config.get_value('smtp-from') # Full-text indexing self.index_text = config.get_value('index-text', type=Boolean, default=True) # Accept cors self.accept_cors = config.get_value( 'accept-cors', type=Boolean, default=False) # Profile Memory if profile_space is True: import guppy.heapy.RM # The database if cache_size is None: cache_size = config.get_value('database-size') if ':' in cache_size: size_min, size_max = cache_size.split(':') else: size_min = size_max = cache_size size_min, size_max = int(size_min), int(size_max) read_only = read_only or config.get_value('database-readonly') database = get_database(target, size_min, size_max, read_only) self.database = database # Find out the root class root = get_root(database) # Load environment file root_file_path = inspect.getfile(root.__class__) environement_path = str(get_reference(root_file_path).resolve('environment.json')) if vfs.exists(environement_path): with open(environement_path, 'r') as f: data = f.read() self.environment = json.loads(data) # Init fake context context = get_fake_context(database, root.context_cls) context.server = self # Initialize access_log = '%s/log/access' % target super(Server, self).__init__(root, access_log=access_log) # Email service self.spool = lfs.resolve2(self.target, 'spool') spool_failed = '%s/failed' % self.spool if not lfs.exists(spool_failed): lfs.make_folder(spool_failed) # Configuration variables get_value = config.get_value self.smtp_host = get_value('smtp-host') self.smtp_login = get_value('smtp-login', default='').strip() self.smtp_password = get_value('smtp-password', default='').strip() # Email is sent asynchronously self.flush_spool() # Logging log_file = '%s/log/events' % target log_level = config.get_value('log-level') if log_level not in log_levels: msg = 'configuraion error, unexpected "%s" value for log-level' raise ValueError, msg % log_level log_level = log_levels[log_level] logger = Logger(log_file, log_level, rotate=timedelta(weeks=3)) register_logger(logger, None) logger = WebLogger(log_file, log_level) register_logger(logger, 'itools.web') # Session timeout self.session_timeout = get_value('session-timeout') # Register routes self.register_dispatch_routes() def check_consistency(self, quick): # Check the server is not running pid = get_pid('%s/pid' % self.target) if pid is not None: print '[%s] The Web Server is already running.' % self.target return False # Check for database consistency if quick is False and check_database(self.target) is False: return False # Check instance is up to date if not is_instance_up_to_date(self.target): print 'The instance is not up-to-date, please type:' print print ' $ icms-update.py %s' % self.target print return False return True def start(self, detach=False, profile=False, loop=True): profile = ('%s/log/profile' % self.target) if profile else None self.loop = ServerLoop( target=self.target, server=self, profile=profile) # Daemon mode if detach: become_daemon() # Update Git tree-cache, to speed things up self.database.worktree.update_tree_cache() # Find out the IP to listen to address = self.config.get_value('listen-address').strip() if not address: raise ValueError, 'listen-address is missing from config.conf' if address == '*': address = None # Find out the port to listen port = self.config.get_value('listen-port') if port is None: raise ValueError, 'listen-port is missing from config.conf' # Listen & set context root = self.root self.listen(address, port) # Call method on root at start context = get_context() root.launch_at_start(context) # Set cron interval interval = self.config.get_value('cron-interval') if interval: cron(self.cron_manager, interval) # Init loop if loop: try: self.loop.run() except KeyboardInterrupt: self.close() # Ok return True def reindex_catalog(self, quiet=False, quick=False, as_test=False): if self.is_running_in_rw_mode(): print 'Cannot proceed, the server is running in read-write mode.' return # Check for database consistency if quick is False and check_database(self.target) is False: return False # Create a temporary new catalog catalog_path = '%s/catalog.new' % self.target if lfs.exists(catalog_path): lfs.remove(catalog_path) catalog = make_catalog(catalog_path, get_register_fields()) # Get the root root = self.root # Build a fake context context = self.get_fake_context() # Update t0, v0 = time(), vmsize() doc_n = 0 error_detected = False if as_test: log = open('%s/log/update-catalog' % self.target, 'w').write for obj in root.traverse_resources(): if not isinstance(obj, Resource): continue if not quiet: print doc_n, obj.abspath doc_n += 1 context.resource = obj # Index the document try: catalog.index_document(obj) except Exception: if as_test: error_detected = True log('*** Error detected ***\n') log('Abspath of the resource: %r\n\n' % str(obj.abspath)) log(format_exc()) log('\n') else: raise # Free Memory del obj self.database.make_room() if not error_detected: if as_test: # Delete the empty log file remove('%s/log/update-catalog' % self.target) # Update / Report t1, v1 = time(), vmsize() v = (v1 - v0)/1024 print '[Update] Time: %.02f seconds. Memory: %s Kb' % (t1 - t0, v) # Commit print '[Commit]', sys.stdout.flush() catalog.save_changes() # Commit / Replace old_catalog_path = '%s/catalog' % self.target if lfs.exists(old_catalog_path): lfs.remove(old_catalog_path) lfs.move(catalog_path, old_catalog_path) # Commit / Report t2, v2 = time(), vmsize() v = (v2 - v1)/1024 print 'Time: %.02f seconds. Memory: %s Kb' % (t2 - t1, v) return True else: print '[Update] Error(s) detected, the new catalog was NOT saved' print ('[Update] You can find more infos in %r' % join(self.target, 'log/update-catalog')) return False def get_pid(self): return get_pid('%s/pid' % self.target) def is_running(self): pid = self.get_pid() return pid_exists(pid) def stop(self, force=False): proxy = super(Server, self) proxy.stop() print 'Stoping server...' self.kill(force) def kill(self, force=False): pid = get_pid('%s/pid' % self.target) if pid is None: print '[%s] Web Server not running.' % self.target else: signal = SIGTERM if force else SIGINT kill(pid, signal) if force: print '[%s] Web Server shutting down...' % self.target else: print '[%s] Web Server shutting down (gracefully)...' % self.target def listen(self, address, port): super(Server, self).listen(address, port) self.port = port def save_running_informations(self): # Save server running informations kw = {'pid': getpid(), 'target': self.target, 'read_only': self.read_only} data = pickle.dumps(kw) with open(self.target + '/running', 'w') as output_file: output_file.write(data) def get_running_informations(self): try: with open(self.target + '/running', 'r') as output_file: data = output_file.read() return pickle.loads(data) except IOError: return None def is_running_in_rw_mode(self, mode='running'): is_running = self.is_running() if not is_running: return False if mode == 'request': address = self.config.get_value('listen-address').strip() if address == '*': address = '127.0.0.1' port = self.config.get_value('listen-port') url = 'http://%s:%s/;_ctrl' % (address, port) try: h = vfs.open(url) except GError: # The server is not running return False data = h.read() return json.loads(data)['read-only'] is False elif mode == 'running': kw = self.get_running_informations() return not kw.get('read_only', False) ####################################################################### # Mailing ####################################################################### def get_spool_size(self): spool = lfs.open(self.spool) # We have always a 'failed' directory => "-1" return len(spool.get_names()) - 1 def save_email(self, message): # Check the SMTP host is defined if not self.smtp_host: raise ValueError, '"smtp-host" is not set in config.conf' spool = lfs.resolve2(self.target, 'spool') tmp_file, tmp_path = mkstemp(dir=spool) file = fdopen(tmp_file, 'w') try: file.write(message.as_string()) finally: file.close() def flush_spool(self): cron(self._smtp_send, timedelta(seconds=1)) def send_email(self, message): self.save_email(message) self.flush_spool() def _smtp_send(self): nb_max_mails_to_send = 2 spool = lfs.open(self.spool) def get_names(): # Find out emails to send locks = set() names = set() for name in spool.get_names(): if name == 'failed': # Skip "failed" special directory continue if name[-5:] == '.lock': locks.add(name[:-5]) else: names.add(name) names.difference_update(locks) return names # Send emails names = get_names() smtp_host = self.smtp_host for name in list(names)[:nb_max_mails_to_send]: # 1. Open connection try: smtp = SMTP(smtp_host) except gaierror, excp: log_warning('%s: "%s"' % (excp[1], smtp_host)) return 60 # 1 minute except Exception: self.smtp_log_error() return 60 # 1 minute log_info('CONNECTED to %s' % smtp_host) # 2. Login if self.smtp_login and self.smtp_password: smtp.login(self.smtp_login, self.smtp_password) # 3. Send message try: message = spool.open(name).read() headers = HeaderParser().parsestr(message) subject = headers['subject'] from_addr = headers['from'] to_addr = headers['to'] smtp.sendmail(from_addr, to_addr, message) # Remove spool.remove(name) # Log log_msg = 'Email "%s" sent from "%s" to "%s"' log_info(log_msg % (subject, from_addr, to_addr)) except SMTPRecipientsRefused: # The recipient addresses has been refused self.smtp_log_error() spool.move(name, 'failed/%s' % name) except SMTPResponseException, excp: # The SMTP server returns an error code self.smtp_log_error() spool.move(name, 'failed/%s_%s' % (excp.smtp_code, name))
def stop_server(target): msg = 'Stoping server...' log_info(msg) pid = get_pid('%s/pid' % target) if pid: kill(pid, SIGTERM)
exit(1) config.options = options # Case 0: Nothing, print help if not args: print 'Usage:', usage print print 'Modules:' print for name in sorted(modules): module = modules[name] if module.class_title: print u' %s: %s' % (name, module.class_title) exit(0) log_info('> Command : ' + ' '.join(args)) # Get the module module_name, args = args[0], args[1:] module = modules.get(module_name) if not module or not module.class_title: print 'Error: unexpected "%s" module' % module_name exit(1) # Update configuration if module_name == 'config': config.action_update() exit(1) # Case 1: Just the module, print help if not args:
def reindex_catalog(self, quiet=False, quick=False, as_test=False): # FIXME: should be moved into backend from itools.database.backends.catalog import make_catalog msg = 'reindex catalog %s %s %s' % (quiet, quick, as_test) log_info(msg) if self.is_running_in_rw_mode(): print 'Cannot proceed, the server is running in read-write mode.' return # Create a temporary new catalog catalog_path = '%s/catalog.new' % self.target if lfs.exists(catalog_path): lfs.remove(catalog_path) catalog = make_catalog(catalog_path, get_register_fields()) # Get the root root = self.root # Update t0, v0 = time(), vmsize() doc_n = 0 error_detected = False if as_test: log = open('%s/log/update-catalog' % self.target, 'w').write with self.database.init_context() as context: for obj in root.traverse_resources(): if not quiet or doc_n % 10000==0: print('{0} {1}'.format(doc_n, obj.abspath)) doc_n += 1 context.resource = obj values = obj.get_catalog_values() # Index the document try: catalog.index_document(values) except Exception: if as_test: error_detected = True log('*** Error detected ***\n') log('Abspath of the resource: %r\n\n' % str(obj.abspath)) log(format_exc()) log('\n') else: raise # Free Memory del obj self.database.make_room() if not error_detected: if as_test: # Delete the empty log file remove('%s/log/update-catalog' % self.target) # Update / Report t1, v1 = time(), vmsize() v = (v1 - v0)/1024 print '[Update] Time: %.02f seconds. Memory: %s Kb' % (t1 - t0, v) # Commit print '[Commit]', sys.stdout.flush() catalog.save_changes() catalog.close() # Commit / Replace old_catalog_path = '%s/catalog' % self.target if lfs.exists(old_catalog_path): lfs.remove(old_catalog_path) lfs.move(catalog_path, old_catalog_path) # Commit / Report t2, v2 = time(), vmsize() v = (v2 - v1)/1024 print 'Time: %.02f seconds. Memory: %s Kb' % (t2 - t1, v) return True else: print '[Update] Error(s) detected, the new catalog was NOT saved' print ('[Update] You can find more infos in %r' % join(self.target, 'log/update-catalog')) return False
def close(self): log_info('Close server') self.database.close()
def index_document(self, document): self.nb_changes += 1 abspath, term, xdoc = self.get_xdoc_from_document(document) self._db.replace_document(term, xdoc) if self.logger: log_info(abspath, domain='itools.catalog')