def run(agent_id, force=False): """Runs the livesync agent""" update_session_options(db) if agent_id is None: agent_list = LiveSyncAgent.find_all() else: agent = LiveSyncAgent.find_first(id=int(agent_id)) if agent is None: print 'No such agent' return agent_list = [agent] for agent in agent_list: if agent.backend is None: print cformat('Skipping agent: %{red!}{}%{reset} (backend not found)').format(agent.name) continue if not agent.initial_data_exported and not force: print cformat('Skipping agent: %{red!}{}%{reset} (initial export not performed)').format(agent.name) continue print cformat('Running agent: %{white!}{}%{reset}').format(agent.name) with DBMgr.getInstance().global_connection(): try: agent.create_backend().run() db.session.commit() finally: transaction.abort()
def run(agent_id, force=False): """Runs the livesync agent""" update_session_options(db) if agent_id is None: agent_list = LiveSyncAgent.find_all() else: agent = LiveSyncAgent.find_first(id=int(agent_id)) if agent is None: print 'No such agent' return agent_list = [agent] for agent in agent_list: if agent.backend is None: print cformat( 'Skipping agent: %{red!}{}%{reset} (backend not found)' ).format(agent.name) continue if not agent.initial_data_exported and not force: print cformat( 'Skipping agent: %{red!}{}%{reset} (initial export not performed)' ).format(agent.name) continue print cformat('Running agent: %{white!}{}%{reset}').format(agent.name) with DBMgr.getInstance().global_connection(): try: agent.create_backend().run() db.session.commit() finally: transaction.abort()
def run(agent_id, force=False): """Runs the livesync agent""" if agent_id is None: agent_list = LiveSyncAgent.find_all() else: agent = LiveSyncAgent.find_first(id=agent_id) if agent is None: print 'No such agent' return agent_list = [agent] for agent in agent_list: if agent.backend is None: print cformat( 'Skipping agent: %{red!}{}%{reset} (backend not found)' ).format(agent.name) continue if not agent.initial_data_exported and not force: print cformat( 'Skipping agent: %{red!}{}%{reset} (initial export not performed)' ).format(agent.name) continue print cformat('Running agent: %{white!}{}%{reset}').format(agent.name) try: agent.create_backend().run() db.session.commit() except: db.session.rollback() raise
def migrate_agents(self): print cformat('%{white!}migrating agents') for old_agent in committing_iterator(self.livesync_root['agent_manager']._agents.itervalues()): if not old_agent._active: print cformat('%{yellow}skipping inactive agent {} ({})%{reset}').format(old_agent._id, old_agent._name) continue agent = LiveSyncAgent(name=convert_to_unicode(old_agent._name), initial_data_exported=True) old_agent_class = old_agent.__class__.__name__ if old_agent_class == 'InvenioBatchUploaderAgent': agent.backend_name = 'invenio' agent.settings = { 'server_url': old_agent._url } elif old_agent_class == 'CERNSearchUploadAgent': agent.backend_name = 'cernsearch' agent.settings = { 'server_url': old_agent._url, 'username': old_agent._username, 'password': old_agent._password, } else: print cformat('%{red!}skipping unknown agent type: {}%{reset}').format(old_agent_class) continue print cformat('- %{cyan}{} ({})').format(agent.name, agent.backend_name) db.session.add(agent)
def run(agent_id, force=False): """Runs the livesync agent""" if agent_id is None: agent_list = LiveSyncAgent.find_all() else: agent = LiveSyncAgent.find_first(id=agent_id) if agent is None: print 'No such agent' return agent_list = [agent] for agent in agent_list: if agent.backend is None: print cformat('Skipping agent: %{red!}{}%{reset} (backend not found)').format(agent.name) continue if not agent.initial_data_exported and not force: print cformat('Skipping agent: %{red!}{}%{reset} (initial export not performed)').format(agent.name) continue print cformat('Running agent: %{white!}{}%{reset}').format(agent.name) try: agent.create_backend().run() db.session.commit() except: db.session.rollback() raise
def test_fetch_records(db, dummy_event_new): """Test if the correct records are fetched""" agent = LiveSyncAgent(backend_name='dummy', name='dummy') backend = DummyBackend(agent) db.session.add(agent) queue = [LiveSyncQueueEntry(change=ChangeType.created, type=EntryType.event, event=dummy_event_new, processed=True), LiveSyncQueueEntry(change=ChangeType.created, type=EntryType.event, event=dummy_event_new)] agent.queue = queue db.session.flush() assert backend.fetch_records() == [queue[1]]
def initial_export(agent_id, force=False): """Performs the initial data export for an agent""" update_session_options(db) agent = LiveSyncAgent.find_first(id=int(agent_id)) if agent is None: print 'No such agent' return if agent.backend is None: print cformat('Cannot run agent %{red!}{}%{reset} (backend not found)').format(agent.name) return print cformat('Selected agent: %{white!}{}%{reset} ({})').format(agent.name, agent.backend.title) if agent.initial_data_exported and not force: print 'The initial export has already been performed for this agent.' print cformat('To re-run it, use %{yellow!}--force%{reset}') return def _iter_events(): for i, (_, event) in enumerate(conferenceHolderIterator(ConferenceHolder(), deepness='event'), 1): yield event if i % 1000 == 0: # Clean local ZEO cache transaction.abort() with DBMgr.getInstance().global_connection(): agent.create_backend().run_initial_export(_iter_events()) agent.initial_data_exported = True db.session.commit()
def agents(): """Lists the currently active agents""" print 'The following LiveSync agents are active:' agent_list = LiveSyncAgent.find().order_by( LiveSyncAgent.backend_name, db.func.lower(LiveSyncAgent.name)).all() table_data = [['ID', 'Name', 'Backend', 'Initial Export', 'Queue']] for agent in agent_list: initial = (cformat('%{green!}done%{reset}') if agent.initial_data_exported else cformat('%{yellow!}pending%{reset}')) if agent.backend is None: backend_title = cformat( '%{red!}invalid backend ({})%{reset}').format( agent.backend_name) else: backend_title = agent.backend.title table_data.append([ unicode(agent.id), agent.name, backend_title, initial, unicode(agent.queue.filter_by(processed=False).count()) ]) table = AsciiTable(table_data) table.justify_columns[4] = 'right' print table.table if not all(a.initial_data_exported for a in agent_list): print print "You need to perform the initial data export for some agents." print cformat( "To do so, run " "%{yellow!}indico livesync initial_export %{reset}%{yellow}<agent_id>%{reset} for those agents." )
def create(cls, changes, ref, excluded_categories=set()): """Create a new change in all queues. :param changes: the change types, an iterable containing :class:`ChangeType` :param ref: the object reference (returned by `obj_ref`) of the changed object :param excluded_categories: set of categories (IDs) whose items will not be tracked """ ref = dict(ref) obj = obj_deref(ref) if isinstance(obj, Category): if any(c.id in excluded_categories for c in obj.chain_query): return else: event = obj if isinstance(obj, Event) else obj.event if excluded_categories & set(event.category_chain): return for change in changes: for agent in LiveSyncAgent.find(): entry = cls(agent=agent, change=change, **ref) db.session.add(entry) db.session.flush()
def _process(self): form = self.backend.form(obj=FormDefaults(name=self.backend.title)) if form.validate_on_submit(): data = form.data name = data.pop('name') agent = LiveSyncAgent(name=name, backend_name=self.backend_name, settings=data) db.session.add(agent) flash(_('Agent added'), 'success') flash(_("Don't forget to run the initial export!"), 'highlight') return jsonify_data(flash=False) return jsonify_template('edit_agent.html', render_plugin_template, form=form, backend=self.backend, edit=False)
def migrate_agents(self): print cformat('%{white!}migrating agents') for old_agent in committing_iterator( self.livesync_root['agent_manager']._agents.itervalues()): if not old_agent._active: print cformat( '%{yellow}skipping inactive agent {} ({})%{reset}').format( old_agent._id, old_agent._name) continue agent = LiveSyncAgent(name=convert_to_unicode(old_agent._name), initial_data_exported=True) old_agent_class = old_agent.__class__.__name__ if old_agent_class == 'InvenioBatchUploaderAgent': agent.backend_name = 'invenio' agent.settings = {'server_url': old_agent._url} elif old_agent_class == 'CERNSearchUploadAgent': agent.backend_name = 'cernsearch' agent.settings = { 'server_url': old_agent._url, 'username': old_agent._username, 'password': old_agent._password, } else: print cformat('%{red!}skipping unknown agent type: {}%{reset}' ).format(old_agent_class) continue print cformat('- %{cyan}{} ({})').format(agent.name, agent.backend_name) db.session.add(agent)
def scheduled_update(): from indico_livesync.plugin import LiveSyncPlugin clean_old_entries() for agent in LiveSyncAgent.find_all(): if agent.backend is None: LiveSyncPlugin.logger.warning('Skipping agent %s; backend not found', agent.name) continue if not agent.initial_data_exported: LiveSyncPlugin.logger.warning('Skipping agent %s; initial export not performed yet', agent.name) continue LiveSyncPlugin.logger.info('Running agent %s', agent.name) agent.create_backend().run() db.session.commit()
def create(cls, changes, ref): """Creates a new change in all queues :param changes: the change types, an iterable containing :class:`ChangeType` :param ref: the object reference (returned by `obj_ref`) of the changed object """ ref = dict(ref) for agent in LiveSyncAgent.find(): for change in changes: entry = cls(agent=agent, change=change, **ref) db.session.add(entry) db.session.flush()
def _process(self): form = self.backend.form(obj=FormDefaults(name=self.backend.title)) if form.validate_on_submit(): data = form.data name = data.pop('name') agent = LiveSyncAgent(name=name, backend_name=self.backend_name, settings=data) db.session.add(agent) flash(_('Agent added'), 'success') flash(_("Don't forget to run the initial export!"), 'highlight') return redirect(url_for('plugins.details', plugin='livesync')) return WPLiveSync.render_template('edit_agent.html', form=form, backend=self.backend)
def scheduled_update(): from indico_livesync.plugin import LiveSyncPlugin clean_old_entries() for agent in LiveSyncAgent.find_all(): if agent.backend is None: LiveSyncPlugin.logger.warning( 'Skipping agent %s; backend not found', agent.name) continue if not agent.initial_data_exported: LiveSyncPlugin.logger.warning( 'Skipping agent %s; initial export not performed yet', agent.name) continue LiveSyncPlugin.logger.info('Running agent %s', agent.name) agent.create_backend().run() db.session.commit()
def initial_export(agent_id, force): """Performs the initial data export for an agent""" agent = LiveSyncAgent.get(agent_id) if agent is None: print('No such agent') return if agent.backend is None: print(cformat('Cannot run agent %{red!}{}%{reset} (backend not found)').format(agent.name)) return print(cformat('Selected agent: %{white!}{}%{reset} ({})').format(agent.name, agent.backend.title)) if agent.initial_data_exported and not force: print('The initial export has already been performed for this agent.') print(cformat('To re-run it, use %{yellow!}--force%{reset}')) return agent.create_backend().run_initial_export(Event.query.filter_by(is_deleted=False)) agent.initial_data_exported = True db.session.commit()
def initial_export(agent_id, force): """Performs the initial data export for an agent""" agent = LiveSyncAgent.find_first(id=agent_id) if agent is None: print 'No such agent' return if agent.backend is None: print cformat('Cannot run agent %{red!}{}%{reset} (backend not found)').format(agent.name) return print cformat('Selected agent: %{white!}{}%{reset} ({})').format(agent.name, agent.backend.title) if agent.initial_data_exported and not force: print 'The initial export has already been performed for this agent.' print cformat('To re-run it, use %{yellow!}--force%{reset}') return agent.create_backend().run_initial_export(Event.find(is_deleted=False)) agent.initial_data_exported = True db.session.commit()
def reset(agent_id): """Resets all livesync data for an agent.""" agent = LiveSyncAgent.get(agent_id) if agent is None: print('No such agent') return if agent.backend is None: print(cformat('Cannot run agent %{red!}{}%{reset} (backend not found)').format(agent.name)) return backend = agent.create_backend() reset_allowed, message = backend.check_reset_status() if not reset_allowed: print(f'Resetting is not possible: {message}') return print(cformat('Selected agent: %{white!}{}%{reset} ({})').format(agent.name, backend.title)) print(cformat('%{yellow!}!!! %{red!}DANGER %{yellow!}!!!%{reset}')) if backend.reset_deletes_indexed_data: print(cformat('%{yellow!}This command will delete all indexed data on this backend.%{reset}') .format(backend.title)) else: print(cformat('%{yellow!}This command should only be used if the data on this backend ' 'has been deleted.%{reset}') .format(backend.title)) print(cformat('%{yellow!}After resetting you need to perform a new initial export.%{reset}')) click.confirm(click.style('Do you really want to perform the reset?', fg='red', bold=True), default=False, abort=True) if not config.DEBUG: click.confirm(click.style('Are you absolutely sure?', fg='red', bold=True), default=False, abort=True) for i in range(5): print(cformat('\rResetting in %{white!}{}%{reset}s (CTRL+C to abort)').format(5 - i), end='') time.sleep(1) print('') backend.reset() db.session.commit() print(cformat('Reset complete; run %{green!}indico livesync initial-export {}%{reset} for a new export') .format(agent.id))
def agents(): """Lists the currently active agents""" print 'The following LiveSync agents are active:' agent_list = LiveSyncAgent.find().order_by(LiveSyncAgent.backend_name, db.func.lower(LiveSyncAgent.name)).all() table_data = [['ID', 'Name', 'Backend', 'Initial Export', 'Queue']] for agent in agent_list: initial = (cformat('%{green!}done%{reset}') if agent.initial_data_exported else cformat('%{yellow!}pending%{reset}')) if agent.backend is None: backend_title = cformat('%{red!}invalid backend ({})%{reset}').format(agent.backend_name) else: backend_title = agent.backend.title table_data.append([unicode(agent.id), agent.name, backend_title, initial, unicode(agent.queue.filter_by(processed=False).count())]) table = AsciiTable(table_data) table.justify_columns[4] = 'right' print table.table if not all(a.initial_data_exported for a in agent_list): print print "You need to perform the initial data export for some agents." print cformat("To do so, run " "%{yellow!}indico livesync initial_export %{reset}%{yellow}<agent_id>%{reset} for those agents.")
def initial_export(agent_id, batch, force, verbose, retry): """Performs the initial data export for an agent.""" agent = LiveSyncAgent.get(agent_id) if agent is None: print('No such agent') return if agent.backend is None: print(cformat('Cannot run agent %{red!}{}%{reset} (backend not found)').format(agent.name)) return print(cformat('Selected agent: %{white!}{}%{reset} ({})').format(agent.name, agent.backend.title)) backend = agent.create_backend() if not backend.is_configured(): print(cformat('Agent %{red!}{}%{reset} is not properly configured').format(agent.name)) return if agent.initial_data_exported and not force: print('The initial export has already been performed for this agent.') print(cformat('To re-run it, use %{yellow!}--force%{reset}')) return try: if not backend.run_initial_export(batch, force, verbose): print('The initial export failed; not marking it as done') return except Exception: if not retry: raise traceback.print_exc() print('Restarting in 2 seconds') time.sleep(2) os.execl(sys.argv[0], *sys.argv) return # exec doesn't return but just in case... agent.initial_data_exported = True db.session.commit()
def run(agent_id, force, verbose, allowed_categories): """Runs the livesync agent.""" from indico_livesync.plugin import LiveSyncPlugin if LiveSyncPlugin.settings.get('disable_queue_runs'): print(cformat('%{yellow!}Queue runs are disabled%{reset}')) if LiveSyncPlugin.settings.get('skip_category_changes'): print(cformat('%{yellow!}Category changes are currently being skipped%{reset}')) if allowed_categories: print(cformat('Whitelisted categories: %{green}{}%{reset}') .format(', '.join(map(str, sorted(allowed_categories))))) if agent_id is None: agent_list = LiveSyncAgent.query.all() else: agent = LiveSyncAgent.get(agent_id) if agent is None: print('No such agent') return agent_list = [agent] for agent in agent_list: if agent.backend is None: print(cformat('Skipping agent: %{red!}{}%{reset} (backend not found)').format(agent.name)) continue backend = agent.create_backend() queue_allowed, reason = backend.check_queue_status() if not queue_allowed and not force: print(cformat('Skipping agent: %{red!}{}%{reset} ({})').format(agent.name, reason)) continue print(cformat('Running agent: %{white!}{}%{reset}').format(agent.name)) try: backend.run(verbose, from_cli=True, allowed_categories=allowed_categories) db.session.commit() except Exception: db.session.rollback() raise
def extend_plugin_details(): agents = LiveSyncAgent.find().order_by(LiveSyncAgent.name, LiveSyncAgent.id).all() return render_plugin_template('plugin_details_extra.html', agents=agents, backends=current_plugin.backend_classes)
def _checkParams(self): self.agent = LiveSyncAgent.find_one(id=request.view_args['agent_id']) if self.agent.backend is None: flash(_('Cannot edit an agent that is not loaded'), 'error') return redirect(url_for('plugins.details', plugin='livesync'))
def dummy_agent(db): agent = LiveSyncAgent(backend_name='dummy', name='dummy') db.session.add(agent) return agent
def has_data(self): return LiveSyncAgent.find().count()
def _process_args(self): self.agent = LiveSyncAgent.get_or_404(request.view_args['agent_id']) if self.agent.backend is None: flash(_('Cannot edit an agent that is not loaded'), 'error') return redirect(url_for('plugins.details', plugin='livesync'))
def _checkParams(self): self.agent = LiveSyncAgent.find_one(id=request.view_args['agent_id'])
def _process_args(self): self.agent = LiveSyncAgent.get_one(request.view_args['agent_id'])
def _checkParams(self): self.agent = LiveSyncAgent.get_one(request.view_args['agent_id'])