Пример #1
0
def run(agent_id, force=False):
    """Runs the livesync agent"""
    update_session_options(db)
    if agent_id is None:
        agent_list = LiveSyncAgent.find_all()
    else:
        agent = LiveSyncAgent.find_first(id=int(agent_id))
        if agent is None:
            print 'No such agent'
            return
        agent_list = [agent]

    for agent in agent_list:
        if agent.backend is None:
            print cformat('Skipping agent: %{red!}{}%{reset} (backend not found)').format(agent.name)
            continue
        if not agent.initial_data_exported and not force:
            print cformat('Skipping agent: %{red!}{}%{reset} (initial export not performed)').format(agent.name)
            continue
        print cformat('Running agent: %{white!}{}%{reset}').format(agent.name)
        with DBMgr.getInstance().global_connection():
            try:
                agent.create_backend().run()
                db.session.commit()
            finally:
                transaction.abort()
Пример #2
0
def run(agent_id, force=False):
    """Runs the livesync agent"""
    update_session_options(db)
    if agent_id is None:
        agent_list = LiveSyncAgent.find_all()
    else:
        agent = LiveSyncAgent.find_first(id=int(agent_id))
        if agent is None:
            print 'No such agent'
            return
        agent_list = [agent]

    for agent in agent_list:
        if agent.backend is None:
            print cformat(
                'Skipping agent: %{red!}{}%{reset} (backend not found)'
            ).format(agent.name)
            continue
        if not agent.initial_data_exported and not force:
            print cformat(
                'Skipping agent: %{red!}{}%{reset} (initial export not performed)'
            ).format(agent.name)
            continue
        print cformat('Running agent: %{white!}{}%{reset}').format(agent.name)
        with DBMgr.getInstance().global_connection():
            try:
                agent.create_backend().run()
                db.session.commit()
            finally:
                transaction.abort()
Пример #3
0
def run(agent_id, force=False):
    """Runs the livesync agent"""
    if agent_id is None:
        agent_list = LiveSyncAgent.find_all()
    else:
        agent = LiveSyncAgent.find_first(id=agent_id)
        if agent is None:
            print 'No such agent'
            return
        agent_list = [agent]

    for agent in agent_list:
        if agent.backend is None:
            print cformat(
                'Skipping agent: %{red!}{}%{reset} (backend not found)'
            ).format(agent.name)
            continue
        if not agent.initial_data_exported and not force:
            print cformat(
                'Skipping agent: %{red!}{}%{reset} (initial export not performed)'
            ).format(agent.name)
            continue
        print cformat('Running agent: %{white!}{}%{reset}').format(agent.name)
        try:
            agent.create_backend().run()
            db.session.commit()
        except:
            db.session.rollback()
            raise
Пример #4
0
    def migrate_agents(self):
        print cformat('%{white!}migrating agents')
        for old_agent in committing_iterator(self.livesync_root['agent_manager']._agents.itervalues()):
            if not old_agent._active:
                print cformat('%{yellow}skipping inactive agent {} ({})%{reset}').format(old_agent._id, old_agent._name)
                continue

            agent = LiveSyncAgent(name=convert_to_unicode(old_agent._name), initial_data_exported=True)
            old_agent_class = old_agent.__class__.__name__
            if old_agent_class == 'InvenioBatchUploaderAgent':
                agent.backend_name = 'invenio'
                agent.settings = {
                    'server_url': old_agent._url
                }
            elif old_agent_class == 'CERNSearchUploadAgent':
                agent.backend_name = 'cernsearch'
                agent.settings = {
                    'server_url': old_agent._url,
                    'username': old_agent._username,
                    'password': old_agent._password,
                }
            else:
                print cformat('%{red!}skipping unknown agent type: {}%{reset}').format(old_agent_class)
                continue

            print cformat('- %{cyan}{} ({})').format(agent.name, agent.backend_name)
            db.session.add(agent)
Пример #5
0
def run(agent_id, force=False):
    """Runs the livesync agent"""
    if agent_id is None:
        agent_list = LiveSyncAgent.find_all()
    else:
        agent = LiveSyncAgent.find_first(id=agent_id)
        if agent is None:
            print 'No such agent'
            return
        agent_list = [agent]

    for agent in agent_list:
        if agent.backend is None:
            print cformat('Skipping agent: %{red!}{}%{reset} (backend not found)').format(agent.name)
            continue
        if not agent.initial_data_exported and not force:
            print cformat('Skipping agent: %{red!}{}%{reset} (initial export not performed)').format(agent.name)
            continue
        print cformat('Running agent: %{white!}{}%{reset}').format(agent.name)
        try:
            agent.create_backend().run()
            db.session.commit()
        except:
            db.session.rollback()
            raise
Пример #6
0
def test_fetch_records(db, dummy_event_new):
    """Test if the correct records are fetched"""
    agent = LiveSyncAgent(backend_name='dummy', name='dummy')
    backend = DummyBackend(agent)
    db.session.add(agent)
    queue = [LiveSyncQueueEntry(change=ChangeType.created, type=EntryType.event, event=dummy_event_new, processed=True),
             LiveSyncQueueEntry(change=ChangeType.created, type=EntryType.event, event=dummy_event_new)]
    agent.queue = queue
    db.session.flush()
    assert backend.fetch_records() == [queue[1]]
Пример #7
0
def initial_export(agent_id, force=False):
    """Performs the initial data export for an agent"""
    update_session_options(db)
    agent = LiveSyncAgent.find_first(id=int(agent_id))
    if agent is None:
        print 'No such agent'
        return
    if agent.backend is None:
        print cformat('Cannot run agent %{red!}{}%{reset} (backend not found)').format(agent.name)
        return
    print cformat('Selected agent: %{white!}{}%{reset} ({})').format(agent.name, agent.backend.title)
    if agent.initial_data_exported and not force:
        print 'The initial export has already been performed for this agent.'
        print cformat('To re-run it, use %{yellow!}--force%{reset}')
        return

    def _iter_events():
        for i, (_, event) in enumerate(conferenceHolderIterator(ConferenceHolder(), deepness='event'), 1):
            yield event
            if i % 1000 == 0:
                # Clean local ZEO cache
                transaction.abort()

    with DBMgr.getInstance().global_connection():
        agent.create_backend().run_initial_export(_iter_events())

    agent.initial_data_exported = True
    db.session.commit()
Пример #8
0
def agents():
    """Lists the currently active agents"""
    print 'The following LiveSync agents are active:'
    agent_list = LiveSyncAgent.find().order_by(
        LiveSyncAgent.backend_name, db.func.lower(LiveSyncAgent.name)).all()
    table_data = [['ID', 'Name', 'Backend', 'Initial Export', 'Queue']]
    for agent in agent_list:
        initial = (cformat('%{green!}done%{reset}')
                   if agent.initial_data_exported else
                   cformat('%{yellow!}pending%{reset}'))
        if agent.backend is None:
            backend_title = cformat(
                '%{red!}invalid backend ({})%{reset}').format(
                    agent.backend_name)
        else:
            backend_title = agent.backend.title
        table_data.append([
            unicode(agent.id), agent.name, backend_title, initial,
            unicode(agent.queue.filter_by(processed=False).count())
        ])
    table = AsciiTable(table_data)
    table.justify_columns[4] = 'right'
    print table.table
    if not all(a.initial_data_exported for a in agent_list):
        print
        print "You need to perform the initial data export for some agents."
        print cformat(
            "To do so, run "
            "%{yellow!}indico livesync initial_export %{reset}%{yellow}<agent_id>%{reset} for those agents."
        )
Пример #9
0
    def create(cls, changes, ref, excluded_categories=set()):
        """Create a new change in all queues.

        :param changes: the change types, an iterable containing
                        :class:`ChangeType`
        :param ref: the object reference (returned by `obj_ref`)
                        of the changed object
        :param excluded_categories: set of categories (IDs) whose items
                                    will not be tracked
        """
        ref = dict(ref)
        obj = obj_deref(ref)

        if isinstance(obj, Category):
            if any(c.id in excluded_categories for c in obj.chain_query):
                return
        else:
            event = obj if isinstance(obj, Event) else obj.event
            if excluded_categories & set(event.category_chain):
                return

        for change in changes:
            for agent in LiveSyncAgent.find():
                entry = cls(agent=agent, change=change, **ref)
                db.session.add(entry)

        db.session.flush()
Пример #10
0
    def _process(self):
        form = self.backend.form(obj=FormDefaults(name=self.backend.title))
        if form.validate_on_submit():
            data = form.data
            name = data.pop('name')
            agent = LiveSyncAgent(name=name, backend_name=self.backend_name, settings=data)
            db.session.add(agent)
            flash(_('Agent added'), 'success')
            flash(_("Don't forget to run the initial export!"), 'highlight')
            return jsonify_data(flash=False)

        return jsonify_template('edit_agent.html', render_plugin_template, form=form, backend=self.backend, edit=False)
Пример #11
0
    def migrate_agents(self):
        print cformat('%{white!}migrating agents')
        for old_agent in committing_iterator(
                self.livesync_root['agent_manager']._agents.itervalues()):
            if not old_agent._active:
                print cformat(
                    '%{yellow}skipping inactive agent {} ({})%{reset}').format(
                        old_agent._id, old_agent._name)
                continue

            agent = LiveSyncAgent(name=convert_to_unicode(old_agent._name),
                                  initial_data_exported=True)
            old_agent_class = old_agent.__class__.__name__
            if old_agent_class == 'InvenioBatchUploaderAgent':
                agent.backend_name = 'invenio'
                agent.settings = {'server_url': old_agent._url}
            elif old_agent_class == 'CERNSearchUploadAgent':
                agent.backend_name = 'cernsearch'
                agent.settings = {
                    'server_url': old_agent._url,
                    'username': old_agent._username,
                    'password': old_agent._password,
                }
            else:
                print cformat('%{red!}skipping unknown agent type: {}%{reset}'
                              ).format(old_agent_class)
                continue

            print cformat('- %{cyan}{} ({})').format(agent.name,
                                                     agent.backend_name)
            db.session.add(agent)
Пример #12
0
def scheduled_update():
    from indico_livesync.plugin import LiveSyncPlugin
    clean_old_entries()
    for agent in LiveSyncAgent.find_all():
        if agent.backend is None:
            LiveSyncPlugin.logger.warning('Skipping agent %s; backend not found', agent.name)
            continue
        if not agent.initial_data_exported:
            LiveSyncPlugin.logger.warning('Skipping agent %s; initial export not performed yet', agent.name)
            continue
        LiveSyncPlugin.logger.info('Running agent %s', agent.name)
        agent.create_backend().run()
        db.session.commit()
Пример #13
0
    def create(cls, changes, ref):
        """Creates a new change in all queues

        :param changes: the change types, an iterable containing
                        :class:`ChangeType`
        :param ref: the object reference (returned by `obj_ref`)
                        of the changed object
        """
        ref = dict(ref)
        for agent in LiveSyncAgent.find():
            for change in changes:
                entry = cls(agent=agent, change=change, **ref)
                db.session.add(entry)
        db.session.flush()
Пример #14
0
    def _process(self):
        form = self.backend.form(obj=FormDefaults(name=self.backend.title))
        if form.validate_on_submit():
            data = form.data
            name = data.pop('name')
            agent = LiveSyncAgent(name=name,
                                  backend_name=self.backend_name,
                                  settings=data)
            db.session.add(agent)
            flash(_('Agent added'), 'success')
            flash(_("Don't forget to run the initial export!"), 'highlight')
            return redirect(url_for('plugins.details', plugin='livesync'))

        return WPLiveSync.render_template('edit_agent.html',
                                          form=form,
                                          backend=self.backend)
Пример #15
0
def scheduled_update():
    from indico_livesync.plugin import LiveSyncPlugin
    clean_old_entries()
    for agent in LiveSyncAgent.find_all():
        if agent.backend is None:
            LiveSyncPlugin.logger.warning(
                'Skipping agent %s; backend not found', agent.name)
            continue
        if not agent.initial_data_exported:
            LiveSyncPlugin.logger.warning(
                'Skipping agent %s; initial export not performed yet',
                agent.name)
            continue
        LiveSyncPlugin.logger.info('Running agent %s', agent.name)
        agent.create_backend().run()
        db.session.commit()
Пример #16
0
def initial_export(agent_id, force):
    """Performs the initial data export for an agent"""
    agent = LiveSyncAgent.get(agent_id)
    if agent is None:
        print('No such agent')
        return
    if agent.backend is None:
        print(cformat('Cannot run agent %{red!}{}%{reset} (backend not found)').format(agent.name))
        return
    print(cformat('Selected agent: %{white!}{}%{reset} ({})').format(agent.name, agent.backend.title))
    if agent.initial_data_exported and not force:
        print('The initial export has already been performed for this agent.')
        print(cformat('To re-run it, use %{yellow!}--force%{reset}'))
        return

    agent.create_backend().run_initial_export(Event.query.filter_by(is_deleted=False))
    agent.initial_data_exported = True
    db.session.commit()
Пример #17
0
def initial_export(agent_id, force):
    """Performs the initial data export for an agent"""
    agent = LiveSyncAgent.find_first(id=agent_id)
    if agent is None:
        print 'No such agent'
        return
    if agent.backend is None:
        print cformat('Cannot run agent %{red!}{}%{reset} (backend not found)').format(agent.name)
        return
    print cformat('Selected agent: %{white!}{}%{reset} ({})').format(agent.name, agent.backend.title)
    if agent.initial_data_exported and not force:
        print 'The initial export has already been performed for this agent.'
        print cformat('To re-run it, use %{yellow!}--force%{reset}')
        return

    agent.create_backend().run_initial_export(Event.find(is_deleted=False))
    agent.initial_data_exported = True
    db.session.commit()
Пример #18
0
def reset(agent_id):
    """Resets all livesync data for an agent."""
    agent = LiveSyncAgent.get(agent_id)
    if agent is None:
        print('No such agent')
        return

    if agent.backend is None:
        print(cformat('Cannot run agent %{red!}{}%{reset} (backend not found)').format(agent.name))
        return

    backend = agent.create_backend()
    reset_allowed, message = backend.check_reset_status()

    if not reset_allowed:
        print(f'Resetting is not possible: {message}')
        return

    print(cformat('Selected agent: %{white!}{}%{reset} ({})').format(agent.name, backend.title))
    print(cformat('%{yellow!}!!! %{red!}DANGER %{yellow!}!!!%{reset}'))
    if backend.reset_deletes_indexed_data:
        print(cformat('%{yellow!}This command will delete all indexed data on this backend.%{reset}')
              .format(backend.title))
    else:
        print(cformat('%{yellow!}This command should only be used if the data on this backend '
                      'has been deleted.%{reset}')
              .format(backend.title))
    print(cformat('%{yellow!}After resetting you need to perform a new initial export.%{reset}'))
    click.confirm(click.style('Do you really want to perform the reset?', fg='red', bold=True),
                  default=False, abort=True)
    if not config.DEBUG:
        click.confirm(click.style('Are you absolutely sure?', fg='red', bold=True), default=False, abort=True)
        for i in range(5):
            print(cformat('\rResetting in %{white!}{}%{reset}s (CTRL+C to abort)').format(5 - i), end='')
            time.sleep(1)
        print('')

    backend.reset()
    db.session.commit()
    print(cformat('Reset complete; run %{green!}indico livesync initial-export {}%{reset} for a new export')
          .format(agent.id))
Пример #19
0
def agents():
    """Lists the currently active agents"""
    print 'The following LiveSync agents are active:'
    agent_list = LiveSyncAgent.find().order_by(LiveSyncAgent.backend_name, db.func.lower(LiveSyncAgent.name)).all()
    table_data = [['ID', 'Name', 'Backend', 'Initial Export', 'Queue']]
    for agent in agent_list:
        initial = (cformat('%{green!}done%{reset}') if agent.initial_data_exported else
                   cformat('%{yellow!}pending%{reset}'))
        if agent.backend is None:
            backend_title = cformat('%{red!}invalid backend ({})%{reset}').format(agent.backend_name)
        else:
            backend_title = agent.backend.title
        table_data.append([unicode(agent.id), agent.name, backend_title, initial,
                           unicode(agent.queue.filter_by(processed=False).count())])
    table = AsciiTable(table_data)
    table.justify_columns[4] = 'right'
    print table.table
    if not all(a.initial_data_exported for a in agent_list):
        print
        print "You need to perform the initial data export for some agents."
        print cformat("To do so, run "
                      "%{yellow!}indico livesync initial_export %{reset}%{yellow}<agent_id>%{reset} for those agents.")
Пример #20
0
def initial_export(agent_id, batch, force, verbose, retry):
    """Performs the initial data export for an agent."""
    agent = LiveSyncAgent.get(agent_id)
    if agent is None:
        print('No such agent')
        return

    if agent.backend is None:
        print(cformat('Cannot run agent %{red!}{}%{reset} (backend not found)').format(agent.name))
        return

    print(cformat('Selected agent: %{white!}{}%{reset} ({})').format(agent.name, agent.backend.title))

    backend = agent.create_backend()
    if not backend.is_configured():
        print(cformat('Agent %{red!}{}%{reset} is not properly configured').format(agent.name))
        return

    if agent.initial_data_exported and not force:
        print('The initial export has already been performed for this agent.')
        print(cformat('To re-run it, use %{yellow!}--force%{reset}'))
        return

    try:
        if not backend.run_initial_export(batch, force, verbose):
            print('The initial export failed; not marking it as done')
            return
    except Exception:
        if not retry:
            raise
        traceback.print_exc()
        print('Restarting in 2 seconds')
        time.sleep(2)
        os.execl(sys.argv[0], *sys.argv)
        return  # exec doesn't return but just in case...

    agent.initial_data_exported = True
    db.session.commit()
Пример #21
0
def run(agent_id, force, verbose, allowed_categories):
    """Runs the livesync agent."""
    from indico_livesync.plugin import LiveSyncPlugin

    if LiveSyncPlugin.settings.get('disable_queue_runs'):
        print(cformat('%{yellow!}Queue runs are disabled%{reset}'))
    if LiveSyncPlugin.settings.get('skip_category_changes'):
        print(cformat('%{yellow!}Category changes are currently being skipped%{reset}'))
        if allowed_categories:
            print(cformat('Whitelisted categories: %{green}{}%{reset}')
                  .format(', '.join(map(str, sorted(allowed_categories)))))

    if agent_id is None:
        agent_list = LiveSyncAgent.query.all()
    else:
        agent = LiveSyncAgent.get(agent_id)
        if agent is None:
            print('No such agent')
            return
        agent_list = [agent]

    for agent in agent_list:
        if agent.backend is None:
            print(cformat('Skipping agent: %{red!}{}%{reset} (backend not found)').format(agent.name))
            continue
        backend = agent.create_backend()
        queue_allowed, reason = backend.check_queue_status()
        if not queue_allowed and not force:
            print(cformat('Skipping agent: %{red!}{}%{reset} ({})').format(agent.name, reason))
            continue
        print(cformat('Running agent: %{white!}{}%{reset}').format(agent.name))
        try:
            backend.run(verbose, from_cli=True, allowed_categories=allowed_categories)
            db.session.commit()
        except Exception:
            db.session.rollback()
            raise
Пример #22
0
def extend_plugin_details():
    agents = LiveSyncAgent.find().order_by(LiveSyncAgent.name, LiveSyncAgent.id).all()
    return render_plugin_template('plugin_details_extra.html', agents=agents, backends=current_plugin.backend_classes)
Пример #23
0
 def _checkParams(self):
     self.agent = LiveSyncAgent.find_one(id=request.view_args['agent_id'])
     if self.agent.backend is None:
         flash(_('Cannot edit an agent that is not loaded'), 'error')
         return redirect(url_for('plugins.details', plugin='livesync'))
Пример #24
0
def dummy_agent(db):
    agent = LiveSyncAgent(backend_name='dummy', name='dummy')
    db.session.add(agent)
    return agent
Пример #25
0
 def has_data(self):
     return LiveSyncAgent.find().count()
Пример #26
0
 def has_data(self):
     return LiveSyncAgent.find().count()
Пример #27
0
 def _checkParams(self):
     self.agent = LiveSyncAgent.find_one(id=request.view_args['agent_id'])
     if self.agent.backend is None:
         flash(_('Cannot edit an agent that is not loaded'), 'error')
         return redirect(url_for('plugins.details', plugin='livesync'))
Пример #28
0
 def _process_args(self):
     self.agent = LiveSyncAgent.get_or_404(request.view_args['agent_id'])
     if self.agent.backend is None:
         flash(_('Cannot edit an agent that is not loaded'), 'error')
         return redirect(url_for('plugins.details', plugin='livesync'))
Пример #29
0
 def _checkParams(self):
     self.agent = LiveSyncAgent.find_one(id=request.view_args['agent_id'])
Пример #30
0
def extend_plugin_details():
    agents = LiveSyncAgent.find().order_by(LiveSyncAgent.name,
                                           LiveSyncAgent.id).all()
    return render_plugin_template('plugin_details_extra.html',
                                  agents=agents,
                                  backends=current_plugin.backend_classes)
Пример #31
0
 def _checkParams(self):
     self.agent = LiveSyncAgent.find_one(id=request.view_args['agent_id'])
 def _process_args(self):
     self.agent = LiveSyncAgent.get_one(request.view_args['agent_id'])
Пример #33
0
 def _checkParams(self):
     self.agent = LiveSyncAgent.get_one(request.view_args['agent_id'])