def add_artifacts(ref_ids, update_solr=True, update_refs=True):
    '''Add the referenced artifacts to SOLR and shortlinks'''
    from allura import model as M
    from allura.lib.search import find_shortlinks, solarize
    exceptions = []
    solr_updates = []
    with _indexing_disabled(M.session.artifact_orm_session._get()):
        for ref in M.ArtifactReference.query.find(dict(_id={'$in': ref_ids})):
            try:
                artifact = ref.artifact
                s = solarize(artifact)
                if s is None:
                    continue
                if update_solr:
                    solr_updates.append(s)
                if update_refs:
                    if isinstance(artifact, M.Snapshot):
                        continue
                    ref.references = [
                        link.ref_id for link in find_shortlinks(s['text'])
                    ]
            except Exception:
                log.error('Error indexing artifact %s', ref._id)
                exceptions.append(sys.exc_info())
        g.solr.add(solr_updates)

    if len(exceptions) == 1:
        raise exceptions[0][0], exceptions[0][1], exceptions[0][2]
    if exceptions:
        raise CompoundError(*exceptions)
Esempio n. 2
0
    def test_add_artifacts(self):
        from allura.lib.search import find_shortlinks
        with mock.patch('allura.lib.search.find_shortlinks') as find_slinks:
            find_slinks.side_effect = lambda s: find_shortlinks(s)

            old_shortlinks = M.Shortlink.query.find().count()
            old_solr_size = len(g.solr.db)
            artifacts = [_TestArtifact() for x in range(5)]
            for i, a in enumerate(artifacts):
                a._shorthand_id = 't%d' % i
                a.text = 'This is a reference to [t3]'
            arefs = [M.ArtifactReference.from_artifact(a) for a in artifacts]
            ref_ids = [r._id for r in arefs]
            M.artifact_orm_session.flush()
            index_tasks.add_artifacts(ref_ids)
            new_shortlinks = M.Shortlink.query.find().count()
            new_solr_size = len(g.solr.db)
            assert old_shortlinks + \
                5 == new_shortlinks, 'Shortlinks not created'
            assert old_solr_size + \
                5 == new_solr_size, "Solr additions didn't happen"
            M.main_orm_session.flush()
            M.main_orm_session.clear()
            t3 = _TestArtifact.query.get(_shorthand_id='t3')
            assert len(t3.backrefs) == 5, t3.backrefs
            assert_equal(find_slinks.call_args_list,
                         [mock.call(a.index().get('text')) for a in artifacts])
Esempio n. 3
0
    def test_add_artifacts(self):
        from allura.lib.search import find_shortlinks
        with mock.patch('allura.lib.search.find_shortlinks') as find_slinks:
            find_slinks.side_effect = lambda s: find_shortlinks(s)

            old_shortlinks = M.Shortlink.query.find().count()
            old_solr_size = len(g.solr.db)
            artifacts = [_TestArtifact() for x in range(5)]
            for i, a in enumerate(artifacts):
                a._shorthand_id = 't%d' % i
                a.text = 'This is a reference to [t3]'
            arefs = [M.ArtifactReference.from_artifact(a) for a in artifacts]
            ref_ids = [r._id for r in arefs]
            M.artifact_orm_session.flush()
            index_tasks.add_artifacts(ref_ids)
            new_shortlinks = M.Shortlink.query.find().count()
            new_solr_size = len(g.solr.db)
            assert old_shortlinks + \
                5 == new_shortlinks, 'Shortlinks not created'
            assert old_solr_size + \
                5 == new_solr_size, "Solr additions didn't happen"
            M.main_orm_session.flush()
            M.main_orm_session.clear()
            t3 = _TestArtifact.query.get(_shorthand_id='t3')
            assert len(t3.backrefs) == 5, t3.backrefs
            assert_equal(find_slinks.call_args_list,
                         [mock.call(a.index().get('text')) for a in artifacts])
Esempio n. 4
0
def add_artifacts(ref_ids, update_solr=True, update_refs=True):
    '''Add the referenced artifacts to SOLR and shortlinks'''
    from allura import model as M
    from allura.lib.search import find_shortlinks, solarize
    exceptions = []
    solr_updates = []
    with _indexing_disabled(M.session.artifact_orm_session._get()):
        for ref in M.ArtifactReference.query.find(dict(_id={'$in': ref_ids})):
            try:
                artifact = ref.artifact
                s = solarize(artifact)
                if s is None:
                    continue
                if update_solr:
                    solr_updates.append(s)
                if update_refs:
                    if isinstance(artifact, M.Snapshot):
                        continue
                    ref.references = [
                        link.ref_id for link in find_shortlinks(s['text']) ]
            except Exception:
                log.error('Error indexing artifact %s', ref._id)
                exceptions.append(sys.exc_info())
        g.solr.add(solr_updates)

    if len(exceptions) == 1:
        raise exceptions[0][0], exceptions[0][1], exceptions[0][2]
    if exceptions:
        raise CompoundError(*exceptions)
Esempio n. 5
0
def add_artifacts(ref_ids,
                  update_solr=True,
                  update_refs=True,
                  solr_hosts=None):
    '''
    Add the referenced artifacts to SOLR and shortlinks.

    :param solr_hosts: a list of solr hosts to use instead of the defaults
    :type solr_hosts: [str]
    '''
    from allura import model as M
    from allura.lib.search import find_shortlinks

    exceptions = []
    solr_updates = []
    with _indexing_disabled(M.session.artifact_orm_session._get()):
        for ref in M.ArtifactReference.query.find(dict(_id={'$in': ref_ids})):
            try:
                artifact = ref.artifact
                if artifact is None:
                    continue
                # c.app is normally set, so keep using it.  During a reindex its not though, so set it from artifact
                with h.push_config(c,
                                   app=getattr(c, 'app', None)
                                   or artifact.app):
                    s = artifact.solarize()
                    if s is None:
                        continue
                    if update_solr:
                        solr_updates.append(s)
                    if update_refs:
                        if isinstance(artifact, M.Snapshot):
                            continue
                        # Find shortlinks in the raw text, not the escaped html
                        # created by the `solarize()`.
                        link_text = artifact.index().get('text') or ''
                        shortlinks = find_shortlinks(link_text)
                        ref.references = [link.ref_id for link in shortlinks]
            except Exception:
                log.error('Error indexing artifact %s', ref._id)
                exceptions.append(sys.exc_info())
        __get_solr(solr_hosts).add(solr_updates)

    if len(exceptions) == 1:
        six.reraise(exceptions[0][0], exceptions[0][1], exceptions[0][2])
    if exceptions:
        raise CompoundError(*exceptions)
Esempio n. 6
0
def add_artifacts(ref_ids, update_solr=True, update_refs=True, solr_hosts=None):
    '''
    Add the referenced artifacts to SOLR and shortlinks.

    :param solr_hosts: a list of solr hosts to use instead of the defaults
    :type solr_hosts: [str]
    '''
    from allura import model as M
    from allura.lib.search import find_shortlinks

    exceptions = []
    solr_updates = []
    with _indexing_disabled(M.session.artifact_orm_session._get()):
        for ref in M.ArtifactReference.query.find(dict(_id={'$in': ref_ids})):
            try:
                artifact = ref.artifact
                if artifact is None:
                    continue
                # c.app is normally set, so keep using it.  During a reindex its not though, so set it from artifact
                with h.push_config(c, app=getattr(c, 'app', None) or artifact.app):
                    s = artifact.solarize()
                    if s is None:
                        continue
                    if update_solr:
                        solr_updates.append(s)
                    if update_refs:
                        if isinstance(artifact, M.Snapshot):
                            continue
                        # Find shortlinks in the raw text, not the escaped html
                        # created by the `solarize()`.
                        link_text = artifact.index().get('text') or ''
                        shortlinks = find_shortlinks(link_text)
                        ref.references = [link.ref_id for link in shortlinks]
            except Exception:
                log.error('Error indexing artifact %s', ref._id)
                exceptions.append(sys.exc_info())
        __get_solr(solr_hosts).add(solr_updates)

    if len(exceptions) == 1:
        raise exceptions[0][0], exceptions[0][1], exceptions[0][2]
    if exceptions:
        raise CompoundError(*exceptions)
Esempio n. 7
0
 def handle_command(self, sender, cmd, rest):
     if cmd == 'NOTICE': pass
     elif cmd == '433':
         self.set_nick()
         self.channels = {}
         self.configure()
     elif cmd == 'PING':
         self.say('PONG ' + rest)
     elif cmd in ('NOTICE', 'PRIVMSG'):
         rcpt, msg = rest.split(' ', 1)
         if not self.set_context(rcpt): return
         if msg.startswith(':'): msg = msg[1:]
         self.log_channel(sender, cmd, rcpt, msg)
         if cmd == 'NOTICE': return
         for lnk in search.find_shortlinks(msg):
             self.handle_shortlink(lnk, sender, rcpt)
     ThreadLocalORMSession.flush_all()
     ThreadLocalORMSession.close_all()
     self.check_configure()
     ThreadLocalORMSession.close_all()
Esempio n. 8
0
 def handle_command(self, sender, cmd, rest):
     if cmd == 'NOTICE': pass
     elif cmd == '433':
         self.set_nick()
         self.channels = {}
         self.configure()
     elif cmd == 'PING':
         self.say('PONG ' + rest)
     elif cmd in ('NOTICE', 'PRIVMSG'):
         rcpt, msg = rest.split(' ', 1)
         if not self.set_context(rcpt): return
         if msg.startswith(':'): msg = msg[1:]
         self.log_channel(sender, cmd, rcpt, msg)
         if cmd == 'NOTICE': return
         for lnk in search.find_shortlinks(msg):
             self.handle_shortlink(lnk, sender, rcpt)
     ThreadLocalORMSession.flush_all()
     ThreadLocalORMSession.close_all()
     self.check_configure()
     ThreadLocalORMSession.close_all()
Esempio n. 9
0
def add_artifacts(ref_ids, update_solr=True, update_refs=True, solr_hosts=None):
    '''
    Add the referenced artifacts to SOLR and shortlinks.

    :param solr_hosts: a list of solr hosts to use instead of the defaults
    :type solr_hosts: [str]
    '''
    from allura import model as M
    from allura.lib.search import find_shortlinks, solarize
    if solr_hosts:
        solr = make_solr_from_config(solr_hosts)
    else:
        solr = g.solr
    exceptions = []
    solr_updates = []
    with _indexing_disabled(M.session.artifact_orm_session._get()):
        for ref in M.ArtifactReference.query.find(dict(_id={'$in': ref_ids})):
            try:
                artifact = ref.artifact
                s = solarize(artifact)
                if s is None:
                    continue
                if update_solr:
                    solr_updates.append(s)
                if update_refs:
                    if isinstance(artifact, M.Snapshot):
                        continue
                    # Find shortlinks in the raw text, not the escaped html
                    # created by the `solarize()`.
                    ref.references = [
                        link.ref_id for link in find_shortlinks(artifact.index().get('text') or '')]
            except Exception:
                log.error('Error indexing artifact %s', ref._id)
                exceptions.append(sys.exc_info())
        solr.add(solr_updates)

    if len(exceptions) == 1:
        raise exceptions[0][0], exceptions[0][1], exceptions[0][2]
    if exceptions:
        raise CompoundError(*exceptions)
Esempio n. 10
0
def add_artifacts(ref_ids, update_solr=True, update_refs=True, solr_hosts=None):
    '''
    Add the referenced artifacts to SOLR and shortlinks.

    :param solr_hosts: a list of solr hosts to use instead of the defaults
    :type solr_hosts: [str]
    '''
    from allura import model as M
    from allura.lib.search import find_shortlinks, solarize
    if solr_hosts:
        solr = make_solr_from_config(solr_hosts)
    else:
        solr = g.solr
    exceptions = []
    solr_updates = []
    with _indexing_disabled(M.session.artifact_orm_session._get()):
        for ref in M.ArtifactReference.query.find(dict(_id={'$in': ref_ids})):
            try:
                artifact = ref.artifact
                s = solarize(artifact)
                if s is None:
                    continue
                if update_solr:
                    solr_updates.append(s)
                if update_refs:
                    if isinstance(artifact, M.Snapshot):
                        continue
                    # Find shortlinks in the raw text, not the escaped html
                    # created by the `solarize()`.
                    ref.references = [
                        link.ref_id for link in find_shortlinks(artifact.index().get('text') or '')]
            except Exception:
                log.error('Error indexing artifact %s', ref._id)
                exceptions.append(sys.exc_info())
        solr.add(solr_updates)

    if len(exceptions) == 1:
        raise exceptions[0][0], exceptions[0][1], exceptions[0][2]
    if exceptions:
        raise CompoundError(*exceptions)