def add_artifacts(ref_ids, update_solr=True, update_refs=True): '''Add the referenced artifacts to SOLR and shortlinks''' from allura import model as M from allura.lib.search import find_shortlinks, solarize exceptions = [] solr_updates = [] with _indexing_disabled(M.session.artifact_orm_session._get()): for ref in M.ArtifactReference.query.find(dict(_id={'$in': ref_ids})): try: artifact = ref.artifact s = solarize(artifact) if s is None: continue if update_solr: solr_updates.append(s) if update_refs: if isinstance(artifact, M.Snapshot): continue ref.references = [ link.ref_id for link in find_shortlinks(s['text']) ] except Exception: log.error('Error indexing artifact %s', ref._id) exceptions.append(sys.exc_info()) g.solr.add(solr_updates) if len(exceptions) == 1: raise exceptions[0][0], exceptions[0][1], exceptions[0][2] if exceptions: raise CompoundError(*exceptions)
def test_add_artifacts(self): from allura.lib.search import find_shortlinks with mock.patch('allura.lib.search.find_shortlinks') as find_slinks: find_slinks.side_effect = lambda s: find_shortlinks(s) old_shortlinks = M.Shortlink.query.find().count() old_solr_size = len(g.solr.db) artifacts = [_TestArtifact() for x in range(5)] for i, a in enumerate(artifacts): a._shorthand_id = 't%d' % i a.text = 'This is a reference to [t3]' arefs = [M.ArtifactReference.from_artifact(a) for a in artifacts] ref_ids = [r._id for r in arefs] M.artifact_orm_session.flush() index_tasks.add_artifacts(ref_ids) new_shortlinks = M.Shortlink.query.find().count() new_solr_size = len(g.solr.db) assert old_shortlinks + \ 5 == new_shortlinks, 'Shortlinks not created' assert old_solr_size + \ 5 == new_solr_size, "Solr additions didn't happen" M.main_orm_session.flush() M.main_orm_session.clear() t3 = _TestArtifact.query.get(_shorthand_id='t3') assert len(t3.backrefs) == 5, t3.backrefs assert_equal(find_slinks.call_args_list, [mock.call(a.index().get('text')) for a in artifacts])
def add_artifacts(ref_ids, update_solr=True, update_refs=True, solr_hosts=None): ''' Add the referenced artifacts to SOLR and shortlinks. :param solr_hosts: a list of solr hosts to use instead of the defaults :type solr_hosts: [str] ''' from allura import model as M from allura.lib.search import find_shortlinks exceptions = [] solr_updates = [] with _indexing_disabled(M.session.artifact_orm_session._get()): for ref in M.ArtifactReference.query.find(dict(_id={'$in': ref_ids})): try: artifact = ref.artifact if artifact is None: continue # c.app is normally set, so keep using it. During a reindex its not though, so set it from artifact with h.push_config(c, app=getattr(c, 'app', None) or artifact.app): s = artifact.solarize() if s is None: continue if update_solr: solr_updates.append(s) if update_refs: if isinstance(artifact, M.Snapshot): continue # Find shortlinks in the raw text, not the escaped html # created by the `solarize()`. link_text = artifact.index().get('text') or '' shortlinks = find_shortlinks(link_text) ref.references = [link.ref_id for link in shortlinks] except Exception: log.error('Error indexing artifact %s', ref._id) exceptions.append(sys.exc_info()) __get_solr(solr_hosts).add(solr_updates) if len(exceptions) == 1: six.reraise(exceptions[0][0], exceptions[0][1], exceptions[0][2]) if exceptions: raise CompoundError(*exceptions)
def add_artifacts(ref_ids, update_solr=True, update_refs=True, solr_hosts=None): ''' Add the referenced artifacts to SOLR and shortlinks. :param solr_hosts: a list of solr hosts to use instead of the defaults :type solr_hosts: [str] ''' from allura import model as M from allura.lib.search import find_shortlinks exceptions = [] solr_updates = [] with _indexing_disabled(M.session.artifact_orm_session._get()): for ref in M.ArtifactReference.query.find(dict(_id={'$in': ref_ids})): try: artifact = ref.artifact if artifact is None: continue # c.app is normally set, so keep using it. During a reindex its not though, so set it from artifact with h.push_config(c, app=getattr(c, 'app', None) or artifact.app): s = artifact.solarize() if s is None: continue if update_solr: solr_updates.append(s) if update_refs: if isinstance(artifact, M.Snapshot): continue # Find shortlinks in the raw text, not the escaped html # created by the `solarize()`. link_text = artifact.index().get('text') or '' shortlinks = find_shortlinks(link_text) ref.references = [link.ref_id for link in shortlinks] except Exception: log.error('Error indexing artifact %s', ref._id) exceptions.append(sys.exc_info()) __get_solr(solr_hosts).add(solr_updates) if len(exceptions) == 1: raise exceptions[0][0], exceptions[0][1], exceptions[0][2] if exceptions: raise CompoundError(*exceptions)
def handle_command(self, sender, cmd, rest): if cmd == 'NOTICE': pass elif cmd == '433': self.set_nick() self.channels = {} self.configure() elif cmd == 'PING': self.say('PONG ' + rest) elif cmd in ('NOTICE', 'PRIVMSG'): rcpt, msg = rest.split(' ', 1) if not self.set_context(rcpt): return if msg.startswith(':'): msg = msg[1:] self.log_channel(sender, cmd, rcpt, msg) if cmd == 'NOTICE': return for lnk in search.find_shortlinks(msg): self.handle_shortlink(lnk, sender, rcpt) ThreadLocalORMSession.flush_all() ThreadLocalORMSession.close_all() self.check_configure() ThreadLocalORMSession.close_all()
def add_artifacts(ref_ids, update_solr=True, update_refs=True, solr_hosts=None): ''' Add the referenced artifacts to SOLR and shortlinks. :param solr_hosts: a list of solr hosts to use instead of the defaults :type solr_hosts: [str] ''' from allura import model as M from allura.lib.search import find_shortlinks, solarize if solr_hosts: solr = make_solr_from_config(solr_hosts) else: solr = g.solr exceptions = [] solr_updates = [] with _indexing_disabled(M.session.artifact_orm_session._get()): for ref in M.ArtifactReference.query.find(dict(_id={'$in': ref_ids})): try: artifact = ref.artifact s = solarize(artifact) if s is None: continue if update_solr: solr_updates.append(s) if update_refs: if isinstance(artifact, M.Snapshot): continue # Find shortlinks in the raw text, not the escaped html # created by the `solarize()`. ref.references = [ link.ref_id for link in find_shortlinks(artifact.index().get('text') or '')] except Exception: log.error('Error indexing artifact %s', ref._id) exceptions.append(sys.exc_info()) solr.add(solr_updates) if len(exceptions) == 1: raise exceptions[0][0], exceptions[0][1], exceptions[0][2] if exceptions: raise CompoundError(*exceptions)