def add_artifacts(ref_ids, update_solr=True, update_refs=True):
'''Add the referenced artifacts to SOLR and shortlinks'''
from allura import model as M
from allura.lib.search import find_shortlinks, solarize
exceptions = []
solr_updates = []
with _indexing_disabled(M.session.artifact_orm_session._get()):
for ref in M.ArtifactReference.query.find(dict(_id={'$in': ref_ids})):
try:
artifact = ref.artifact
s = solarize(artifact)
if s is None:
continue
if update_solr:
solr_updates.append(s)
if update_refs:
if isinstance(artifact, M.Snapshot):
continue
ref.references = [
link.ref_id for link in find_shortlinks(s['text'])
]
except Exception:
log.error('Error indexing artifact %s', ref._id)
exceptions.append(sys.exc_info())
g.solr.add(solr_updates)
if len(exceptions) == 1:
raise exceptions[0][0], exceptions[0][1], exceptions[0][2]
if exceptions:
raise CompoundError(*exceptions)
def test_add_artifacts(self):
from allura.lib.search import find_shortlinks
with mock.patch('allura.lib.search.find_shortlinks') as find_slinks:
find_slinks.side_effect = lambda s: find_shortlinks(s)
old_shortlinks = M.Shortlink.query.find().count()
old_solr_size = len(g.solr.db)
artifacts = [_TestArtifact() for x in range(5)]
for i, a in enumerate(artifacts):
a._shorthand_id = 't%d' % i
a.text = 'This is a reference to [t3]'
arefs = [M.ArtifactReference.from_artifact(a) for a in artifacts]
ref_ids = [r._id for r in arefs]
M.artifact_orm_session.flush()
index_tasks.add_artifacts(ref_ids)
new_shortlinks = M.Shortlink.query.find().count()
new_solr_size = len(g.solr.db)
assert old_shortlinks + \
5 == new_shortlinks, 'Shortlinks not created'
assert old_solr_size + \
5 == new_solr_size, "Solr additions didn't happen"
M.main_orm_session.flush()
M.main_orm_session.clear()
t3 = _TestArtifact.query.get(_shorthand_id='t3')
assert len(t3.backrefs) == 5, t3.backrefs
assert_equal(find_slinks.call_args_list,
[mock.call(a.index().get('text')) for a in artifacts])
def test_add_artifacts(self):
from allura.lib.search import find_shortlinks
with mock.patch('allura.lib.search.find_shortlinks') as find_slinks:
find_slinks.side_effect = lambda s: find_shortlinks(s)
old_shortlinks = M.Shortlink.query.find().count()
old_solr_size = len(g.solr.db)
artifacts = [_TestArtifact() for x in range(5)]
for i, a in enumerate(artifacts):
a._shorthand_id = 't%d' % i
a.text = 'This is a reference to [t3]'
arefs = [M.ArtifactReference.from_artifact(a) for a in artifacts]
ref_ids = [r._id for r in arefs]
M.artifact_orm_session.flush()
index_tasks.add_artifacts(ref_ids)
new_shortlinks = M.Shortlink.query.find().count()
new_solr_size = len(g.solr.db)
assert old_shortlinks + \
5 == new_shortlinks, 'Shortlinks not created'
assert old_solr_size + \
5 == new_solr_size, "Solr additions didn't happen"
M.main_orm_session.flush()
M.main_orm_session.clear()
t3 = _TestArtifact.query.get(_shorthand_id='t3')
assert len(t3.backrefs) == 5, t3.backrefs
assert_equal(find_slinks.call_args_list,
[mock.call(a.index().get('text')) for a in artifacts])
def add_artifacts(ref_ids, update_solr=True, update_refs=True):
'''Add the referenced artifacts to SOLR and shortlinks'''
from allura import model as M
from allura.lib.search import find_shortlinks, solarize
exceptions = []
solr_updates = []
with _indexing_disabled(M.session.artifact_orm_session._get()):
for ref in M.ArtifactReference.query.find(dict(_id={'$in': ref_ids})):
try:
artifact = ref.artifact
s = solarize(artifact)
if s is None:
continue
if update_solr:
solr_updates.append(s)
if update_refs:
if isinstance(artifact, M.Snapshot):
continue
ref.references = [
link.ref_id for link in find_shortlinks(s['text']) ]
except Exception:
log.error('Error indexing artifact %s', ref._id)
exceptions.append(sys.exc_info())
g.solr.add(solr_updates)
if len(exceptions) == 1:
raise exceptions[0][0], exceptions[0][1], exceptions[0][2]
if exceptions:
raise CompoundError(*exceptions)
def add_artifacts(ref_ids,
update_solr=True,
update_refs=True,
solr_hosts=None):
'''
Add the referenced artifacts to SOLR and shortlinks.
:param solr_hosts: a list of solr hosts to use instead of the defaults
:type solr_hosts: [str]
'''
from allura import model as M
from allura.lib.search import find_shortlinks
exceptions = []
solr_updates = []
with _indexing_disabled(M.session.artifact_orm_session._get()):
for ref in M.ArtifactReference.query.find(dict(_id={'$in': ref_ids})):
try:
artifact = ref.artifact
if artifact is None:
continue
# c.app is normally set, so keep using it. During a reindex its not though, so set it from artifact
with h.push_config(c,
app=getattr(c, 'app', None)
or artifact.app):
s = artifact.solarize()
if s is None:
continue
if update_solr:
solr_updates.append(s)
if update_refs:
if isinstance(artifact, M.Snapshot):
continue
# Find shortlinks in the raw text, not the escaped html
# created by the `solarize()`.
link_text = artifact.index().get('text') or ''
shortlinks = find_shortlinks(link_text)
ref.references = [link.ref_id for link in shortlinks]
except Exception:
log.error('Error indexing artifact %s', ref._id)
exceptions.append(sys.exc_info())
__get_solr(solr_hosts).add(solr_updates)
if len(exceptions) == 1:
six.reraise(exceptions[0][0], exceptions[0][1], exceptions[0][2])
if exceptions:
raise CompoundError(*exceptions)
def add_artifacts(ref_ids, update_solr=True, update_refs=True, solr_hosts=None):
'''
Add the referenced artifacts to SOLR and shortlinks.
:param solr_hosts: a list of solr hosts to use instead of the defaults
:type solr_hosts: [str]
'''
from allura import model as M
from allura.lib.search import find_shortlinks
exceptions = []
solr_updates = []
with _indexing_disabled(M.session.artifact_orm_session._get()):
for ref in M.ArtifactReference.query.find(dict(_id={'$in': ref_ids})):
try:
artifact = ref.artifact
if artifact is None:
continue
# c.app is normally set, so keep using it. During a reindex its not though, so set it from artifact
with h.push_config(c, app=getattr(c, 'app', None) or artifact.app):
s = artifact.solarize()
if s is None:
continue
if update_solr:
solr_updates.append(s)
if update_refs:
if isinstance(artifact, M.Snapshot):
continue
# Find shortlinks in the raw text, not the escaped html
# created by the `solarize()`.
link_text = artifact.index().get('text') or ''
shortlinks = find_shortlinks(link_text)
ref.references = [link.ref_id for link in shortlinks]
except Exception:
log.error('Error indexing artifact %s', ref._id)
exceptions.append(sys.exc_info())
__get_solr(solr_hosts).add(solr_updates)
if len(exceptions) == 1:
raise exceptions[0][0], exceptions[0][1], exceptions[0][2]
if exceptions:
raise CompoundError(*exceptions)
def handle_command(self, sender, cmd, rest):
if cmd == 'NOTICE': pass
elif cmd == '433':
self.set_nick()
self.channels = {}
self.configure()
elif cmd == 'PING':
self.say('PONG ' + rest)
elif cmd in ('NOTICE', 'PRIVMSG'):
rcpt, msg = rest.split(' ', 1)
if not self.set_context(rcpt): return
if msg.startswith(':'): msg = msg[1:]
self.log_channel(sender, cmd, rcpt, msg)
if cmd == 'NOTICE': return
for lnk in search.find_shortlinks(msg):
self.handle_shortlink(lnk, sender, rcpt)
ThreadLocalORMSession.flush_all()
ThreadLocalORMSession.close_all()
self.check_configure()
ThreadLocalORMSession.close_all()
def handle_command(self, sender, cmd, rest):
if cmd == 'NOTICE': pass
elif cmd == '433':
self.set_nick()
self.channels = {}
self.configure()
elif cmd == 'PING':
self.say('PONG ' + rest)
elif cmd in ('NOTICE', 'PRIVMSG'):
rcpt, msg = rest.split(' ', 1)
if not self.set_context(rcpt): return
if msg.startswith(':'): msg = msg[1:]
self.log_channel(sender, cmd, rcpt, msg)
if cmd == 'NOTICE': return
for lnk in search.find_shortlinks(msg):
self.handle_shortlink(lnk, sender, rcpt)
ThreadLocalORMSession.flush_all()
ThreadLocalORMSession.close_all()
self.check_configure()
ThreadLocalORMSession.close_all()
def add_artifacts(ref_ids, update_solr=True, update_refs=True, solr_hosts=None):
'''
Add the referenced artifacts to SOLR and shortlinks.
:param solr_hosts: a list of solr hosts to use instead of the defaults
:type solr_hosts: [str]
'''
from allura import model as M
from allura.lib.search import find_shortlinks, solarize
if solr_hosts:
solr = make_solr_from_config(solr_hosts)
else:
solr = g.solr
exceptions = []
solr_updates = []
with _indexing_disabled(M.session.artifact_orm_session._get()):
for ref in M.ArtifactReference.query.find(dict(_id={'$in': ref_ids})):
try:
artifact = ref.artifact
s = solarize(artifact)
if s is None:
continue
if update_solr:
solr_updates.append(s)
if update_refs:
if isinstance(artifact, M.Snapshot):
continue
# Find shortlinks in the raw text, not the escaped html
# created by the `solarize()`.
ref.references = [
link.ref_id for link in find_shortlinks(artifact.index().get('text') or '')]
except Exception:
log.error('Error indexing artifact %s', ref._id)
exceptions.append(sys.exc_info())
solr.add(solr_updates)
if len(exceptions) == 1:
raise exceptions[0][0], exceptions[0][1], exceptions[0][2]
if exceptions:
raise CompoundError(*exceptions)
def add_artifacts(ref_ids, update_solr=True, update_refs=True, solr_hosts=None):
'''
Add the referenced artifacts to SOLR and shortlinks.
:param solr_hosts: a list of solr hosts to use instead of the defaults
:type solr_hosts: [str]
'''
from allura import model as M
from allura.lib.search import find_shortlinks, solarize
if solr_hosts:
solr = make_solr_from_config(solr_hosts)
else:
solr = g.solr
exceptions = []
solr_updates = []
with _indexing_disabled(M.session.artifact_orm_session._get()):
for ref in M.ArtifactReference.query.find(dict(_id={'$in': ref_ids})):
try:
artifact = ref.artifact
s = solarize(artifact)
if s is None:
continue
if update_solr:
solr_updates.append(s)
if update_refs:
if isinstance(artifact, M.Snapshot):
continue
# Find shortlinks in the raw text, not the escaped html
# created by the `solarize()`.
ref.references = [
link.ref_id for link in find_shortlinks(artifact.index().get('text') or '')]
except Exception:
log.error('Error indexing artifact %s', ref._id)
exceptions.append(sys.exc_info())
solr.add(solr_updates)
if len(exceptions) == 1:
raise exceptions[0][0], exceptions[0][1], exceptions[0][2]
if exceptions:
raise CompoundError(*exceptions)