def test_html_in_text(self):
obj = mock.MagicMock()
obj.index.return_value = {'text': '<script>alert(1)</script>'}
assert_equal(solarize(obj), {'text': ''})
obj.index.return_value = {'text': '<script>alert(1)</script>'}
assert_equal(solarize(obj), {'text': '<script>alert(1)</script>'})
def test_del_artifacts(self, solr):
old_shortlinks = M.Shortlink.query.find().count()
artifacts = [ _TestArtifact(_shorthand_id='ta_%s' % x) for x in range(5) ]
M.artifact_orm_session.flush()
arefs = [ M.ArtifactReference.from_artifact(a) for a in artifacts ]
ref_ids = [ r._id for r in arefs ]
M.artifact_orm_session.flush()
index_tasks.add_artifacts(ref_ids)
M.main_orm_session.flush()
M.main_orm_session.clear()
new_shortlinks = M.Shortlink.query.find().count()
assert old_shortlinks + 5 == new_shortlinks, 'Shortlinks not created'
assert solr.add.call_count == 1
sort_key = operator.itemgetter('id')
assert_equal(
sorted(solr.add.call_args[0][0], key=sort_key),
sorted([search.solarize(ref.artifact) for ref in arefs],
key=sort_key))
index_tasks.del_artifacts(ref_ids)
M.main_orm_session.flush()
M.main_orm_session.clear()
new_shortlinks = M.Shortlink.query.find().count()
assert old_shortlinks == new_shortlinks, 'Shortlinks not deleted'
solr_query = 'id:({0})'.format(' || '.join(ref_ids))
solr.delete.assert_called_once_with(q=solr_query)
def test_del_artifacts(self, solr):
old_shortlinks = M.Shortlink.query.find().count()
artifacts = [
_TestArtifact(_shorthand_id='ta_%s' % x) for x in range(5)
]
M.artifact_orm_session.flush()
arefs = [M.ArtifactReference.from_artifact(a) for a in artifacts]
ref_ids = [r._id for r in arefs]
M.artifact_orm_session.flush()
index_tasks.add_artifacts(ref_ids)
M.main_orm_session.flush()
M.main_orm_session.clear()
new_shortlinks = M.Shortlink.query.find().count()
assert old_shortlinks + 5 == new_shortlinks, 'Shortlinks not created'
assert solr.add.call_count == 1
sort_key = operator.itemgetter('id')
assert_equal(
sorted(solr.add.call_args[0][0], key=sort_key),
sorted([search.solarize(ref.artifact) for ref in arefs],
key=sort_key))
index_tasks.del_artifacts(ref_ids)
M.main_orm_session.flush()
M.main_orm_session.clear()
new_shortlinks = M.Shortlink.query.find().count()
assert old_shortlinks == new_shortlinks, 'Shortlinks not deleted'
solr_query = 'id:({0})'.format(' || '.join(ref_ids))
solr.delete.assert_called_once_with(q=solr_query)
def add_artifacts(ref_ids, update_solr=True, update_refs=True):
'''Add the referenced artifacts to SOLR and shortlinks'''
from allura import model as M
from allura.lib.search import find_shortlinks, solarize
exceptions = []
solr_updates = []
with _indexing_disabled(M.session.artifact_orm_session._get()):
for ref in M.ArtifactReference.query.find(dict(_id={'$in': ref_ids})):
try:
artifact = ref.artifact
s = solarize(artifact)
if s is None:
continue
if update_solr:
solr_updates.append(s)
if update_refs:
if isinstance(artifact, M.Snapshot):
continue
ref.references = [
link.ref_id for link in find_shortlinks(s['text'])
]
except Exception:
log.error('Error indexing artifact %s', ref._id)
exceptions.append(sys.exc_info())
g.solr.add(solr_updates)
if len(exceptions) == 1:
raise exceptions[0][0], exceptions[0][1], exceptions[0][2]
if exceptions:
raise CompoundError(*exceptions)
def add_artifacts(ref_ids, update_solr=True, update_refs=True):
'''Add the referenced artifacts to SOLR and shortlinks'''
from allura import model as M
from allura.lib.search import find_shortlinks, solarize
exceptions = []
solr_updates = []
with _indexing_disabled(M.session.artifact_orm_session._get()):
for ref in M.ArtifactReference.query.find(dict(_id={'$in': ref_ids})):
try:
artifact = ref.artifact
s = solarize(artifact)
if s is None:
continue
if update_solr:
solr_updates.append(s)
if update_refs:
if isinstance(artifact, M.Snapshot):
continue
ref.references = [
link.ref_id for link in find_shortlinks(s['text']) ]
except Exception:
log.error('Error indexing artifact %s', ref._id)
exceptions.append(sys.exc_info())
g.solr.add(solr_updates)
if len(exceptions) == 1:
raise exceptions[0][0], exceptions[0][1], exceptions[0][2]
if exceptions:
raise CompoundError(*exceptions)
def add_artifacts(ref_ids, update_solr=True, update_refs=True, solr_hosts=None):
'''
Add the referenced artifacts to SOLR and shortlinks.
:param solr_hosts: a list of solr hosts to use instead of the defaults
:type solr_hosts: [str]
'''
from allura import model as M
from allura.lib.search import find_shortlinks, solarize
if solr_hosts:
solr = make_solr_from_config(solr_hosts)
else:
solr = g.solr
exceptions = []
solr_updates = []
with _indexing_disabled(M.session.artifact_orm_session._get()):
for ref in M.ArtifactReference.query.find(dict(_id={'$in': ref_ids})):
try:
artifact = ref.artifact
s = solarize(artifact)
if s is None:
continue
if update_solr:
solr_updates.append(s)
if update_refs:
if isinstance(artifact, M.Snapshot):
continue
# Find shortlinks in the raw text, not the escaped html
# created by the `solarize()`.
ref.references = [
link.ref_id for link in find_shortlinks(artifact.index().get('text') or '')]
except Exception:
log.error('Error indexing artifact %s', ref._id)
exceptions.append(sys.exc_info())
solr.add(solr_updates)
if len(exceptions) == 1:
raise exceptions[0][0], exceptions[0][1], exceptions[0][2]
if exceptions:
raise CompoundError(*exceptions)
def add_artifacts(ref_ids, update_solr=True, update_refs=True, solr_hosts=None):
'''
Add the referenced artifacts to SOLR and shortlinks.
:param solr_hosts: a list of solr hosts to use instead of the defaults
:type solr_hosts: [str]
'''
from allura import model as M
from allura.lib.search import find_shortlinks, solarize
if solr_hosts:
solr = make_solr_from_config(solr_hosts)
else:
solr = g.solr
exceptions = []
solr_updates = []
with _indexing_disabled(M.session.artifact_orm_session._get()):
for ref in M.ArtifactReference.query.find(dict(_id={'$in': ref_ids})):
try:
artifact = ref.artifact
s = solarize(artifact)
if s is None:
continue
if update_solr:
solr_updates.append(s)
if update_refs:
if isinstance(artifact, M.Snapshot):
continue
# Find shortlinks in the raw text, not the escaped html
# created by the `solarize()`.
ref.references = [
link.ref_id for link in find_shortlinks(artifact.index().get('text') or '')]
except Exception:
log.error('Error indexing artifact %s', ref._id)
exceptions.append(sys.exc_info())
solr.add(solr_updates)
if len(exceptions) == 1:
raise exceptions[0][0], exceptions[0][1], exceptions[0][2]
if exceptions:
raise CompoundError(*exceptions)
def test_empty_index(self):
obj = mock.MagicMock()
obj.index.return_value = None
assert_equal(solarize(obj), None)
def test_no_object(self):
assert_equal(solarize(None), None)
def test_strip_markdown(self):
obj = mock.MagicMock()
obj.index.return_value = {'text': '# Header'}
assert_equal(solarize(obj), {'text': 'Header'})
def test_doc_without_text(self):
obj = mock.MagicMock()
obj.index.return_value = {}
assert_equal(solarize(obj), {'text': ''})
def test_empty_index(self):
obj = mock.MagicMock()
obj.index.return_value = None
assert_equal(solarize(obj), None)
def test_no_object(self):
assert_equal(solarize(None), None)
def test_strip_markdown(self):
obj = mock.MagicMock()
obj.index.return_value = {'text': '# Header'}
assert_equal(solarize(obj), {'text': 'Header'})
def test_doc_without_text(self):
obj = mock.MagicMock()
obj.index.return_value = {}
assert_equal(solarize(obj), {'text': ''})