def _update_projects(self, writer, projects, clear=False):
add_document = partial(self._add_document, writer)
counter = itertools.count()
count = next(counter)
main_keys = self.project_ix.schema.names()
text_keys = (('author', 0.5), ('author_email', 0.5),
('description', 1.5), ('summary', 1.75), ('keywords',
1.75))
for project in projects:
data = dict((u(x), get_mutable_deepcopy(project[x]))
for x in main_keys if x in project)
data['path'] = u"/{user}/{index}/{name}".format(**data)
if not clear:
# because we use hierarchical documents, we have to delete
# everything we got for this path and index it again
writer.delete_by_term('path', data['path'])
data['type'] = "project"
data['text'] = "%s %s" % (data['name'], project_name(data['name']))
with writer.group():
add_document(**data)
count = next(counter)
for key, boost in text_keys:
if key not in project:
continue
add_document(
**{
"path": data['path'],
"type": key,
"text": project[key],
"_text_boost": boost
})
count = next(counter)
if '+doczip' not in project:
continue
if not project['+doczip'].exists():
log.error("documentation zip file is missing %s",
data['path'])
continue
for page in project['+doczip'].values():
if page is None:
continue
add_document(
**{
"path": data['path'],
"type": "title",
"text": page['title'],
"text_path": page['path'],
"text_title": page['title']
})
count = next(counter)
add_document(
**{
"path": data['path'],
"type": "page",
"text": page['text'],
"text_path": page['path'],
"text_title": page['title']
})
count = next(counter)
return count
def test_simple(self):
x = (1, )
assert not is_deeply_readonly(x)
r = ensure_deeply_readonly(x)
assert is_deeply_readonly(r)
assert r
assert len(r) == 1
assert r[0] == 1
c = get_mutable_deepcopy(r)
assert c == x
def json_info_view(context, request):
baseurl = URL(request.application_url).asdir()
version = context.stage.get_latest_version(context.project, stable=True)
info = get_mutable_deepcopy(context.stage.get_versiondata(context.project, version))
info.pop("+elinks", None)
result = dict(info=info, releases={})
for release in context.stage.get_releaselinks(context.project):
result["releases"].setdefault(release.version, []).append(
dict(url=baseurl.joinpath(release.relpath).url)
)
return result
def record_set(self, typedkey, value=None):
""" record setting typedkey to value (None means it's deleted) """
assert not isinstance(value, ReadonlyView), value
try:
_, back_serial = self.conn.db_read_typedkey(typedkey.relpath)
except KeyError:
back_serial = -1
self.conn.db_write_typedkey(typedkey.relpath, typedkey.name,
self.storage.next_serial)
# at __exit__ time we write out changes to the _changelog_cache
# so we protect here against the caller modifying the value later
value = get_mutable_deepcopy(value)
self.changes[typedkey.relpath] = (typedkey.name, back_serial, value)
def test_simple(self):
x = [1]
assert not is_deeply_readonly(x)
r = ensure_deeply_readonly(x)
assert r
assert is_deeply_readonly(r)
assert len(r) == 1
with pytest.raises(AttributeError):
r.append(2)
c = get_mutable_deepcopy(r)
assert c == x
c.append(1)
assert c != x
def test_include_mirrordata(self, caplog, makeimpexp, maketestapp,
pypistage):
impexp = makeimpexp(options=('--include-mirrored-files', ))
mapp1 = impexp.mapp1
testapp = maketestapp(mapp1.xom)
api = mapp1.use('root/pypi')
pypistage.mock_simple(
"package", '<a href="/package-1.0.zip" />\n'
'<a href="/package-1.1.zip#sha256=a665a45920422f9d417e4867efdc4fb8a04a1f3fff1fa07e998e86f7f7a27ae3" />\n'
'<a href="/package-1.2.zip#sha256=b3a8e0e1f9ab1bfe3a36f231f676f78bb30a519d2b21e6c530c0eee8ebb4a5d0" data-yanked="" />\n'
'<a href="/package-2.0.zip#sha256=35a9e381b1a27567549b5f8a6f783c167ebf809f1c4d6a9e367240484d8ce281" data-requires-python=">=3.5" />'
)
pypistage.mock_extfile("/package-1.1.zip", b"123")
pypistage.mock_extfile("/package-1.2.zip", b"456")
pypistage.mock_extfile("/package-2.0.zip", b"789")
r = testapp.get(api.index + "/+simple/package/")
assert r.status_code == 200
# fetch some files, so they are included in the dump
(_, link1, link2, link3) = sorted(
(x.attrs['href'] for x in r.html.select('a')),
key=lambda x: x.split('/')[-1])
baseurl = URL(r.request.url)
r = testapp.get(baseurl.joinpath(link1).url)
assert r.body == b"123"
r = testapp.get(baseurl.joinpath(link2).url)
assert r.body == b"456"
r = testapp.get(baseurl.joinpath(link3).url)
assert r.body == b"789"
impexp.export()
mapp2 = impexp.new_import()
with mapp2.xom.keyfs.transaction(write=False):
stage = mapp2.xom.model.getstage(api.stagename)
stage.offline = True
projects = stage.list_projects_perstage()
assert projects == {'package'}
links = sorted(
get_mutable_deepcopy(
stage.get_simplelinks_perstage("package")))
assert links == [
('package-1.1.zip',
'root/pypi/+f/a66/5a45920422f9d/package-1.1.zip', None, None),
('package-1.2.zip',
'root/pypi/+f/b3a/8e0e1f9ab1bfe/package-1.2.zip', None, True),
('package-2.0.zip',
'root/pypi/+f/35a/9e381b1a27567/package-2.0.zip', '>=3.5',
None)
]
def _update_projects(self, projects):
main_keys = [
'path', 'name', 'user', 'index', 'classifiers', 'keywords',
'version', 'doc_version', 'type', 'text_path', 'text_title', 'text'
]
for i, project in enumerate(projects, 1):
data = {
x: get_mutable_deepcopy(project[x])
for x in main_keys if x in project
}
data['path'] = u"/{user}/{index}/{name}".format(**data)
data['type'] = "project"
data['text'] = "%s %s" % (data['name'], project_name(data['name']))
yield dict(_index=self.index,
_type=data['type'],
_id=data['path'],
_source=data)
def _update_project(self, project, serial, counter, main_keys, writer,
searcher):
def add_document(**kw):
try:
writer.add_document(**kw)
except Exception:
log.exception(
"Exception while trying to add the following data to the search index:\n%r"
% kw)
raise
text_keys = (('author', 0.5), ('author_email', 0.5),
('description', 1.5), ('summary', 1.75), ('keywords',
1.75))
data = dict((u(x), get_mutable_deepcopy(project[x])) for x in main_keys
if x in project)
data['path'] = u"/{user}/{index}/{name}".format(user=data['user'],
index=data['index'],
name=normalize_name(
data['name']))
existing = None
doc_num = searcher.document_number(path=data['path'])
if doc_num is not None:
existing = searcher.stored_fields(doc_num)
if existing is not None:
needs_reindex = False
if ('+doczip' in project) != ('doc_version' in existing):
needs_reindex = True
existing_serial = existing.get('serial', -1)
if existing_serial < serial:
needs_reindex = True
if not needs_reindex:
return
# because we use hierarchical documents, we have to delete
# everything we got for this path and index it again
writer.delete_by_term('path', data['path'], searcher=searcher)
data['serial'] = serial
data['type'] = "project"
data['text'] = "%s %s" % (data['name'], project_name(data['name']))
with writer.group():
add_document(**data)
next(counter)
for key, boost in text_keys:
if key not in project:
continue
add_document(
**{
"path": data['path'],
"type": key,
"text": project[key],
"_text_boost": boost
})
next(counter)
if '+doczip' not in project:
return
if not project['+doczip'].exists():
log.error("documentation zip file is missing %s", data['path'])
return
for page in project['+doczip'].values():
if page is None:
continue
add_document(
**{
"path": data['path'],
"type": "title",
"text": page['title'],
"text_path": page['path'],
"text_title": page['title']
})
next(counter)
add_document(
**{
"path": data['path'],
"type": "page",
"text": page['text'],
"text_path": page['path'],
"text_title": page['title']
})
next(counter)
