def _update_projects(self, writer, projects, clear=False): add_document = partial(self._add_document, writer) counter = itertools.count() count = next(counter) main_keys = self.project_ix.schema.names() text_keys = (('author', 0.5), ('author_email', 0.5), ('description', 1.5), ('summary', 1.75), ('keywords', 1.75)) for project in projects: data = dict((u(x), get_mutable_deepcopy(project[x])) for x in main_keys if x in project) data['path'] = u"/{user}/{index}/{name}".format(**data) if not clear: # because we use hierarchical documents, we have to delete # everything we got for this path and index it again writer.delete_by_term('path', data['path']) data['type'] = "project" data['text'] = "%s %s" % (data['name'], project_name(data['name'])) with writer.group(): add_document(**data) count = next(counter) for key, boost in text_keys: if key not in project: continue add_document( **{ "path": data['path'], "type": key, "text": project[key], "_text_boost": boost }) count = next(counter) if '+doczip' not in project: continue if not project['+doczip'].exists(): log.error("documentation zip file is missing %s", data['path']) continue for page in project['+doczip'].values(): if page is None: continue add_document( **{ "path": data['path'], "type": "title", "text": page['title'], "text_path": page['path'], "text_title": page['title'] }) count = next(counter) add_document( **{ "path": data['path'], "type": "page", "text": page['text'], "text_path": page['path'], "text_title": page['title'] }) count = next(counter) return count
def test_simple(self): x = (1, ) assert not is_deeply_readonly(x) r = ensure_deeply_readonly(x) assert is_deeply_readonly(r) assert r assert len(r) == 1 assert r[0] == 1 c = get_mutable_deepcopy(r) assert c == x
def json_info_view(context, request): baseurl = URL(request.application_url).asdir() version = context.stage.get_latest_version(context.project, stable=True) info = get_mutable_deepcopy(context.stage.get_versiondata(context.project, version)) info.pop("+elinks", None) result = dict(info=info, releases={}) for release in context.stage.get_releaselinks(context.project): result["releases"].setdefault(release.version, []).append( dict(url=baseurl.joinpath(release.relpath).url) ) return result
def record_set(self, typedkey, value=None): """ record setting typedkey to value (None means it's deleted) """ assert not isinstance(value, ReadonlyView), value try: _, back_serial = self.conn.db_read_typedkey(typedkey.relpath) except KeyError: back_serial = -1 self.conn.db_write_typedkey(typedkey.relpath, typedkey.name, self.storage.next_serial) # at __exit__ time we write out changes to the _changelog_cache # so we protect here against the caller modifying the value later value = get_mutable_deepcopy(value) self.changes[typedkey.relpath] = (typedkey.name, back_serial, value)
def test_simple(self): x = [1] assert not is_deeply_readonly(x) r = ensure_deeply_readonly(x) assert r assert is_deeply_readonly(r) assert len(r) == 1 with pytest.raises(AttributeError): r.append(2) c = get_mutable_deepcopy(r) assert c == x c.append(1) assert c != x
def test_include_mirrordata(self, caplog, makeimpexp, maketestapp, pypistage): impexp = makeimpexp(options=('--include-mirrored-files', )) mapp1 = impexp.mapp1 testapp = maketestapp(mapp1.xom) api = mapp1.use('root/pypi') pypistage.mock_simple( "package", '<a href="/package-1.0.zip" />\n' '<a href="/package-1.1.zip#sha256=a665a45920422f9d417e4867efdc4fb8a04a1f3fff1fa07e998e86f7f7a27ae3" />\n' '<a href="/package-1.2.zip#sha256=b3a8e0e1f9ab1bfe3a36f231f676f78bb30a519d2b21e6c530c0eee8ebb4a5d0" data-yanked="" />\n' '<a href="/package-2.0.zip#sha256=35a9e381b1a27567549b5f8a6f783c167ebf809f1c4d6a9e367240484d8ce281" data-requires-python=">=3.5" />' ) pypistage.mock_extfile("/package-1.1.zip", b"123") pypistage.mock_extfile("/package-1.2.zip", b"456") pypistage.mock_extfile("/package-2.0.zip", b"789") r = testapp.get(api.index + "/+simple/package/") assert r.status_code == 200 # fetch some files, so they are included in the dump (_, link1, link2, link3) = sorted( (x.attrs['href'] for x in r.html.select('a')), key=lambda x: x.split('/')[-1]) baseurl = URL(r.request.url) r = testapp.get(baseurl.joinpath(link1).url) assert r.body == b"123" r = testapp.get(baseurl.joinpath(link2).url) assert r.body == b"456" r = testapp.get(baseurl.joinpath(link3).url) assert r.body == b"789" impexp.export() mapp2 = impexp.new_import() with mapp2.xom.keyfs.transaction(write=False): stage = mapp2.xom.model.getstage(api.stagename) stage.offline = True projects = stage.list_projects_perstage() assert projects == {'package'} links = sorted( get_mutable_deepcopy( stage.get_simplelinks_perstage("package"))) assert links == [ ('package-1.1.zip', 'root/pypi/+f/a66/5a45920422f9d/package-1.1.zip', None, None), ('package-1.2.zip', 'root/pypi/+f/b3a/8e0e1f9ab1bfe/package-1.2.zip', None, True), ('package-2.0.zip', 'root/pypi/+f/35a/9e381b1a27567/package-2.0.zip', '>=3.5', None) ]
def _update_projects(self, projects): main_keys = [ 'path', 'name', 'user', 'index', 'classifiers', 'keywords', 'version', 'doc_version', 'type', 'text_path', 'text_title', 'text' ] for i, project in enumerate(projects, 1): data = { x: get_mutable_deepcopy(project[x]) for x in main_keys if x in project } data['path'] = u"/{user}/{index}/{name}".format(**data) data['type'] = "project" data['text'] = "%s %s" % (data['name'], project_name(data['name'])) yield dict(_index=self.index, _type=data['type'], _id=data['path'], _source=data)
def _update_project(self, project, serial, counter, main_keys, writer, searcher): def add_document(**kw): try: writer.add_document(**kw) except Exception: log.exception( "Exception while trying to add the following data to the search index:\n%r" % kw) raise text_keys = (('author', 0.5), ('author_email', 0.5), ('description', 1.5), ('summary', 1.75), ('keywords', 1.75)) data = dict((u(x), get_mutable_deepcopy(project[x])) for x in main_keys if x in project) data['path'] = u"/{user}/{index}/{name}".format(user=data['user'], index=data['index'], name=normalize_name( data['name'])) existing = None doc_num = searcher.document_number(path=data['path']) if doc_num is not None: existing = searcher.stored_fields(doc_num) if existing is not None: needs_reindex = False if ('+doczip' in project) != ('doc_version' in existing): needs_reindex = True existing_serial = existing.get('serial', -1) if existing_serial < serial: needs_reindex = True if not needs_reindex: return # because we use hierarchical documents, we have to delete # everything we got for this path and index it again writer.delete_by_term('path', data['path'], searcher=searcher) data['serial'] = serial data['type'] = "project" data['text'] = "%s %s" % (data['name'], project_name(data['name'])) with writer.group(): add_document(**data) next(counter) for key, boost in text_keys: if key not in project: continue add_document( **{ "path": data['path'], "type": key, "text": project[key], "_text_boost": boost }) next(counter) if '+doczip' not in project: return if not project['+doczip'].exists(): log.error("documentation zip file is missing %s", data['path']) return for page in project['+doczip'].values(): if page is None: continue add_document( **{ "path": data['path'], "type": "title", "text": page['title'], "text_path": page['path'], "text_title": page['title'] }) next(counter) add_document( **{ "path": data['path'], "type": "page", "text": page['text'], "text_path": page['path'], "text_title": page['title'] }) next(counter)