def create(db):
"""Create a new database with an empty application schema.
The database is specified using a SQLAlchemy style database URI.
"""
from sqlalchemy import create_engine
from pypi.schema import metadata
engine = create_engine(db)
metadata.create_all(engine)
from progressbar import ProgressBar, Bar, Percentage, AdaptiveETA
def pb(name):
return [name, ': ', Percentage(), ' ', Bar(), ' ', AdaptiveETA()]
engine = create_engine('sqlite:///pypi.db')
metadata = MetaData(bind=engine)
releases = Table('releases', metadata, autoload=True)
for n_rel, in select([func.count()]).select_from(releases).execute():
print(n_rel)
e2 = create_engine('sqlite:///p2.db')
from pypi.schema import metadata as m2
m2.bind=e2
m2.create_all()
with e2.begin() as conn:
pbar = ProgressBar(widgets=pb('Packages'))
l = list(select([releases.c.package]).select_from(releases).group_by(releases.c.package).execute())
for p, in pbar(l):
# print(p)
conn.execute(m2.tables['packages'].insert().values(name=p))
pbar = ProgressBar(widgets=pb('Releases'), maxval=n_rel)
pbar.start()
for i, r in enumerate(releases.select().execute()):
pbar.update(i+1)
try:
j = json.loads(r.json)
except ValueError:
