def create(db): """Create a new database with an empty application schema. The database is specified using a SQLAlchemy style database URI. """ from sqlalchemy import create_engine from pypi.schema import metadata engine = create_engine(db) metadata.create_all(engine)
from progressbar import ProgressBar, Bar, Percentage, AdaptiveETA def pb(name): return [name, ': ', Percentage(), ' ', Bar(), ' ', AdaptiveETA()] engine = create_engine('sqlite:///pypi.db') metadata = MetaData(bind=engine) releases = Table('releases', metadata, autoload=True) for n_rel, in select([func.count()]).select_from(releases).execute(): print(n_rel) e2 = create_engine('sqlite:///p2.db') from pypi.schema import metadata as m2 m2.bind=e2 m2.create_all() with e2.begin() as conn: pbar = ProgressBar(widgets=pb('Packages')) l = list(select([releases.c.package]).select_from(releases).group_by(releases.c.package).execute()) for p, in pbar(l): # print(p) conn.execute(m2.tables['packages'].insert().values(name=p)) pbar = ProgressBar(widgets=pb('Releases'), maxval=n_rel) pbar.start() for i, r in enumerate(releases.select().execute()): pbar.update(i+1) try: j = json.loads(r.json) except ValueError: