def committee_summaries(year=2014): from mptracker.scraper.committee_summaries import SummaryScraper patcher = TablePatcher(models.CommitteeSummary, models.db.session, key_columns=['pdf_url']) summary_scraper = SummaryScraper(get_cached_session(), get_cached_session('question-pdf')) records = summary_scraper.fetch_summaries(year, get_pdf_text=True) patcher.update(records) models.db.session.commit()
def committee_summaries(year=2013): from mptracker.scraper.committee_summaries import SummaryScraper patcher = TablePatcher(models.CommitteeSummary, models.db.session, key_columns=['pdf_url']) summary_scraper = SummaryScraper(get_cached_session(), get_cached_session('question-pdf')) records = summary_scraper.fetch_summaries(year, get_pdf_text=True) patcher.update(records) models.db.session.commit()
def load(name, include_columns=None, create=True, remove=False, _file=sys.stdin): from mptracker.patcher import TablePatcher if include_columns: include_columns = set(include_columns.split(',')) def filter_record(r): return {k: r[k] for k in r if k in include_columns} else: filter_record = lambda r: r loader = TableLoader(name) patcher = TablePatcher(loader.model, db.session, key_columns=['id']) records = (filter_record(loader.decode_dict(flask.json.loads(line))) for line in _file) patcher.update(records, create=create, remove=remove) db.session.commit()
def test_remove_extra_records_honors_filter(db_app): records = [{'code': 'an', 'number': 1, 'name': "Anne"}, {'code': 'bo', 'number': 1, 'name': "Bob"}, {'code': 'cl', 'number': 2, 'name': "Claire"}, {'code': 'da', 'number': 2, 'name': "Dan"}] from mptracker.patcher import TablePatcher filter_patcher = TablePatcher( Thing, db.session, key_columns=['code'], filter={'number': 1}, ) filter_patcher.update(records) filter_patcher.update(records[:1], remove=True) assert sorted([t.name for t in Thing.query]) == ["Anne", "Claire", "Dan"]