def committee_summaries(year=2014):
from mptracker.scraper.committee_summaries import SummaryScraper
patcher = TablePatcher(models.CommitteeSummary,
models.db.session,
key_columns=['pdf_url'])
summary_scraper = SummaryScraper(get_cached_session(),
get_cached_session('question-pdf'))
records = summary_scraper.fetch_summaries(year, get_pdf_text=True)
patcher.update(records)
models.db.session.commit()
def committee_summaries(year=2013):
from mptracker.scraper.committee_summaries import SummaryScraper
patcher = TablePatcher(models.CommitteeSummary,
models.db.session,
key_columns=['pdf_url'])
summary_scraper = SummaryScraper(get_cached_session(),
get_cached_session('question-pdf'))
records = summary_scraper.fetch_summaries(year, get_pdf_text=True)
patcher.update(records)
models.db.session.commit()
def load(name, include_columns=None, create=True, remove=False,
_file=sys.stdin):
from mptracker.patcher import TablePatcher
if include_columns:
include_columns = set(include_columns.split(','))
def filter_record(r):
return {k: r[k] for k in r if k in include_columns}
else:
filter_record = lambda r: r
loader = TableLoader(name)
patcher = TablePatcher(loader.model, db.session, key_columns=['id'])
records = (filter_record(loader.decode_dict(flask.json.loads(line)))
for line in _file)
patcher.update(records, create=create, remove=remove)
db.session.commit()
def load(name, include_columns=None, create=True, remove=False,
_file=sys.stdin):
from mptracker.patcher import TablePatcher
if include_columns:
include_columns = set(include_columns.split(','))
def filter_record(r):
return {k: r[k] for k in r if k in include_columns}
else:
filter_record = lambda r: r
loader = TableLoader(name)
patcher = TablePatcher(loader.model, db.session, key_columns=['id'])
records = (filter_record(loader.decode_dict(flask.json.loads(line)))
for line in _file)
patcher.update(records, create=create, remove=remove)
db.session.commit()
def test_remove_extra_records_honors_filter(db_app):
records = [{'code': 'an', 'number': 1, 'name': "Anne"},
{'code': 'bo', 'number': 1, 'name': "Bob"},
{'code': 'cl', 'number': 2, 'name': "Claire"},
{'code': 'da', 'number': 2, 'name': "Dan"}]
from mptracker.patcher import TablePatcher
filter_patcher = TablePatcher(
Thing,
db.session,
key_columns=['code'],
filter={'number': 1},
)
filter_patcher.update(records)
filter_patcher.update(records[:1], remove=True)
assert sorted([t.name for t in Thing.query]) == ["Anne", "Claire", "Dan"]
