def extract(engine): table = sl.get_table(engine, 'meeting') try: sl.update(engine, 'meeting', {}, {'status': 'inactive'}, ensure=False) sl.update(engine, 'meeting_participants', {}, {'status': 'inactive'}, ensure=False) except sqlalchemy.exc.CompileError: pass i=0 for url, org, title in get_urls(): for meeting in scrape(url, title, org): sl.upsert(engine, table, meeting, ['meetid', 'identification_code']) i+=1 if i % 100 == 0: log.info("Extracted: %s...", i)
def extract(engine): try: sl.update(engine, "representative", {}, {"status": "inactive"}, ensure=False) sl.update(engine, "contact", {}, {"status": "inactive"}, ensure=False) sl.update(engine, "financial_data", {}, {"status": "inactive"}, ensure=False) sl.update(engine, "financial_data_turnover", {}, {"status": "inactive"}, ensure=False) sl.update(engine, "person", {}, {"status": "inactive"}, ensure=False) sl.update(engine, "organisation", {}, {"status": "inactive"}, ensure=False) sl.update(engine, "accreditation", {}, {"status": "inactive"}, ensure=False) sl.update(engine, "country_of_member", {}, {"status": "inactive"}, ensure=False) sl.update(engine, "associated_action", {}, {"status": "inactive"}, ensure=False) except sqlalchemy.exc.CompileError: pass res = requests.get(URL) extract_data(engine, res.content.decode("utf-8"))
def extract(engine): try: sl.update(engine, 'representative', {}, {'status': 'inactive'}, ensure=False) sl.update(engine, 'financial_data', {}, {'status': 'inactive'}, ensure=False) sl.update(engine, 'financial_data_turnover', {}, {'status': 'inactive'}, ensure=False) sl.update(engine, 'person', {}, {'status': 'inactive'}, ensure=False) sl.update(engine, 'organisation', {}, {'status': 'inactive'}, ensure=False) sl.update(engine, 'accreditation', {}, {'status': 'inactive'}, ensure=False) sl.update(engine, 'country_of_member', {}, {'status': 'inactive'}, ensure=False) except sqlalchemy.exc.CompileError: pass res = requests.get(URL) extract_data(engine, res.content.decode('utf-8'))
def remap_subcategories(engine): table = sl.get_table(engine, 'representative') for cat in sl.distinct(engine, table, 'sub_category'): c=newsubcats.get(cat['sub_category']) if c: sl.update(engine, 'representative', {'sub_category': cat['sub_category']}, {'sub_category': c}, ensure=False)