def extract(engine):
table = sl.get_table(engine, 'meeting')
try:
sl.update(engine, 'meeting', {}, {'status': 'inactive'}, ensure=False)
sl.update(engine, 'meeting_participants', {}, {'status': 'inactive'}, ensure=False)
except sqlalchemy.exc.CompileError:
pass
i=0
for url, org, title in get_urls():
for meeting in scrape(url, title, org):
sl.upsert(engine, table, meeting, ['meetid', 'identification_code'])
i+=1
if i % 100 == 0:
log.info("Extracted: %s...", i)
def extract(engine):
try:
sl.update(engine, "representative", {}, {"status": "inactive"}, ensure=False)
sl.update(engine, "contact", {}, {"status": "inactive"}, ensure=False)
sl.update(engine, "financial_data", {}, {"status": "inactive"}, ensure=False)
sl.update(engine, "financial_data_turnover", {}, {"status": "inactive"}, ensure=False)
sl.update(engine, "person", {}, {"status": "inactive"}, ensure=False)
sl.update(engine, "organisation", {}, {"status": "inactive"}, ensure=False)
sl.update(engine, "accreditation", {}, {"status": "inactive"}, ensure=False)
sl.update(engine, "country_of_member", {}, {"status": "inactive"}, ensure=False)
sl.update(engine, "associated_action", {}, {"status": "inactive"}, ensure=False)
except sqlalchemy.exc.CompileError:
pass
res = requests.get(URL)
extract_data(engine, res.content.decode("utf-8"))
def extract(engine):
try:
sl.update(engine, 'representative', {}, {'status': 'inactive'}, ensure=False)
sl.update(engine, 'financial_data', {}, {'status': 'inactive'}, ensure=False)
sl.update(engine, 'financial_data_turnover', {}, {'status': 'inactive'}, ensure=False)
sl.update(engine, 'person', {}, {'status': 'inactive'}, ensure=False)
sl.update(engine, 'organisation', {}, {'status': 'inactive'}, ensure=False)
sl.update(engine, 'accreditation', {}, {'status': 'inactive'}, ensure=False)
sl.update(engine, 'country_of_member', {}, {'status': 'inactive'}, ensure=False)
except sqlalchemy.exc.CompileError:
pass
res = requests.get(URL)
extract_data(engine, res.content.decode('utf-8'))
def remap_subcategories(engine):
table = sl.get_table(engine, 'representative')
for cat in sl.distinct(engine, table, 'sub_category'):
c=newsubcats.get(cat['sub_category'])
if c:
sl.update(engine, 'representative', {'sub_category': cat['sub_category']}, {'sub_category': c}, ensure=False)
