def update_metadata(session, tablename, user_lmds, source=-1):
"""
XXX: the wrong inputs could wipe out all of your metadata!!
existing MDs without corresonding object in user_lmds is deleted
MDs with ID values is updated
MDs without ID value is added
"""
# q = "select id, fmt, source from %s where tablename = :tn and deleted = false"
# rows = self.session.execute(q, {'tn' : tablename}).fetchall()
q = LMD.current(session).filter(LMD.tname == tablename)
resproxy = q.all()
existing_lmds = [res for res in resproxy]
if not existing_lmds:
session.add_all(user_lmds)
session.commit()
return []
if not user_lmds:
for lmd in existing_lmds:
if lmd.source <= source:
lmd.deleted = True
session.add_all(existing_lmds)
session.commit()
return []
if source == -1:
source = max([lmd.source for lmd in user_lmds])
if max([lmd.source for lmd in existing_lmds]) > source:
return []
existing_id_to_lmd = dict([(lmd.id, lmd) for lmd in existing_lmds])
user_ids = set([lmd.id for lmd in user_lmds if lmd.id is not None])
existing_ids = set([lmd.id for lmd in existing_lmds])
rm_ids = existing_ids.difference(user_ids)
updated_ids = existing_ids.intersection(user_ids)
updated_ids.update(user_ids.difference(existing_ids))
rm_lmds = filter(lambda lmd: lmd.id in rm_ids, existing_lmds)
updated_lmds = [lmd for lmd in user_lmds if lmd.id in updated_ids]
new_lmds = [lmd for lmd in user_lmds if lmd.id is None]
# of the updated lmds, which need to be re-processed?
reprocess = list(new_lmds)
for lmd in updated_lmds:
if existing_id_to_lmd[lmd.id].fmt != lmd.fmt:
reprocess.append(lmd)
for lmd in rm_lmds:
lmd.deleted = True
session.add_all(rm_lmds)
session.add_all(user_lmds)
session.commit()
return reprocess
def metadata(self):
try:
q = LMD.current(self.session)
q = q.filter(LMD.tname==self.tablename)
self.lmds = q.all()
self.session.commit()
return self.lmds
except:
traceback.print_exc()
return []
def __call__(self, tablename, lmd_ids=None):
"""
look up all LocationMetaData objects relating to this table and
call the appropriate geocoders for them
"""
if lmd_ids is not None:
lmds = self.session.query(LMD).filter(LMD.id.in_(lmd_ids)).all()
else:
q = LMD.current(meta.session)
q = q.filter(LMD.tname == tablename)
q = q.order_by(asc(LMD.loc_type))
lmds = q.all()
print "extracting from ", map(str,lmds)
annoklass = get_table_annotation(tablename)
for lmd in lmds:
extractor = LocExtractor(self.session, lmd, annoklass)
extractor()
def create_metadata_from_user_inputs(session, tablename, user_inputs):
lmds = []
for user_input in user_inputs:
if user_input.get('deleted', False):
continue
if 'id' in user_input:
lmd = session.query(LMD).get(user_input['id'])
else:
lmd = LMD()
lmd.tname = tablename
lmd.col_name = user_input['col_name']
lmd.loc_type = user_input['loc_type']
lmd.extract_type = ExtractType.FMT
lmd.fmt = user_input['format'].strip()
lmd.source = LocSource.USER
lmds.append(lmd)
return lmds