def zip_ingress(data, study_id):
print "Starting zip ingress for study %s" % study_id
conn = db.engine.connect()
zpf = zipfile.ZipFile(data)
print zpf.namelist()
for file in zpf.namelist():
if not file.endswith(".csv"):
continue
if file.startswith("__MACOSX"):
continue
print "Trying to parse %s" % file
# first add empty dataset to get key
dataset = Datasets(file, study_id)
db.session.add(dataset)
db.session.commit()
dataset_id = dataset.id
# then parse file and add notes and points with Core bulk inserts
parsed = parser.parse(zpf.open(file))
selector = generate_selector(parsed['data'])
notes = [Notes.dict_from_parsed(note_text, dataset_id) for note_text in parsed['notes']]
data_points = [DataPoints.dict_from_parsed(parsed_point, dataset_id, selector) for parsed_point in parsed['data']]
# db.session.bulk_insert_mappings(DataPoints, data_points)
conn.execute(DataPoints.__table__.insert(), data_points)
# db.session.bulk_insert_mappings(Notes, notes)
conn.execute(Notes.__table__.insert(), notes)
db.session.commit()
study = Studies.query.get(study_id)
study.update_range()