def upload(path, handle, user, db, fs): """ upload(query_model, path, user, db, fs) -> None Retrieve the query result in the format specified and upload to hdfs. """ if fs.do_as_user(user.username, fs.exists, path): raise Exception(_("%s already exists.") % path) else: fs.do_as_user(user.username, fs.create, path) content_generator = HS2DataAdapter(handle, db, -1, start_over=True) for header, data in content_generator: dataset = export_csvxls.dataset(None, data) fs.do_as_user(user.username, fs.append, path, dataset.csv)
def upload(path, handle, user, db, fs): """ upload(query_model, path, user, db, fs) -> None Retrieve the query result in the format specified and upload to hdfs. """ has_more = True start_over = True fs.do_as_user(user.username, fs.create, path, overwrite=True) while has_more: data, has_more = HS2DataAdapter(handle, db, conf.DOWNLOAD_ROW_LIMIT.get(), start_over=start_over) dataset = export_csvxls.dataset(None, data[1:]) fs.do_as_user(user.username, fs.append, path, dataset.csv) if start_over: start_over = False
def upload(path, handle, user, db, fs): """ upload(query_model, path, user, db, fs) -> None Retrieve the query result in the format specified and upload to hdfs. """ has_more = True start_over = True if fs.do_as_user(user.username, fs.exists, path): raise Exception(_("%s already exists.") % path) else: fs.do_as_user(user.username, fs.create, path) while has_more: data, has_more = HS2DataAdapter(handle, db, conf.DOWNLOAD_ROW_LIMIT.get(), start_over=start_over) dataset = export_csvxls.dataset(None, data[1:]) fs.do_as_user(user.username, fs.append, path, dataset.csv) if start_over: start_over = False