def unload(table, partial_path, mode="overwrite"): path = data_utils.make_data_path_from_key(partial_path[0]) if os.path.isdir(path): shutil.rmtree(path) if mode == "append": try: old_frame = table_reader.read_parquet( util.make_data_path_from_key(partial_path[0])) new_frame = pd.concat([old_frame, table], axis=0, ignore_index=True) _write_dataframe(new_frame, path) except: _write_dataframe(table, path) else: _write_dataframe(table, path)
def read_parquet_or_csv(path): try: # try parquet data storage first using path as key df = pd.read_parquet(path=data_util.make_data_path_from_key(path), engine='pyarrow') except pyarrow.lib.ArrowIOError: df = read_csv(path) data_util.validate_column_name(df) return df
def unload(table, partial_path, mode="overwrite"): path = data_utils.make_data_path_from_key(partial_path[0]) if path == gateway.data_root or path == gateway.data_root + '/': raise_runtime_error( 'Please check a path String and a type of path. Cannot use a root of directory for the path.' ) if os.path.isdir(path): shutil.rmtree(path) if mode == "append": try: old_frame = table_reader.read_parquet( util.make_data_path_from_key(partial_path[0])) new_frame = pd.concat([old_frame, table], axis=0, ignore_index=True) _write_dataframe(new_frame, path) except: _write_dataframe(table, path) else: _write_dataframe(table, path)
def load(partial_path=['/[email protected]/upload/sample_iris.csv']): if isinstance(partial_path, list): partial_path = partial_path[0] if partial_path.endswith("/"): partial_path = partial_path[:-1] return { 'table': table_reader.read_parquet(util.make_data_path_from_key(partial_path)) }
def load(partial_path=['/[email protected]/upload/sample_iris.csv']): return {'table': table_reader.read_parquet(util.make_data_path_from_key(partial_path[0]))}
def unload(table, partial_path): table.to_parquet(util.make_data_path_from_key(partial_path[0]))
def load(partial_path): return { 'table': table_reader.read_parquet(util.make_data_path_from_key( partial_path[0])) }
def unload(table, partial_path): path = data_utils.make_data_path_from_key(partial_path[0]) if os.path.isdir(path): shutil.rmtree(path) _write_dataframe(table, path)