def unload(table, partial_path, mode="overwrite"):
path = data_utils.make_data_path_from_key(partial_path[0])
if os.path.isdir(path):
shutil.rmtree(path)
if mode == "append":
try:
old_frame = table_reader.read_parquet(
util.make_data_path_from_key(partial_path[0]))
new_frame = pd.concat([old_frame, table],
axis=0,
ignore_index=True)
_write_dataframe(new_frame, path)
except:
_write_dataframe(table, path)
else:
_write_dataframe(table, path)
def read_parquet_or_csv(path):
try:
# try parquet data storage first using path as key
df = pd.read_parquet(path=data_util.make_data_path_from_key(path),
engine='pyarrow')
except pyarrow.lib.ArrowIOError:
df = read_csv(path)
data_util.validate_column_name(df)
return df
def unload(table, partial_path, mode="overwrite"):
path = data_utils.make_data_path_from_key(partial_path[0])
if path == gateway.data_root or path == gateway.data_root + '/':
raise_runtime_error(
'Please check a path String and a type of path. Cannot use a root of directory for the path.'
)
if os.path.isdir(path):
shutil.rmtree(path)
if mode == "append":
try:
old_frame = table_reader.read_parquet(
util.make_data_path_from_key(partial_path[0]))
new_frame = pd.concat([old_frame, table],
axis=0,
ignore_index=True)
_write_dataframe(new_frame, path)
except:
_write_dataframe(table, path)
else:
_write_dataframe(table, path)
def load(partial_path=['/[email protected]/upload/sample_iris.csv']): if isinstance(partial_path, list): partial_path = partial_path[0] if partial_path.endswith("/"): partial_path = partial_path[:-1] return { 'table': table_reader.read_parquet(util.make_data_path_from_key(partial_path)) }
def load(partial_path=['/[email protected]/upload/sample_iris.csv']): return {'table': table_reader.read_parquet(util.make_data_path_from_key(partial_path[0]))}
def unload(table, partial_path):
table.to_parquet(util.make_data_path_from_key(partial_path[0]))
def load(partial_path):
return {
'table':
table_reader.read_parquet(util.make_data_path_from_key(
partial_path[0]))
}
def unload(table, partial_path):
path = data_utils.make_data_path_from_key(partial_path[0])
if os.path.isdir(path):
shutil.rmtree(path)
_write_dataframe(table, path)
