def copy_delim_file_to_records(
from_name: str,
to_name: str,
conversion: Conversion,
from_storage_api: StorageApi,
to_storage_api: StorageApi,
schema: Schema,
):
assert isinstance(from_storage_api, FileSystemStorageApi)
assert isinstance(to_storage_api, PythonStorageApi)
with from_storage_api.open(from_name) as f:
records = list(read_csv(f.readlines()))
mdr = as_records(records, data_format=RecordsFormat, schema=schema)
mdr = mdr.conform_to_schema()
to_storage_api.put(to_name, mdr)
def copy_delim_file_to_file_object(
from_name: str,
to_name: str,
conversion: Conversion,
from_storage_api: StorageApi,
to_storage_api: StorageApi,
schema: Schema,
):
assert isinstance(from_storage_api, FileSystemStorageApi)
assert isinstance(to_storage_api, PythonStorageApi)
with from_storage_api.open(from_name) as f:
mdr = as_records(f,
data_format=DelimitedFileObjectFormat,
schema=schema)
mdr = mdr.conform_to_schema()
to_storage_api.put(to_name, mdr)
def copy_file_object_to_delim_file(
from_name: str,
to_name: str,
conversion: Conversion,
from_storage_api: StorageApi,
to_storage_api: StorageApi,
schema: Schema,
):
assert isinstance(from_storage_api, PythonStorageApi)
assert isinstance(to_storage_api, FileSystemStorageApi)
mdr = from_storage_api.get(from_name)
file_obj_iterator = mdr.records_object
if isinstance(mdr.records_object, IOBase):
file_obj_iterator = [file_obj_iterator]
with to_storage_api.open(to_name, "w") as to_file:
for file_obj in file_obj_iterator:
to_file.write(file_obj)
def copy_records_to_delim_file(
from_name: str,
to_name: str,
conversion: Conversion,
from_storage_api: StorageApi,
to_storage_api: StorageApi,
schema: Schema,
):
assert isinstance(from_storage_api, PythonStorageApi)
assert isinstance(to_storage_api, FileSystemStorageApi)
mdr = from_storage_api.get(from_name)
records_iterator = mdr.records_object
if not isinstance(mdr.records_object, Iterator):
records_iterator = [records_iterator]
with to_storage_api.open(to_name, "w") as f:
append = False
for records in records_iterator:
write_csv(records, f, append=append)
append = True