def copy_delim_file_to_records( from_name: str, to_name: str, conversion: Conversion, from_storage_api: StorageApi, to_storage_api: StorageApi, schema: Schema, ): assert isinstance(from_storage_api, FileSystemStorageApi) assert isinstance(to_storage_api, PythonStorageApi) with from_storage_api.open(from_name) as f: records = list(read_csv(f.readlines())) mdr = as_records(records, data_format=RecordsFormat, schema=schema) mdr = mdr.conform_to_schema() to_storage_api.put(to_name, mdr)
def copy_delim_file_to_file_object( from_name: str, to_name: str, conversion: Conversion, from_storage_api: StorageApi, to_storage_api: StorageApi, schema: Schema, ): assert isinstance(from_storage_api, FileSystemStorageApi) assert isinstance(to_storage_api, PythonStorageApi) with from_storage_api.open(from_name) as f: mdr = as_records(f, data_format=DelimitedFileObjectFormat, schema=schema) mdr = mdr.conform_to_schema() to_storage_api.put(to_name, mdr)
def copy_file_object_to_delim_file( from_name: str, to_name: str, conversion: Conversion, from_storage_api: StorageApi, to_storage_api: StorageApi, schema: Schema, ): assert isinstance(from_storage_api, PythonStorageApi) assert isinstance(to_storage_api, FileSystemStorageApi) mdr = from_storage_api.get(from_name) file_obj_iterator = mdr.records_object if isinstance(mdr.records_object, IOBase): file_obj_iterator = [file_obj_iterator] with to_storage_api.open(to_name, "w") as to_file: for file_obj in file_obj_iterator: to_file.write(file_obj)
def copy_records_to_delim_file( from_name: str, to_name: str, conversion: Conversion, from_storage_api: StorageApi, to_storage_api: StorageApi, schema: Schema, ): assert isinstance(from_storage_api, PythonStorageApi) assert isinstance(to_storage_api, FileSystemStorageApi) mdr = from_storage_api.get(from_name) records_iterator = mdr.records_object if not isinstance(mdr.records_object, Iterator): records_iterator = [records_iterator] with to_storage_api.open(to_name, "w") as f: append = False for records in records_iterator: write_csv(records, f, append=append) append = True