def _store(self, df): tmp_path = temp_file.make_temporary_file() with temp_file.deleting(tmp_path): print('storing to temp hdf') store = HdfDataStore(self.schema, tmp_path) store._store(df) print('saving to s3') store_file_to_s3(self.boto_bucket, self.schema.name, tmp_path)
def _store(self, df): tmp_path = temp_file.make_temporary_file() with temp_file.deleting(tmp_path): print('storing to temp hdf') store = HdfDataStore(self.schema, tmp_path) store._store(df) print('saving to s3') store_file_to_s3(self.boto_bucket, self.schema.name, tmp_path)
def _store_chunks(self, chunks): for i, chunk in enumerate(chunks): k = self._chunk_key(i) tmp_path = temp_file.make_temporary_file() with temp_file.deleting(tmp_path): print('storing chunk to temp hdf') store = HdfDataStore(self.schema, tmp_path) store._store(chunk) print('saving chunk to s3') store_file_to_s3(self.boto_bucket, k, tmp_path)