def _load(self):
tmp_path = temp_file.make_temporary_file()
with temp_file.deleting(tmp_path):
print('loading from s3')
load_file_from_s3(self.boto_bucket, self.schema.name, tmp_path)
print('loading from hdf')
store = HdfDataStore(self.schema, tmp_path)
return store._load()
def _load(self):
tmp_path = temp_file.make_temporary_file()
with temp_file.deleting(tmp_path):
print('loading from s3')
load_file_from_s3(self.boto_bucket, self.schema.name, tmp_path)
print('loading from hdf')
store = HdfDataStore(self.schema, tmp_path)
return store._load()
def _load_chunks(self):
for i in count():
k = self._chunk_key(i)
if not key_exists(self.boto_bucket, k):
break
tmp_path = temp_file.make_temporary_file()
with temp_file.deleting(tmp_path):
print('loading from s3')
load_file_from_s3(self.boto_bucket, k, tmp_path)
print('loading from hdf')
store = HdfDataStore(self.schema, tmp_path)
chunk = store._load()
yield chunk
