def _download_key(self, key):
r = self.keyspace_table_matcher.search(key.name)
keyspace = r.group(2)
table = r.group(3)
host = key.name.split('/')[2]
file = key.name.split('/')[-1]
filename = "{}/{}/{}".format(keyspace, table, file)
key_full_path = os.path.join(self.restore_dir, filename)
try:
if filename.endswith('.lzo'):
uncompressed_key_full_path = re.sub('\.lzo$', '',
key_full_path)
logging.info("Decompressing %s..." % key_full_path)
lzop_pipe = decompression_pipe(uncompressed_key_full_path)
key.open_read()
for chunk in key:
lzop_pipe.stdin.write(chunk)
key.close()
out, err = lzop_pipe.communicate()
errcode = lzop_pipe.returncode
if errcode != 0:
logging.error("lzop Out: %s\nError:%s\nExit Code %d: " %
(out, err, errcode))
else:
logging.info("Downloading %s..." % key_full_path)
key.get_contents_to_filename(key_full_path)
except Exception as e:
logging.error('Unable to create "{!s}": {!s}'.format(
key_full_path, e))
return key.size
def _download_key(self, key):
r = self.keyspace_table_matcher.search(key.key)
keyspace = r.group(2)
table = r.group(3)
host = key.key.split('/')[2]
file = key.key.split('/')[-1]
prefix = '%s_' % host
# We don't want any host prefix because we restore from only one host
if self.local_restore:
prefix = ''
filename = "{}/{}/{}{}".format(keyspace, table, prefix, file)
key_full_path = os.path.join(self.restore_dir, filename)
key_object = key.get()
key_final_path = key_full_path
key_metadata_mtime = key_object['Metadata']['mtime']
key_metadata_atime = key_object['Metadata']['atime']
if key_metadata_mtime is not None and key_metadata_atime is not None:
key_mtime = int(float(key_metadata_mtime))
key_atime = int(float(key_metadata_atime))
try:
if filename.endswith('.lzo'):
uncompressed_key_full_path = re.sub('\.lzo$', '', key_full_path)
logging.info("Decompressing %s..." % key_full_path)
lzop_pipe = decompression_pipe(uncompressed_key_full_path)
key_body = key_object['Body']
for chunk in iter(lambda: key_body.read(), b''):
lzop_pipe.stdin.write(chunk)
out, err = lzop_pipe.communicate()
errcode = lzop_pipe.returncode
key_final_path = uncompressed_key_full_path
if errcode != 0:
logging.error("lzop Out: %s\nError:%s\nExit Code %d: " % (out, err, errcode))
else:
logging.info("Downloading %s..." % key_full_path)
bucket = boto3.client('s3',
aws_access_key_id=self.aws_access_key_id,
aws_secret_access_key=self.aws_secret_access_key,
region_name=self.s3_bucket_region
)
bucket.download_file(self.s3_bucket_name,key.key,key_full_path)
# Setting original timestamps from original filesystem
logging.info("Changing atime %s and mtime %s for %s..." % (key_metadata_atime, key_metadata_mtime, key_final_path))
os.utime(key_final_path, (key_atime, key_mtime))
except Exception as e:
logging.error('Unable to create "{!s}": {!s}'.format(
key_full_path, e))
return key.size
def _download_key(self, key):
r = self.keyspace_table_matcher.search(key.name)
filename = "./{!s}/{!s}/{!s}_{!s}".format(
r.group(2), r.group(3),
key.name.split('/')[2], key.name.split('/')[-1])
if filename.endswith('.lzo'):
filename = re.sub('\.lzo$', '', filename)
lzop_pipe = decompression_pipe(filename)
key.open_read()
for chunk in key:
lzop_pipe.stdin.write(chunk)
key.close()
out, err = lzop_pipe.communicate()
errcode = lzop_pipe.returncode
if errcode != 0:
logging.exception("lzop Out: %s\nError:%s\nExit Code %d: " % (out, err, errcode))
else:
key.get_contents_to_filename(filename)
return key.size
def _download_key(self, key):
r = self.keyspace_table_matcher.search(key.name)
filename = "./{!s}/{!s}/{!s}_{!s}".format(r.group(2), r.group(3),
key.name.split('/')[2],
key.name.split('/')[-1])
if filename.endswith('.lzo'):
filename = re.sub('\.lzo$', '', filename)
lzop_pipe = decompression_pipe(filename)
key.open_read()
for chunk in key:
lzop_pipe.stdin.write(chunk)
key.close()
out, err = lzop_pipe.communicate()
errcode = lzop_pipe.returncode
if errcode != 0:
logging.exception("lzop Out: %s\nError:%s\nExit Code %d: " %
(out, err, errcode))
else:
key.get_contents_to_filename(filename)
return key.size
