def download_metadata(bucket, storage_url):
'''Download old metadata backups'''
backups = sorted(bucket.list('s3ql_metadata_bak_'))
if not backups:
raise QuietError('No metadata backups found.')
log.info('The following backups are available:')
log.info('%3s %-23s %-15s', 'No', 'Name', 'Date')
for (i, name) in enumerate(backups):
params = bucket.lookup(name)
if 'last-modified' in params:
date = Datetime.fromtimestamp(params['last-modified']).strftime('%Y-%m-%d %H:%M:%S')
else:
# (metadata might from an older fs revision)
date = '(unknown)'
log.info('%3d %-23s %-15s', i, name, date)
name = None
while name is None:
buf = raw_input('Enter no to download: ')
try:
name = backups[int(buf.strip())]
except:
log.warn('Invalid input')
log.info('Downloading %s...', name)
cachepath = get_bucket_cachedir(storage_url, '.')
for i in ('.db', '.params'):
if os.path.exists(cachepath + i):
raise QuietError('%s already exists, aborting.' % cachepath+i)
os.close(os.open(cachepath + '.db', os.O_RDWR | os.O_CREAT,
stat.S_IRUSR | stat.S_IWUSR), 'w+b')
param = bucket.lookup(name)
try:
db = Connection(cachepath + '.db')
log.info('Reading metadata...')
with bucket.open_read(name) as fh:
restore_metadata(fh, db)
restore_metadata(fh, db)
except:
# Don't keep file if it doesn't contain anything sensible
os.unlink(cachepath + '.db')
raise
# Raise sequence number so that fsck.s3ql actually uses the
# downloaded backup
seq_nos = [ int(x[len('s3ql_seq_no_'):]) for x in bucket.list('s3ql_seq_no_') ]
param['seq_no'] = max(seq_nos) + 1
pickle.dump(param, open(cachepath + '.params', 'wb'), 2)
def run(self):
log.debug('MetadataDownloadThread: start')
while not self.quit:
self.event.wait(self.interval)
self.event.clear()
if self.quit:
break
with self.bucket_pool() as bucket:
#XXX: call bucket.is_get_consistent() to verify data consistency later
seq_no = get_seq_no(bucket)
if seq_no > self.param['seq_no']:
log.info('Remote metadata is newer than local (%d vs %d), '
'download it', seq_no, self.param['seq_no'])
elif seq_no < self.param['seq_no']:
log.warn('Remote metadata is older than local (%s vs %d), '
'ignore the bucket until upload metadata thread done',
seq_no, self.param['seq_no'])
continue
else:
log.info('seqno equals local (%d vs %d), ignore then download',
seq_no, self.param['seq_no'])
continue
log.info("Downloading & uncompressing metadata...")
os.close(os.open(self.cachepath + '.db.tmp',
os.O_RDWR | os.O_CREAT | os.O_TRUNC,
stat.S_IRUSR | stat.S_IWUSR))
db_conn = Connection(self.cachepath + '.db.tmp', fast_mode=True)
with bucket.open_read("s3ql_metadata") as fh:
restore_metadata(fh, db_conn)
db_conn.close()
with llfuse.lock:
if self.quit:
break
os.rename(self.cachepath + '.db.tmp', self.cachepath + '.db')
self.db_mtime = os.stat(self.cachepath + '.db').st_mtime
self.param['seq_no'] = seq_no
log.debug('MetadataDownloadThread: end')
def main(args=None):
if args is None:
args = sys.argv[1:]
options = parse_args(args)
setup_logging(options)
# Check if fs is mounted on this computer
# This is not foolproof but should prevent common mistakes
match = options.storage_url + ' /'
with open('/proc/mounts', 'r') as fh:
for line in fh:
if line.startswith(match):
raise QuietError('Can not check mounted file system.')
bucket = get_bucket(options)
cachepath = get_bucket_cachedir(options.storage_url, options.cachedir)
seq_no = get_seq_no(bucket)
param_remote = bucket.lookup('s3ql_metadata')
db = None
if os.path.exists(cachepath + '.params'):
assert os.path.exists(cachepath + '.db')
param = pickle.load(open(cachepath + '.params', 'rb'))
if param['seq_no'] < seq_no:
log.info('Ignoring locally cached metadata (outdated).')
param = bucket.lookup('s3ql_metadata')
else:
log.info('Using cached metadata.')
db = Connection(cachepath + '.db')
assert not os.path.exists(cachepath + '-cache') or param['needs_fsck']
if param_remote['seq_no'] != param['seq_no']:
log.warn('Remote metadata is outdated.')
param['needs_fsck'] = True
else:
param = param_remote
assert not os.path.exists(cachepath + '-cache')
# .db might exist if mount.s3ql is killed at exactly the right instant
# and should just be ignored.
# Check revision
if param['revision'] < CURRENT_FS_REV:
raise QuietError('File system revision too old, please run `s3qladm upgrade` first.')
elif param['revision'] > CURRENT_FS_REV:
raise QuietError('File system revision too new, please update your '
'S3QL installation.')
if param['seq_no'] < seq_no:
if bucket.is_get_consistent():
print(textwrap.fill(textwrap.dedent('''\
Up to date metadata is not available. Probably the file system has not
been properly unmounted and you should try to run fsck on the computer
where the file system has been mounted most recently.
''')))
else:
print(textwrap.fill(textwrap.dedent('''\
Up to date metadata is not available. Either the file system has not
been unmounted cleanly or the data has not yet propagated through the backend.
In the later case, waiting for a while should fix the problem, in
the former case you should try to run fsck on the computer where
the file system has been mounted most recently
''')))
print('Enter "continue" to use the outdated data anyway:',
'> ', sep='\n', end='')
if options.batch:
raise QuietError('(in batch mode, exiting)')
if sys.stdin.readline().strip() != 'continue':
raise QuietError()
param['seq_no'] = seq_no
param['needs_fsck'] = True
if (not param['needs_fsck']
and ((time.time() - time.timezone) - param['last_fsck'])
< 60 * 60 * 24 * 31): # last check more than 1 month ago
if options.force:
log.info('File system seems clean, checking anyway.')
else:
log.info('File system is marked as clean. Use --force to force checking.')
return
# If using local metadata, check consistency
if db:
log.info('Checking DB integrity...')
try:
# get_list may raise CorruptError itself
res = db.get_list('PRAGMA integrity_check(20)')
if res[0][0] != u'ok':
log.error('\n'.join(x[0] for x in res ))
raise apsw.CorruptError()
except apsw.CorruptError:
raise QuietError('Local metadata is corrupted. Remove or repair the following '
'files manually and re-run fsck:\n'
+ cachepath + '.db (corrupted)\n'
+ cachepath + '.param (intact)')
else:
log.info("Downloading & uncompressing metadata...")
os.close(os.open(cachepath + '.db.tmp', os.O_RDWR | os.O_CREAT | os.O_TRUNC,
stat.S_IRUSR | stat.S_IWUSR))
db = Connection(cachepath + '.db.tmp', fast_mode=True)
with bucket.open_read("s3ql_metadata") as fh:
restore_metadata(fh, db)
db.close()
os.rename(cachepath + '.db.tmp', cachepath + '.db')
db = Connection(cachepath + '.db')
# Increase metadata sequence no
param['seq_no'] += 1
param['needs_fsck'] = True
bucket['s3ql_seq_no_%d' % param['seq_no']] = 'Empty'
pickle.dump(param, open(cachepath + '.params', 'wb'), 2)
fsck = Fsck(cachepath + '-cache', bucket, param, db)
fsck.check()
if fsck.uncorrectable_errors:
raise QuietError("Uncorrectable errors found, aborting.")
if os.path.exists(cachepath + '-cache'):
os.rmdir(cachepath + '-cache')
log.info('Saving metadata...')
fh = tempfile.TemporaryFile()
dump_metadata(fh, db)
log.info("Compressing & uploading metadata..")
cycle_metadata(bucket)
fh.seek(0)
param['needs_fsck'] = False
param['last_fsck'] = time.time() - time.timezone
param['last-modified'] = time.time() - time.timezone
with bucket.open_write("s3ql_metadata", param) as dst:
fh.seek(0)
shutil.copyfileobj(fh, dst)
fh.close()
pickle.dump(param, open(cachepath + '.params', 'wb'), 2)
db.execute('ANALYZE')
db.execute('VACUUM')
db.close()
def upgrade(bucket):
'''Upgrade file system to newest revision'''
# Access to protected member
#pylint: disable=W0212
log.info('Getting file system parameters..')
seq_nos = [ int(x[len('s3ql_seq_no_'):]) for x in bucket.list('s3ql_seq_no_') ]
seq_no = max(seq_nos)
if not seq_nos:
raise QuietError(textwrap.dedent('''
File system revision too old to upgrade!
You need to use an older S3QL version to upgrade to a more recent
revision before you can use this version to upgrade to the newest
revision.
'''))
param = bucket.lookup('s3ql_metadata')
# Check for unclean shutdown
if param['seq_no'] < seq_no:
if bucket.is_get_consistent():
raise QuietError(textwrap.fill(textwrap.dedent('''\
It appears that the file system is still mounted somewhere else. If this is not
the case, the file system may have not been unmounted cleanly and you should try
to run fsck on the computer where the file system has been mounted most recently.
''')))
else:
raise QuietError(textwrap.fill(textwrap.dedent('''\
It appears that the file system is still mounted somewhere else. If this is not the
case, the file system may have not been unmounted cleanly or the data from the
most-recent mount may have not yet propagated through the backend. In the later case,
waiting for a while should fix the problem, in the former case you should try to run
fsck on the computer where the file system has been mounted most recently.
''')))
# Check that the fs itself is clean
if param['needs_fsck']:
raise QuietError("File system damaged, run fsck!")
# Check revision
if param['revision'] < CURRENT_FS_REV - 1:
raise QuietError(textwrap.dedent('''
File system revision too old to upgrade!
You need to use an older S3QL version to upgrade to a more recent
revision before you can use this version to upgrade to the newest
revision.
'''))
elif param['revision'] >= CURRENT_FS_REV:
print('File system already at most-recent revision')
return
print(textwrap.dedent('''
I am about to update the file system to the newest revision.
You will not be able to access the file system with any older version
of S3QL after this operation.
You should make very sure that this command is not interrupted and
that no one else tries to mount, fsck or upgrade the file system at
the same time.
'''))
print('Please enter "yes" to continue.', '> ', sep='\n', end='')
if sys.stdin.readline().strip().lower() != 'yes':
raise QuietError()
log.info('Upgrading from revision %d to %d...', CURRENT_FS_REV - 1,
CURRENT_FS_REV)
if 's3ql_hash_check_status' not in bucket:
if (isinstance(bucket, LegacyLocalBucket) or
(isinstance(bucket, BetterBucket) and
isinstance(bucket.bucket, LegacyLocalBucket))):
if isinstance(bucket, LegacyLocalBucket):
bucketpath = bucket.name
else:
bucketpath = bucket.bucket.name
for (path, _, filenames) in os.walk(bucketpath, topdown=True):
for name in filenames:
if not name.endswith('.meta'):
continue
basename = os.path.splitext(name)[0]
if '=00' in basename:
raise RuntimeError("No, seriously, you tried to break things, didn't you?")
with open(os.path.join(path, name), 'r+b') as dst:
dst.seek(0, os.SEEK_END)
with open(os.path.join(path, basename + '.dat'), 'rb') as src:
shutil.copyfileobj(src, dst)
basename = basename.replace('#', '=23')
os.rename(os.path.join(path, name),
os.path.join(path, basename))
os.unlink(os.path.join(path, basename + '.dat'))
if isinstance(bucket, LegacyLocalBucket):
bucket = LocalBucket(bucket.name, None, None)
else:
bucket.bucket = LocalBucket(bucket.bucket.name, None, None)
# Download metadata
log.info("Downloading & uncompressing metadata...")
dbfile = tempfile.NamedTemporaryFile()
with tempfile.TemporaryFile() as tmp:
with bucket.open_read("s3ql_metadata") as fh:
shutil.copyfileobj(fh, tmp)
db = Connection(dbfile.name, fast_mode=True)
tmp.seek(0)
restore_legacy_metadata(tmp, db)
# Increase metadata sequence no
param['seq_no'] += 1
bucket['s3ql_seq_no_%d' % param['seq_no']] = 'Empty'
for i in seq_nos:
if i < param['seq_no'] - 5:
del bucket['s3ql_seq_no_%d' % i ]
log.info("Uploading database..")
cycle_metadata(bucket)
param['last-modified'] = time.time() - time.timezone
with bucket.open_write("s3ql_metadata", param) as fh:
dump_metadata(fh, db)
else:
log.info("Downloading & uncompressing metadata...")
dbfile = tempfile.NamedTemporaryFile()
with tempfile.TemporaryFile() as tmp:
with bucket.open_read("s3ql_metadata") as fh:
shutil.copyfileobj(fh, tmp)
db = Connection(dbfile.name, fast_mode=True)
tmp.seek(0)
restore_metadata(tmp, db)
print(textwrap.dedent('''
The following process may take a long time, but can be interrupted
with Ctrl-C and resumed from this point by calling `s3qladm upgrade`
again. Please see Changes.txt for why this is necessary.
'''))
if 's3ql_hash_check_status' not in bucket:
log.info("Starting hash verification..")
start_obj = 0
else:
start_obj = int(bucket['s3ql_hash_check_status'])
log.info("Resuming hash verification with object %d..", start_obj)
try:
total = db.get_val('SELECT COUNT(id) FROM objects')
i = 0
for (obj_id, hash_) in db.query('SELECT obj_id, hash FROM blocks JOIN objects '
'ON obj_id == objects.id WHERE obj_id > ? '
'ORDER BY obj_id ASC', (start_obj,)):
if i % 100 == 0:
log.info(' ..checked %d/%d objects..', i, total)
sha = hashlib.sha256()
with bucket.open_read("s3ql_data_%d" % obj_id) as fh:
while True:
buf = fh.read(128*1024)
if not buf:
break
sha.update(buf)
if sha.digest() != hash_:
log.warn('Object %d corrupted! Deleting..', obj_id)
bucket.delete('s3ql_data_%d' % obj_id)
i += 1
except KeyboardInterrupt:
log.info("Storing verification status...")
bucket['s3ql_hash_check_status'] = '%d' % obj_id
raise QuietError('Aborting..')
log.info('Running fsck...')
fsck = Fsck(tempfile.mkdtemp(), bucket, param, db)
fsck.check()
if fsck.uncorrectable_errors:
raise QuietError("Uncorrectable errors found, aborting.")
param['revision'] = CURRENT_FS_REV
param['seq_no'] += 1
bucket['s3ql_seq_no_%d' % param['seq_no']] = 'Empty'
log.info("Uploading database..")
cycle_metadata(bucket)
param['last-modified'] = time.time() - time.timezone
with bucket.open_write("s3ql_metadata", param) as fh:
dump_metadata(fh, db)
def get_metadata(bucket, cachepath, readonly=False):
'''Retrieve metadata
Checks:
- Revision
- Unclean mounts
Locally cached metadata is used if up-to-date.
'''
seq_no = get_seq_no(bucket)
# Check for cached metadata
db = None
if os.path.exists(cachepath + '.params'):
param = pickle.load(open(cachepath + '.params', 'rb'))
if param['seq_no'] < seq_no:
log.info('Ignoring locally cached metadata (outdated).')
param = bucket.lookup('s3ql_metadata')
else:
log.info('Using cached metadata.')
db = Connection(cachepath + '.db')
else:
param = bucket.lookup('s3ql_metadata')
# Check for unclean shutdown
if param['seq_no'] < seq_no:
if bucket.is_get_consistent():
#raise QuietError(textwrap.fill(textwrap.dedent('''\
# It appears that the file system is still mounted somewhere else. If this is not
# the case, the file system may have not been unmounted cleanly and you should try
# to run fsck on the computer where the file system has been mounted most recently.
# ''')))
log.warn("local seqno is smaller than bucket seqno, which implies another mountpoint but local bucket is consistent, just ignore it")
else:
#raise QuietError(textwrap.fill(textwrap.dedent('''\
# It appears that the file system is still mounted somewhere else. If this is not the
# case, the file system may have not been unmounted cleanly or the data from the
# most-recent mount may have not yet propagated through the backend. In the later case,
# waiting for a while should fix the problem, in the former case you should try to run
# fsck on the computer where the file system has been mounted most recently.
# ''')))
log.warn("local seqno is smaller than bucket seqno, which implies another mountpoint and local bucket is inconsistent, could not ignore the error")
# Check revision
if param['revision'] < CURRENT_FS_REV:
raise QuietError('File system revision too old, please run `s3qladm upgrade` first.')
elif param['revision'] > CURRENT_FS_REV:
raise QuietError('File system revision too new, please update your '
'S3QL installation.')
# Check that the fs itself is clean
if not readonly and param['needs_fsck']:
raise QuietError("File system damaged or not unmounted cleanly, run fsck!")
if (time.time() - time.timezone) - param['last_fsck'] > 60 * 60 * 24 * 31:
log.warn('Last file system check was more than 1 month ago, '
'running fsck.s3ql is recommended.')
# Download metadata
if not db:
log.info("Downloading & uncompressing metadata...")
os.close(os.open(cachepath + '.db.tmp', os.O_RDWR | os.O_CREAT | os.O_TRUNC,
stat.S_IRUSR | stat.S_IWUSR))
db = Connection(cachepath + '.db.tmp', fast_mode=True)
with bucket.open_read("s3ql_metadata") as fh:
restore_metadata(fh, db)
db.close()
os.rename(cachepath + '.db.tmp', cachepath + '.db')
db = Connection(cachepath + '.db')
# Increase metadata sequence no
param['seq_no'] += 1
param['needs_fsck'] = True
bucket['s3ql_seq_no_%d' % param['seq_no']] = 'Empty'
pickle.dump(param, open(cachepath + '.params', 'wb'), 2)
param['needs_fsck'] = False
return (param, db)
