def backup_dir_ok(backup_dir=None):
'''
Verify the backup dir can be written to.
>>> from blockchain_backup.bitcoin.tests import utils as test_utils
>>> test_utils.init_database()
>>> backup_dir_ok(backup_dir='/tmp/bitcoin/backup')
(True, None)
>>> ok, error_message = backup_dir_ok(backup_dir='/test')
>>> ok
False
>>> error_message.startswith('Unable to create /test as')
True
>>> backup_dir_ok(backup_dir=get_data_dir())
(False, 'The backup and data directories must be different.')
'''
user = whoami()
error = None
if backup_dir is None:
backup_dir = get_backup_dir()
if backup_dir == get_data_dir():
ok = False
error = 'The backup and data directories must be different.'
else:
if os.path.exists(backup_dir):
dir_preexists = True
else:
dir_preexists = False
try:
os.makedirs(backup_dir)
except: # 'bare except' because it catches more than "except Exception"
error = f'Unable to create {backup_dir} as {user}'
if os.path.exists(backup_dir):
try:
filename = os.path.join(backup_dir, '.test')
with open(filename, "wt") as output_file:
output_file.write('test')
os.remove(filename)
ok = True
except: # 'bare except' because it catches more than "except Exception"
ok = False
error = f'Unable to write to the backup dir in {backup_dir} as {whoami()}.'
else:
ok = False
error = f'Unable to create {backup_dir} as {user}'
# don't create the directory while verifying its ok to create it
if not dir_preexists and os.path.exists(backup_dir):
rmtree(backup_dir)
if error is not None:
log(error)
return ok, error
def wait_for_restore(self, restore_process):
'''
Wait for the restore to finish and display data while waiting.
>>> from blockchain_backup.bitcoin.tests import utils as test_utils
>>> test_utils.init_database()
>>> restore_task = RestoreTask(os.path.join(gettempdir(), 'bitcoin/data/testnet3/backups/level1'))
>>> restore_task.manager = BitcoinManager(restore_task.log_name)
>>> restore_process = restore_task.start_restore()
>>> restore_process is not None
True
>>> restore_task.wait_for_restore(restore_process)
>>> test_utils.stop_restore()
>>> restore_task.wait_for_restore(None)
>>> test_utils.stop_restore()
'''
def show_line(line):
if line is not None and line.startswith('Copying '):
end_index = line.find(' to ')
if end_index > 0:
line = line[:end_index].strip()
if line.startswith('Copying metadata '):
filename = line[len('Copying metadata '):]
title = 'Copying metadata'
else:
filename = line[len('Copying '):]
title = 'Copying'
line = f'<strong>{title}</strong> {filename}'
self.manager.update_progress(line)
self.log('starting to wait for restore')
if restore_process is None:
log_path = os.path.join(BASE_LOG_DIR, whoami(), 'bcb-restore.log')
# wait until the log appears
while is_restore_running() and not self.is_interrupted():
if not os.path.exists(log_path):
sleep(1)
# then display the restore details
while is_restore_running() and not self.is_interrupted():
with open(log_path, 'rt') as restore_log:
show_line(restore_log.readline())
else:
while (restore_process.poll() is None
and not self.is_interrupted()):
show_line(restore_process.stdout.readline())
self.log('finished waiting for restore')
def wait_for_backup(self, backup_process):
'''
Wait for the backup to finish and display data while waiting.
>>> from blockchain_backup.bitcoin.tests import utils as test_utils
>>> test_utils.init_database()
>>> backup_task = BackupTask()
>>> backup_task.manager = BitcoinManager(backup_task.log_name)
>>> backup_task.prep_backup()
>>> backup_process, backup_pid = backup_task.start_backup()
>>> backup_process is not None
True
>>> backup_pid is None
True
>>> backup_task.wait_for_backup(backup_process)
>>> test_utils.stop_backup()
>>> backup_task.wait_for_backup(None)
>>> test_utils.stop_backup()
'''
def show_line(line):
if line is not None and line.startswith('Copying:'):
index = line.rfind(os.sep)
if index > 0:
line = self.COPYING.format(line[index + 1:])
if line != self.COPYING:
self.manager.update_progress(line)
self.log('starting to wait for backup')
if backup_process is None:
log_path = os.path.join(BASE_LOG_DIR, whoami(), 'bcb-backup.log')
# wait until the log appears
while (is_backup_running() and not self.is_interrupted()):
if not os.path.exists(log_path):
sleep(1)
# then display the backup details
while (is_backup_running() and not self.is_interrupted()):
with open(log_path, 'rt') as backup_log:
show_line(backup_log.readline())
else:
while (backup_process.poll() is None
and not self.is_interrupted()):
show_line(backup_process.stdout.readline())
if self.is_interrupted():
self.log('waiting for backup interrupted')
else:
self.log('finished waiting for backup')
def match_parent_owner(path, mode=None):
''' Chown to the parent dir's uid and gid.
If mode is present, it is passed to chmod(). '''
try:
statinfo = os.stat(os.path.dirname(path))
os.chown(path, statinfo.st_uid, statinfo.st_gid)
except: # 'bare except' because it catches more than "except Exception"
# import delayed to avoid infinite recursion
from denova.os.user import whoami
# probably don't have the right to change the file at all,
# or to change the owner to the specified user
log.error('unable to chown: user={}, uid={}, gid={}, path={}'.
format(whoami(), statinfo.st_uid, statinfo.st_gid, path))
pass
finally:
if mode is not None:
chmod(mode, path)
def data_dir_ok(data_dir=None):
'''
Verify the data dir includes the appropriate subdirs.
>>> from blockchain_backup.bitcoin.tests import utils as test_utils
>>> test_utils.init_database()
>>> data_dir_ok('/tmp/bitcoin/data')
(True, None)
>>> result, error_message = data_dir_ok('/usr/bin/backups')
>>> result == False
True
>>> error_message == 'Unable to create /usr/bin/backups as {}'.format(whoami())
True
'''
user = whoami()
error = None
if data_dir is None:
data_dir = get_data_dir()
if not os.path.exists(data_dir):
try:
log(f'trying to make data dir at: {data_dir}')
os.makedirs(data_dir)
except: # 'bare except' because it catches more than "except Exception"
error = f'Unable to create {data_dir} as {user}'
if os.path.exists(data_dir):
from blockchain_backup.bitcoin.utils import is_dir_writeable
ok, error = is_dir_writeable(data_dir)
else:
ok = False
error = f'Unable to create {data_dir} as {user}'
if error is not None:
log(error)
return ok, error
'django.middleware.locale.LocaleMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
'denova.django_addons.middleware.template.RequestMiddleware',
'denova.django_addons.middleware.template.SettingsMiddleware',
#'denova.django_addons.middleware.debug.StripCodeComments',
#'django.middleware.cache.FetchFromCacheMiddleware', # This must be last to use caching
]
# Log everything to a file. Log errors, such as HTTP 500 error when DEBUG=False, to email.
# See http://docs.djangoproject.com/en/dev/topics/logging for
# more details on how to customize your logging configuration.
DJANGO_USER = whoami()
DJANGO_LOG_DIR = f'/var/local/log/{DJANGO_USER}'
DJANGO_LOG_FILENAME = f'{DJANGO_LOG_DIR}/django.log'
try:
if not os.path.exists(DJANGO_LOG_DIR):
os.makedirs(DJANGO_LOG_DIR)
except Exception:
try:
# each log needs a unique name per user or there are permission conflicts
DJANGO_LOG_FILENAME = os.path.join(gettempdir(),
f'django_{DJANGO_USER}.log')
except Exception:
from tempfile import NamedTemporaryFile
DJANGO_LOG_FILENAME = NamedTemporaryFile(mode='a',
prefix='django',
dir=gettempdir())
def minimal_env(user=None):
'''
Get very minimal, safe chroot env.
Be sure to validate anything that comes from environment variables
before using it. According to David A. Wheeler, a common cracker's
technique is to change an environment variable.
If user is not set, gets the user from denova.os.user.whoami(). This
can flood /var/log/auth.log, so call with user set when you can.
>>> from denova.os.user import whoami
>>> env = minimal_env()
>>> '/bin:/usr/bin:/usr/local/bin' in env['PATH']
True
>>> if whoami() == 'root':
... '/sbin:/usr/sbin:/usr/local/sbin' in env['PATH']
... else:
... '/sbin:/usr/sbin:/usr/local/sbin' not in env['PATH']
True
'''
# import delayed to avoid recursive imports
from denova.os.user import whoami
if not user:
user = whoami()
env = {}
# use a minimal path
if user == 'root':
env['PATH'] = '/bin:/usr/bin:/usr/local/bin:/sbin:/usr/sbin:/usr/local/sbin'
else:
env['PATH'] = '/bin:/usr/bin:/usr/local/bin'
env_var = 'HOME'
if env_var in os.environ:
var = os.environ[env_var]
# make sure the home directory is something reasonable and reasonably safe
# Wheeler's Secure Programming warns against directories with '..'
var = os.path.abspath(var)
if os.path.exists(var):
env[env_var] = var
env_var = 'TZ'
if env_var in os.environ:
var = os.environ[env_var]
# only set the variable if it's reasonable
m = re.match(
'^([A-Za-z]+[A-Za-z_-]*/?[A-Za-z_-]*/?[A-Za-z_-]*?[A-Za-z0-9]*)$',
var)
if m and (m.group(1) == var):
env[env_var] = var
env_var = 'IFS'
if env_var in os.environ:
# force the variable to a known good value
env[env_var] = "$' \t\n'"
env_var = 'LC_ALL'
if env_var in os.environ:
# force the variable to a known good value
env[env_var] = 'C'
return env
def hostaddress(name=None):
''' Get the host ip address.
Returns None if not found.
Default is to return this host's ip.
Because this function uses gethostbyname(), be sure you are not
vulnerable to the GHOST attack.
https://security-tracker.debian.org/tracker/CVE-2015-0235
'''
ip = None
host = name or hostname()
#log.debug(f'host: {host}')
try:
host_by_name = socket.gethostbyname(host)
except socket.gaierror:
log.debug(f'no address for hostname: {host}')
else:
#log.debug(f'host by name: {host_by_name}')
if name:
ip = host_by_name
else:
# socket.gethostbyname(hostname()) can be wrong depending on what is in /etc/hosts
# but interface_from_ip() requires we are root, and we want to continue if possible
if whoami() == 'root':
interface = interface_from_ip(host_by_name)
if interface and not interface == 'lo':
log.debug(f'setting ip to host by name: {host_by_name}')
ip = host_by_name
else:
log.warning(
f'socket.gethostbyname(hostname()) returned {host_by_name}, '
+
'but no interface has that address. Is /etc/hosts wrong?'
)
else:
# accept socket.gethostbyname() because we can't verify it
ip = host_by_name
if not ip:
# use the first net device with an ip address, excluding 'lo'
for interface in interfaces():
if not ip:
if interface != 'lo':
ip = device_address(interface)
log.debug(
f'set ip to {ip} from first net device {interface}')
if ip:
log.debug(f'ip address: {ip}')
else:
msg = 'no ip address'
log.debug(msg)
raise NetException(msg)
return ip