def stop_bitcoin_qt():
'''
Stop bitcoin_qt and determine if it ended properly.
>>> init_database()
>>> stop_bitcoin_qt()
'''
seconds = 0
while (bitcoin_utils.is_bitcoin_qt_running() and seconds < 60):
try:
send_bitcoin_cli_cmd('stop')
sleep(5)
seconds += 5
except:
pass
# use brute force if necessary
if bitcoin_utils.is_bitcoin_qt_running():
bin_dir = os.path.join(virtualenv_dir(), 'bin')
args = [os.path.join(bin_dir, 'killmatch'), bitcoin_utils.bitcoin_qt()]
command.run(*args).stdout
# give it a little more time to settle down
sleep(5)
log(f'bitcoin-qt running: {bitcoin_utils.is_bitcoin_qt_running()}')
def finish_build(self):
''' Finish build with finally links.'''
virtual_bin_dir = os.path.join(self.virtualenv_dir(), 'bin')
if not os.path.exists(os.path.join(virtual_bin_dir, 'uwsgi')):
command_args = ['ln', '-s', '/usr/bin/uwsgi', virtual_bin_dir]
run(*command_args)
def move(source, dest, owner=None, group=None, perms=None):
''' Move source to dest.
move() tries to generally follow the behavior of the
'mv --force' command.
move() creates any missing parent dirs of dest.
If dest is a dir, source is copied into dest.
Otherwise, source will overwrite any existing dest.
>>> import os.path, tempfile
>>> def temp_filename(dir=None):
... if dir is None:
... dir = test_dir
... handle, path = tempfile.mkstemp(dir=dir)
... # we don't need the handle
... os.close(handle)
... return path
>>>
>>> test_dir = tempfile.mkdtemp()
>>>
>>> # test move to dir
>>> source = temp_filename()
>>> dest_dir = tempfile.mkdtemp(dir=test_dir)
>>> move(source, dest_dir)
>>> assert not os.path.exists(source)
>>> basename = os.path.basename(source)
>>> assert os.path.exists(os.path.join(dest_dir, basename))
>>>
>>> # test move to new filename
>>> source = temp_filename()
>>> dest_filename = temp_filename(dir=test_dir)
>>> move(source, dest_filename)
>>>
>>> # test move to existing filename
>>> DATA = 'data'
>>> source = temp_filename()
>>> dest_filename = temp_filename()
>>> with open(source, 'w') as sourcefile:
... sourcefile.write(DATA)
4
>>> move(source, dest_filename)
>>> assert not os.path.exists(source)
>>> assert os.path.exists(dest_filename)
>>> with open(dest_filename) as destfile:
... assert DATA == destfile.read()
>>>
>>> # clean up
>>> result = run(*['rm', '--force', '--recursive', test_dir])
>>> assert result.returncode == 0
'''
parent_dir = os.path.dirname(dest)
if not os.path.exists(parent_dir):
makedir(parent_dir, owner=owner, group=None, perms=None)
run(*['mv', '--force', source, dest])
set_attributes(dest, owner, group, perms, recursive=True)
def say(message):
''' Speak a message.
Runs a "say" program, passing the message on the command line.
Because most systems are not set up for speech, it is not an
error if the "say" program is missing or fails.
It is often easy to add a "say" program to a system. For example,
a linux system using festival for speech can use a one line script:
festival --batch "(SayText \"$*\")"
Depending on the underlying 'say' command's implementation, say()
probably does not work unless user is in the 'audio' group.
>>> say('test say')
'''
enabled = True
if enabled:
try:
from denova.os import command
# the words are unintelligible, and usually all we want is to know something happened
# message = 'tick' # just a sound #DEBUG
# os.system passes successive lines to sh
message = message.split('\n')[0]
command.run('say', *message)
except: # 'bare except' because it catches more than "except Exception"
pass
def main():
'''
Systemd shutdown for blockchain_backup.
>>> main()
not running
'''
if utils.is_bitcoin_core_running():
log.debug('bitcoin core is running')
bin_dir = utils.get_bitcoin_bin_dir()
if bin_dir:
bitcoin_cli = os.path.join(bin_dir, utils.bitcoin_cli())
log.debug('stop bitcoin core')
run(bitcoin_cli, 'stop')
# bitcoin core returns immediatly after 'stop'
# but takes a while to shut down
time.sleep(1)
if utils.is_bitcoin_core_running():
log.debug('waiting for bitcoin core to stop')
while utils.is_bitcoin_core_running():
time.sleep(1)
log.debug('stopped bitcoin core')
print('stopped bitcoin core')
else:
log.debug('unable to find bitcoin core bin dir')
print('no bitcoin core bin dir')
else:
log.debug('bitcoin core is not running')
print('not running')
def start_gunicorn_server_for_django(bin_dir, python_cmd):
''' Start the gunicorn server for django connections.'''
log('starting django webserver')
try:
GUNICORN_CONFIG_PATH = os.path.abspath(os.path.join(CURRENT_DIR, 'gunicorn.conf.py'))
log('starting gunicorn for django connections')
gunicorn_cmd = os.path.join(bin_dir, 'gunicorn')
args = []
args.append(python_cmd)
args.append(gunicorn_cmd)
args.append('--config')
args.append(GUNICORN_CONFIG_PATH)
args.append(f'{TOP_LEVEL_DOMAIN}.wsgi_django:application')
try:
run(*args)
log(f'started gunicorn for {TOP_LEVEL_DOMAIN}')
except CalledProcessError as scpe:
log('gunicorn threw a CalledProcessError while starting')
log(scpe.stderr)
raise
except Exception as e:
log('gunicorn threw an unexpected exception while starting')
log(e)
raise
except: # 'bare except' because it catches more than "except Exception"
error = format_exc()
log(error)
sys.exit(error)
def verify_failure(self, description, *test_args):
# this test should fail
log(f'Test {description}\n\t')
with cd(TMP_DIR):
try:
args = ['python3', SAFEGET_APP
] + list(test_args) + ['--verbose']
log(f'{description} args: {args}')
run(*args)
except CalledProcessError as cpe:
log(cpe)
log('Passed: Test of failure condition failed as expected')
except Exception:
log('Error in test')
self.assertFalse()
else:
log('Failed: Test of failure condition incorrectly succeeded')
self.assertFalse()
def start_uwsgi_server_for_socketio(bin_dir):
''' Start the uwsgi server for socketio connections.'''
log('starting uwsgi for socketio connections')
uwsgi_cmd = os.path.join(bin_dir, 'uwsgi')
if not os.path.exists(uwsgi_cmd):
uwsgi_cmd = 'uwsgi'
ini_path = os.path.abspath(
os.path.join(PROJECT_PATH, 'config/uwsgi_socketio.ini'))
args = [uwsgi_cmd, ini_path]
print(f'args: {args}')
try:
run(*args)
log('uwsgi socketio server started')
except CalledProcessError as scpe:
log('uwsgi socketio server threw a CalledProcessError')
log(scpe.stderr)
raise
except Exception as e:
log('uwsgi socketio server threw an unexpected exception')
log(e)
raise
except: # 'bare except' because it catches more than "except Exception"
log('uwsgi socketio server threw an unexpected error')
raise
log('socketio server started')
def stop_bitcoind():
''' Stop bitcoind if it's running. '''
log('stopping bitcoind')
bin_dir = os.path.join(virtualenv_dir(), 'bin')
try:
SHUTDOWN_PATH = os.path.abspath(os.path.join(CURRENT_DIR, 'bitcoin_shutdown.py'))
python_cmd = os.path.join(bin_dir, 'python')
args = []
args.append(python_cmd)
args.append(SHUTDOWN_PATH)
try:
run(*args)
except CalledProcessError as scpe:
log(scpe.stderr)
except Exception as e:
log(e)
except: # 'bare except' because it catches more than "except Exception"
error = format_exc()
log(error)
log('bitcoind stopped')
def remove(path):
''' Remove the path.
If path is a dir, empty it first by rsyncing an empty dir to the
path. With a large directory this is much faster than rm or
shutil.rmtree().
It is not an error if the path does not exist.
If path is a link, only remove the link, not the target.
If path is a mount, raise ValueError.
'''
global empty_dir
if os.path.exists(path):
if empty_dir is None:
empty_dir = tempfile.mkdtemp(prefix='denova.os.fs.empty.')
if os.path.ismount(path):
raise ValueError(f'path is mount point: {path}')
elif os.path.islink(path) or os.path.isfile(path):
run(*['rm', '--force', path])
else:
assert os.path.isdir(path)
run(*['rm', '--force', '--recursive', path])
assert not os.path.exists(path), f'could not remove {path}'
def pip_install_pkgs(pip3_command):
''' Pip install packages needed during setup. '''
# pip install without input and ignore if it already exists
run(*[
pip3_command, 'install', 'pexpect', '--no-input', '--exists-action',
'i'
])
def update(self):
'''
Automatically update the blockchain.
Returns:
True if successful.
False if any errors.
'''
ok = True
try:
bitcoind_process, bitcoind_pid = core_utils.start_bitcoind(self.bin_dir, self.data_dir)
if bitcoind_process is None and bitcoind_pid is None:
self.print_on_same_line('Error starting bitcoind')
ok = False
else:
self.wait_while_updating(bitcoind_process, secs_to_wait=self.WAIT_SECONDS)
ok, error_message = core_utils.stop_bitcoind(
bitcoind_process, bitcoind_pid, self.bin_dir, self.data_dir)
except BitcoinException as be:
ok = False
error_message = str(be)
log(error_message)
except KeyboardInterrupt:
self.interrupted = True
log('^C typed; interrupting update')
if core_utils.is_bitcoind_running():
print('^C typed; Stopping update. Please wait...')
program = os.path.join(PROJECT_PATH, 'config', 'bitcoin_shutdown.py')
try:
command.run('python3', program)
except Exception as exc:
log(exc)
raise
ok = False
self.interrupted = False
except: # 'bare except' because it catches more than "except Exception"
log(format_exc())
# sometimes bitcoin exits with a non-zero return code,
# but it was still ok, so check the logs
ok, error_message = core_utils.bitcoin_finished_ok(self.data_dir,
core_utils.is_bitcoind_running)
if ok:
self.print_on_same_line(f'Remaining blocks to update at {self.get_current_time()}: 0')
self.print_on_same_line('Finished updating blockchain')
return ok
def mount(*args, **kwargs):
''' Convenience function for mount. '''
try:
all_args = ['mount']
for arg in args:
all_args.append(arg)
run(*all_args, **kwargs)
except Exception as exc:
log.debug(exc)
raise
def run(self):
try:
if DEBUG: log(f'in run kill pid: {self.killpid}') # DEBUG
#kill(self.killpid)
run('killsafe', self.killpid)
except Exception as e:
# unclear why killsafe returns an error even though it
# does successfully kill the job, but we'll just
# ignore the error
log(e)
print(e)
def stop():
log('stopping django server')
try:
run('fuser', '--kill', f'{DJANGO_PORT}/tcp')
except CalledProcessError as scpe:
log('django server threw a CalledProcessError while stopping')
log(scpe)
run('killmatch', 'blockchain-backup-django-server')
except Exception as e:
log('django servers threw an unexpected exception while stopping')
log(e)
log('django server stopped')
def different(file1, file2):
''' Returns whether the files are different. '''
# diff succeeds if there is a difference, and fails if no difference
try:
from denova.os import command
command.run('diff', file1, file2, brief=True)
different = False
except CalledProcessError:
different = True
return different
def copy_service_file(self, filename):
'''
Copy service file to the config subdirectory.
'''
SERVICE_PATH = os.path.join('/etc/systemd/system', filename)
if os.path.exists(SERVICE_PATH):
try:
command_args = ['cp', SERVICE_PATH, self.current_dir]
run(*command_args)
except:
# if this fails, that's ok
pass
def make_exec_and_link(virtual_bin_dir, path, to_path=None):
''' Make the path executable and link to virtualenv bin. '''
command_args = ['chmod', '+x', path]
run(*command_args)
if to_path:
command_args = [
'ln', '-s', path,
os.path.join(virtual_bin_dir, to_path)
]
else:
command_args = ['ln', '-s', path, virtual_bin_dir]
run(*command_args)
def interrupt_restore(self):
'''
User wants the restoration interrupted.
This is not recommended because it almost
always leaves the blockchain in unusable shape.
>>> from blockchain_backup.bitcoin.tests import utils as test_utils
>>> test_utils.init_database()
>>> restore_dir = os.path.join(gettempdir(), 'bitcoin/data/testnet3/backups/level1')
>>> restore_task = RestoreTask(restore_dir)
>>> restore_task.manager = BitcoinManager(restore_task.log_name)
>>> test_utils.start_fake_restore()
>>> restore_task.interrupt_restore()
True
'''
max_secs = 3
seconds = 0
try:
bin_dir = os.path.join(virtualenv_dir(), 'bin')
args = [
os.path.join(bin_dir, 'killmatch'), constants.RESTORE_PROGRAM
]
args = [
os.path.join('/usr/local/bin', 'killmatch'),
constants.RESTORE_PROGRAM
]
self.log(f'args: {args}')
attempts = 0
while bitcoin_utils.is_restore_running() and attempts < 5:
command.run(*args)
if bitcoin_utils.is_restore_running():
sleep(3)
attempts += 1
except CalledProcessError as cpe:
self.log(cpe)
self.log(format_exc())
# a new page was displayed so give socketio time to connect
while seconds < max_secs:
self.manager.update_header(self.STOPPED_RESTORE)
self.manager.update_subnotice(self.STOP_RESTORE_NOT_COMPLETE)
self.manager.notify_done()
sleep(1)
seconds += 1
# return value is for testing purposes only
return not bitcoin_utils.is_restore_running()
def stop():
log('stopping socketio server')
try:
run('fuser', '--kill', f'{SOCKETIO_PORT}/tcp')
except CalledProcessError as cpe:
log('socketio server threw a CalledProcessError')
log(cpe)
try:
run('killmatch', 'blockchain-backup-socketio-server')
except: # 'bare except' because it catches more than "except Exception"
log(format_exc())
except Exception as e:
log('socketio server threw an unexpected exception')
log(e)
log('socketio server stopped')
def list_block_devices():
''' List block devices, i.e. drives and partitions
>>> blocks = list_block_devices()
'''
blocks = []
lsblk_out = run('lsblk', '--list').stdout
""" Example output::
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
sda 8:0 0 698.7G 0 disk
sda1 8:1 0 694.8G 0 part /
sda2 8:2 0 1K 0 part
sda5 8:5 0 3.9G 0 part [SWAP]
sdb 8:16 1 970.5M 0 disk
sdb1 8:17 1 958M 0 part
sr0 11:0 1 1024M 0 rom
"""
lines = lsblk_out.split('\n')
for line in lines:
if line.startswith('s'):
name = line.split()[0]
blocks.append(name)
return blocks
def import_module(name):
'''
Import with debugging
>>> module_str = str(import_module("denova.os.user"))
>>> module_str.startswith("<module 'denova.os.user' from ")
True
>>> module_str.endswith("denova/os/user.py'>")
True
'''
try:
log(f'import_module({name})') #DEBUG
module = importlib.import_module(name)
log(f'import_module() result: {module}') #DEBUG
except ImportError as imp_error:
log(f'unable to import {name}')
log('ImportError: ' + str(imp_error))
msg = f'could not import {name}'
log(msg)
# find out why
from denova.os import command
log(command.run(['python3', '-c', f'import {name}']).stderr)
raise ImportError(msg)
return module
def program_status(program_name):
'''
Returns a list of matching raw lines from "ps" if program is running.
If no lines match, returns an empty list.
Ignores any program_name that is defunct.
>>> lines = program_status('python3')
>>> lines == []
False
'''
PS_ARGS = ['-eo', 'pid,args']
lines = []
try:
raw = run('ps', *PS_ARGS)
raw_lines = raw.stdout.strip().split('\n')
log(f'raw lines:\n{raw_lines}')
for line in raw_lines:
line = str(line).strip()
# apparent the leading space matters
if program_name in line and ' <defunct>' not in line:
lines.append(line)
except: # 'bare except' because it catches more than "except Exception"
log(format_exc())
return lines
def stop():
log('stopping django server')
# make sure bitcoind is shutdown before we shut down this server
stop_bitcoind()
try:
run(*['fuser', '--kill', f'{DJANGO_PORT}/tcp'])
except CalledProcessError as scpe:
log('django server threw a CalledProcessError while stopping')
log(scpe)
run(*['killmatch', 'blockchain-backup-server'])
except Exception as e:
log('django servers threw an unexpected exception while stopping')
log(e)
log('django server stopped')
def devices():
''' Return list of devices that may have filesystems. '''
# block devices may have filesystems
raw_output = run(*['lsblk', '--noheadings', '--list', '--paths', '--output=NAME']).stdout.decode()
devices = raw_output.strip().split('\n')
return devices
def send_bitcoin_cli_cmd(arg):
'''
Send a command via bitcoin-cli.
>>> init_database()
>>> start_bitcoind()
>>> block_count = send_bitcoin_cli_cmd('getblockcount')
>>> stop_bitcoind()
'''
bin_dir, data_dir = preferences.get_bitcoin_dirs()
command_args = []
command_args.append(os.path.join(bin_dir, bitcoin_utils.bitcoin_cli()))
use_test_net = '-testnet' in preferences.get_extra_args()
if use_test_net:
command_args.append('-testnet')
if data_dir is not None:
data_dir = bitcoin_utils.strip_testnet_from_data_dir(data_dir=data_dir)
command_args.append(f'-datadir={data_dir}')
command_args.append(arg)
log(f'running: {command_args}')
try:
result = command.run(*command_args).stdout
log(f'result: {result}')
except CalledProcessError as cpe:
result = None
return result
def build(self):
os.chdir(self.parent_dirname())
self.init_virtualenv()
if self.virtualenv_dir_exists():
self.log(f'building virtual environment in {self.parent_dirname()}')
os.chdir(self.virtualenv_dir())
# activate the virtualenv
with venv(dirname=self.virtualenv_dir()):
# set up a link to the python lib for simplier config
dirname = None
entries = os.scandir(os.path.join(self.virtualenv_dir(), 'lib'))
for entry in entries:
if entry.name.startswith('python3'):
dirname = entry.name
break
if dirname is None:
dirname = 'python3.7'
os.chdir('lib')
run('ln', '-s', dirname, 'python')
self.report(' installing requirements')
with open(self.get_requirements()) as f:
for line in f.readlines():
if line.strip():
app = line.strip()
self.report(f' {app}')
try:
run('pip3', 'install', app)
except CalledProcessError as cpe:
self.log(format_exc())
if cpe.stdout: self.log(f'stdout: {cpe.stdout}')
if cpe.stderr: self.log(f'stderr: {cpe.stderr}')
sys.exit(
f'{cpe.stderr}. For more details see {self.log.pathname}')
self.log(' linking packages')
self.link_packages(os.path.join(self.virtualenv_dir(), 'lib', 'python', 'site-packages'))
self.finish_build()
self.log(' virtual environment built')
else:
self.log(f'!!!Error: Unable to create {self.virtualenv_dir()}')
def config_logs(project_dir, primary_user):
''' Configure the log directory and logging server. '''
def config_user_log_dir(user_log_dir, user):
if not os.path.exists(user_log_dir):
os.mkdir(user_log_dir)
run(*['chown', f'{user}:{user}', user_log_dir])
run(*['chmod', 'u+rwx,g-rwx,o-rwx', user_log_dir])
MAIN_LOG_DIR = os.path.join(os.sep, 'var', 'local', 'log')
if not os.path.exists(MAIN_LOG_DIR):
os.makedirs(MAIN_LOG_DIR)
run(*['chmod', 'u+rwx,g+rx,o+rx', MAIN_LOG_DIR])
config_user_log_dir(os.path.join(MAIN_LOG_DIR, 'root'), 'root')
config_user_log_dir(os.path.join(MAIN_LOG_DIR, 'www-data'), 'www-data')
config_user_log_dir(os.path.join(MAIN_LOG_DIR, primary_user), primary_user)
def clear_user_logs(user):
''' Clear all logs created by this module for the user.
The caller must have the correct file permissions.
'''
logdir = os.path.join(BASE_LOG_DIR, user)
if os.listdir(logdir):
run('rm', f'{logdir}/*')
# if denova.python.log.DEBUGGING is True, then denova.python.log creates alt logs
alt_log = f'/tmp/_log.{user}.log'
if os.path.exists(alt_log):
os.remove(alt_log)
default_log = f'/tmp/python.default.{user}.log'
if os.path.exists(default_log):
os.remove(default_log)
def chgrp(group, path, recursive=False):
''' Change group of path.
'group' can be the group name, gid as an int, or gid as a string.
Log and reraise any exception.
'''
try:
if recursive:
run(*['chgrp', '--recursive', group, path])
else:
run(*['chgrp', group, path])
#log.debug(f'chgrp(group={group}, path=={path})')
except CalledProcessError as cpe:
# import delayed to avoid infinite recursion
from denova.os.user import whoami
log.error(f'unable to chgrp: user={whoami()}, group={group}, path={path}')
log.error(cpe)
raise
# verify. after we have higher confidence, move this into doctests
if type(group) is str and ':' in group:
owner, group = group.split(':')
else:
owner = None
if owner is not None:
try:
uid = int(owner)
except ValueError:
# import delayed to avoid infinite recursion
from denova.os.user import getuid
uid = getuid(owner)
assert getuid(path) == uid, f'uid set to {uid} but is {getuid(path)}'
try:
gid = int(group)
except ValueError:
# import delayed to avoid infinite recursion
from denova.os.user import getuid
gid = getgid(group)
assert getgid(path) == gid, f'gid set to {gid} but is {getgid(path)}'