def __init__(self, ncpus):
self._ncpus = ncpus
if six.PY34:
try:
multiprocessing.set_start_method('spawn')
except RuntimeError:
pass
multiprocessing.util.log_to_stderr(multiprocessing.util.SUBDEBUG)
from system import bpio
if bpio.Windows():
from system import deploy
deploy.init_base_dir()
venv_python_path = os.path.join(deploy.current_base_dir(), 'venv',
'Scripts', 'BitDustNode.exe')
lg.info('will use %s as multiprocessing executable' %
venv_python_path)
multiprocessing.set_executable(venv_python_path)
self.processor = multiprocessing.Pool(ncpus)
#: implement queue per Manager instance
# self.queue = multiprocessing.Queue()
self.tasks = OrderedDict({})
self.task_id = 0
self.thread = Thread(target=func_thread,
args=(self.tasks, self.processor))
self.thread.daemon = True
self.thread.start()
self._propagate_queue()
def main(executable_path=None, start_reactor=True):
"""
THIS IS THE ENTRY POINT OF THE PROGRAM!
"""
global AppDataDir
pars = parser()
(opts, args) = pars.parse_args()
overDict = override_options(opts, args)
cmd = ''
if len(args) > 0:
cmd = args[0].lower()
#---install---
if cmd in [
'deploy',
'install',
'venv',
'virtualenv',
]:
from system import deploy
return deploy.run(args)
try:
from system import deploy
except:
dirpath = os.path.dirname(os.path.abspath(sys.argv[0]))
sys.path.insert(0, os.path.abspath(os.path.join(dirpath, '..')))
# sys.path.insert(0, os.path.abspath(os.path.join(dirpath, '..', '..')))
from distutils.sysconfig import get_python_lib
sys.path.append(os.path.join(get_python_lib(), 'bitdust'))
try:
from system import deploy
except:
print('ERROR! can not import working code. Python Path:')
print('\n'.join(sys.path))
return 1
if opts.appdir:
appdata = opts.appdir
AppDataDir = appdata
else:
curdir = os.getcwd(
) # os.path.dirname(os.path.abspath(sys.executable))
appdatafile = os.path.join(curdir, 'appdata')
# defaultappdata = os.path.join(os.path.expanduser('~'), '.bitdust')
defaultappdata = deploy.base_dir_portable()
appdata = defaultappdata
if os.path.isfile(appdatafile):
try:
appdata = os.path.abspath(
open(appdatafile, 'rb').read().strip())
except:
appdata = defaultappdata
if not os.path.isdir(appdata):
appdata = defaultappdata
AppDataDir = appdata
#---BitDust Home
deploy.init_base_dir(base_dir=AppDataDir)
from logs import lg
# init IO module, update locale
from system import bpio
bpio.init()
# sys.excepthook = lg.exception_hook
if not bpio.isFrozen():
try:
from twisted.internet.defer import setDebugging
setDebugging(True)
# from twisted.python import log as twisted_log
# twisted_log.startLogging(sys.stdout)
except:
lg.warn('python-twisted is not installed')
# ask to count time for each log line from that moment, not absolute time
lg.life_begins()
# try to read debug level value at the early stage - no problem if fail here
try:
if cmd == '' or cmd == 'start' or cmd == 'go' or cmd == 'show' or cmd == 'open':
lg.set_debug_level(
int(
bpio.ReadTextFile(
os.path.abspath(
os.path.join(appdata, 'config', 'logs',
'debug-level')))))
except:
pass
if opts.no_logs:
lg.disable_logs()
#---logpath---
logpath = os.path.join(appdata, 'logs', 'start.log')
if opts.output:
logpath = opts.output
need_redirecting = False
if bpio.Windows() and not bpio.isConsoled():
need_redirecting = True
if logpath != '':
lg.open_log_file(logpath)
lg.out(2, 'bpmain.main log file opened ' + logpath)
if bpio.Windows() and bpio.isFrozen():
need_redirecting = True
if need_redirecting:
lg.stdout_start_redirecting()
lg.out(2, 'bpmain.main redirecting started')
# very basic solution to record run-time errors
try:
if os.path.isfile(os.path.join(appdata, 'logs', 'exception.log')):
os.remove(os.path.join(appdata, 'logs', 'exception.log'))
except:
pass
if opts.debug or str(opts.debug) == '0':
lg.set_debug_level(opts.debug)
# if opts.quite and not opts.verbose:
# lg.disable_output()
if opts.verbose:
copyright_text()
lg.out(2, 'bpmain.main started ' + time.asctime())
lg.out(2, 'bpmain.main args=%s' % str(args))
#---start---
if cmd == '' or cmd == 'start' or cmd == 'go':
appList = bpio.find_main_process(
pid_file_path=os.path.join(appdata, 'metadata', 'processid'))
if appList:
lg.out(
0, 'BitDust already started, found another process: %s\n' %
str(appList))
bpio.shutdown()
return 0
UI = ''
# if cmd == 'show' or cmd == 'open':
# UI = 'show'
try:
ret = run(UI, opts, args, overDict, executable_path, start_reactor)
except:
lg.exc()
ret = 1
bpio.shutdown()
return ret
#---daemon---
elif cmd == 'detach' or cmd == 'daemon':
appList = bpio.find_main_process(
pid_file_path=os.path.join(appdata, 'metadata', 'processid'))
if len(appList) > 0:
lg.out(0,
'main BitDust process already started: %s\n' % str(appList))
bpio.shutdown()
return 0
from lib import misc
lg.out(
0,
'new BitDust process will be started in daemon mode, finishing current process\n'
)
bpio.shutdown()
result = misc.DoRestart(
detach=True,
std_out=os.path.join(appdata, 'logs', 'stdout.log'),
std_err=os.path.join(appdata, 'logs', 'stderr.log'),
)
if result is not None:
try:
result = int(result)
except:
try:
result = result.pid
except:
pass
return 0
#---restart---
elif cmd == 'restart' or cmd == 'reboot':
appList = bpio.find_main_process(
pid_file_path=os.path.join(appdata, 'metadata', 'processid'))
ui = False
if len(appList) > 0:
lg.out(
0,
'found main BitDust process: %s, sending "restart" command ... '
% str(appList), '')
def done(x):
lg.out(0, 'DONE\n', '')
from twisted.internet import reactor # @UnresolvedImport
if reactor.running and not reactor._stopped: # @UndefinedVariable
reactor.stop() # @UndefinedVariable
def failed(x):
ok = str(x).count('Connection was closed cleanly') > 0
from twisted.internet import reactor # @UnresolvedImport
if ok and reactor.running and not reactor._stopped: # @UndefinedVariable
lg.out(0, 'DONE\n', '')
reactor.stop() # @UndefinedVariable
return
lg.out(
0,
'FAILED while killing previous process - do HARD restart\n',
'')
try:
kill()
except:
lg.exc()
from lib import misc
reactor.addSystemEventTrigger( # @UndefinedVariable
'after',
'shutdown',
misc.DoRestart,
param='show' if ui else '',
detach=True,
std_out=os.path.join(appdata, 'logs', 'stdout.log'),
std_err=os.path.join(appdata, 'logs', 'stderr.log'),
)
reactor.stop() # @UndefinedVariable
try:
from twisted.internet import reactor # @UnresolvedImport
# from interface.command_line import run_url_command
# d = run_url_command('?action=restart', False)
# from interface import cmd_line
# d = cmd_line.call_xmlrpc_method('restart', ui)
from interface import cmd_line_json
d = cmd_line_json.call_jsonrpc_method('restart', ui)
d.addCallback(done)
d.addErrback(failed)
reactor.run() # @UndefinedVariable
bpio.shutdown()
return 0
except:
lg.exc()
bpio.shutdown()
return 1
else:
ui = ''
if cmd == 'restart':
ui = 'show'
try:
ret = run(ui, opts, args, overDict, executable_path)
except:
lg.exc()
ret = 1
bpio.shutdown()
return ret
#---show---
elif cmd == 'show' or cmd == 'open':
if not bpio.isGUIpossible():
lg.out(0, 'BitDust GUI is turned OFF\n')
bpio.shutdown()
return 0
if bpio.Linux() and not bpio.X11_is_running():
lg.out(0, 'this operating system not supported X11 interface\n')
bpio.shutdown()
return 0
appList = bpio.find_main_process(
pid_file_path=os.path.join(appdata, 'metadata', 'processid'))
if len(appList) == 0:
try:
ret = run('show', opts, args, overDict, executable_path)
except:
lg.exc()
ret = 1
bpio.shutdown()
return ret
lg.out(
0,
'found main BitDust process: %s, start the GUI\n' % str(appList))
ret = show()
bpio.shutdown()
return ret
#---stop---
elif cmd == 'stop' or cmd == 'kill' or cmd == 'shutdown':
appList = bpio.find_main_process(pid_file_path=os.path.join(
appdata, 'metadata', 'processid'), )
if len(appList) > 0:
lg.out(
0,
'found main BitDust process: %r, sending command "exit" ... ' %
appList, '')
try:
from twisted.internet import reactor # @UnresolvedImport
# from interface.command_line import run_url_command
# url = '?action=exit'
# run_url_command(url, False).addBoth(wait_then_kill)
# reactor.run()
# bpio.shutdown()
def _stopped(x):
lg.out(0, 'BitDust process finished correctly\n')
reactor.stop() # @UndefinedVariable
bpio.shutdown()
# from interface import cmd_line
# cmd_line.call_xmlrpc_method('stop').addBoth(_stopped)
from interface import cmd_line_json
cmd_line_json.call_jsonrpc_method('stop').addBoth(_stopped)
reactor.run() # @UndefinedVariable
return 0
except:
lg.exc()
ret = kill()
bpio.shutdown()
return ret
else:
appListAllChilds = bpio.find_main_process(
check_processid_file=False,
extra_lookups=[
'regexp:^.*python.*bitdust.py.*?$',
],
)
if len(appListAllChilds) > 0:
lg.out(
0,
'BitDust child processes found: %r, performing "kill process" actions ...\n'
% appListAllChilds, '')
ret = kill()
return ret
lg.out(0, 'BitDust is not running at the moment\n')
bpio.shutdown()
return 0
#---command_line---
from interface import cmd_line_json as cmdln
ret = cmdln.run(opts, args, pars, overDict, executable_path)
if ret == 2:
print(usage_text())
bpio.shutdown()
return ret
def main(executable_path=None, start_reactor=True):
"""
THE ENTRY POINT
"""
global AppDataDir
pars = parser()
(opts, args) = pars.parse_args()
if opts.coverage:
import coverage # @UnresolvedImport
cov = coverage.Coverage(config_file=opts.coverage_config)
cov.start()
overDict = override_options(opts, args)
cmd = ''
if len(args) > 0:
cmd = args[0].lower()
try:
from system import deploy
except:
dirpath = os.path.dirname(os.path.abspath(sys.argv[0]))
sys.path.insert(0, os.path.abspath(os.path.join(dirpath, '..')))
from distutils.sysconfig import get_python_lib
sys.path.append(os.path.join(get_python_lib(), 'bitdust'))
try:
from system import deploy
except:
print_text('ERROR! can not import working code. Python Path:')
print_text('\n'.join(sys.path))
return 1
#---install---
if cmd in [
'deploy',
'install',
'venv',
'virtualenv',
]:
from system import deploy
return deploy.run(args)
if opts.appdir:
appdata = opts.appdir
AppDataDir = appdata
else:
curdir = os.getcwd()
appdatafile = os.path.join(curdir, 'appdata')
defaultappdata = deploy.default_base_dir_portable()
appdata = defaultappdata
if os.path.isfile(appdatafile):
try:
appdata = os.path.abspath(
open(appdatafile, 'rb').read().strip())
except:
appdata = defaultappdata
if not os.path.isdir(appdata):
appdata = defaultappdata
AppDataDir = appdata
#---BitDust Home
deploy.init_base_dir(base_dir=AppDataDir)
from logs import lg
#---init IO module
from system import bpio
bpio.init()
appList = bpio.find_main_process(
pid_file_path=os.path.join(appdata, 'metadata', 'processid'))
if bpio.Android():
lg.close_intercepted_log_file()
lg.open_intercepted_log_file(
'/storage/emulated/0/Android/data/org.bitdust_io.bitdust1/files/Documents/.bitdust/logs/android.log'
)
# sys.excepthook = lg.exception_hook
#---init logging
from twisted.internet.defer import setDebugging
if _Debug:
if bpio.isFrozen():
setDebugging(False)
else:
setDebugging(True)
else:
setDebugging(False)
from twisted.logger import globalLogPublisher, LogLevel
tw_log_observer = TwistedUnhandledErrorsObserver(level=LogLevel.critical)
globalLogPublisher.addObserver(tw_log_observer)
#---life begins!
# ask logger to count time for each log line from that moment, not absolute time
lg.life_begins()
# try to read debug level value at the early stage - no problem if fail here
try:
if cmd == '' or cmd == 'start' or cmd == 'go' or cmd == 'show' or cmd == 'open':
lg.set_debug_level(
int(
bpio.ReadTextFile(
os.path.abspath(
os.path.join(appdata, 'config', 'logs',
'debug-level')))))
except:
pass
if opts.no_logs:
lg.disable_logs()
if opts.debug or str(opts.debug) == '0':
lg.set_debug_level(int(opts.debug))
#---logpath---
logpath = None
if opts.output:
logpath = opts.output
else:
try:
os.makedirs(os.path.join(appdata, 'logs'), exist_ok=True)
except:
pass
logpath = os.path.join(appdata, 'logs', 'stdout.log')
need_redirecting = False
if bpio.Windows() and not bpio.isConsoled():
need_redirecting = True
if logpath:
if not appList:
if cmd not in [
'detach',
'daemon',
'stop',
'kill',
'shutdown',
'restart',
'reboot',
'reconnect',
'show',
'open',
]:
lg.open_log_file(logpath)
if bpio.Windows() and bpio.isFrozen():
need_redirecting = True
if bpio.Android():
need_redirecting = True
if opts.quite and not opts.verbose:
lg.disable_output()
else:
if need_redirecting:
lg.stdout_start_redirecting()
lg.stderr_start_redirecting()
#---start---
if cmd == '' or cmd == 'start' or cmd == 'go':
if appList:
print_text('BitDust already started, found another process: %s\n' %
str(appList),
nl='')
bpio.shutdown()
return 0
UI = ''
try:
ret = run(UI, opts, args, overDict, executable_path, start_reactor)
except:
lg.exc()
ret = 1
bpio.shutdown()
if opts.coverage:
cov.stop()
cov.save()
if opts.coverage_report:
cov.report(file=open(opts.coverage_report, 'w'))
return ret
#---daemon---
elif cmd == 'detach' or cmd == 'daemon':
appList = bpio.find_main_process(
pid_file_path=os.path.join(appdata, 'metadata', 'processid'))
if len(appList) > 0:
print_text('main BitDust process already started: %s\n' %
str(appList),
nl='')
bpio.shutdown()
if opts.coverage:
cov.stop()
cov.save()
if opts.coverage_report:
cov.report(file=open(opts.coverage_report, 'w'))
return 0
from lib import misc
print_text('new BitDust process will be started in daemon mode\n',
nl='')
result = misc.DoRestart(
detach=True,
# std_out=os.path.join(appdata, 'logs', 'stdout.log'),
# std_err=os.path.join(appdata, 'logs', 'stderr.log'),
)
if result is not None:
try:
result = int(result)
except:
try:
result = result.pid
except:
pass
bpio.shutdown()
if opts.coverage:
cov.stop()
cov.save()
if opts.coverage_report:
cov.report(file=open(opts.coverage_report, 'w'))
return 0
#---restart---
elif cmd == 'restart' or cmd == 'reboot':
appList = bpio.find_main_process(
pid_file_path=os.path.join(appdata, 'metadata', 'processid'))
ui = False
if len(appList) > 0:
print_text('found main BitDust process: %r ... ' % appList, nl='')
def done(x):
print_text('finished successfully\n', nl='')
from twisted.internet import reactor # @UnresolvedImport
if reactor.running and not reactor._stopped: # @UndefinedVariable
reactor.stop() # @UndefinedVariable
def failed(x):
if isinstance(x, Failure):
print_text('finished with: %s\n' % x.getErrorMessage(),
nl='')
else:
print_text('finished successfully\n', nl='')
ok = str(x).count('Connection was closed cleanly') > 0
from twisted.internet import reactor # @UnresolvedImport
if ok and reactor.running and not reactor._stopped: # @UndefinedVariable
# print_text('DONE\n', '')
reactor.stop() # @UndefinedVariable
return
print_text('forcing previous process shutdown\n', nl='')
try:
kill()
except:
lg.exc()
from lib import misc
reactor.addSystemEventTrigger( # @UndefinedVariable
'after',
'shutdown',
misc.DoRestart,
param='show' if ui else '',
detach=True,
# std_out=os.path.join(appdata, 'logs', 'stdout.log'),
# std_err=os.path.join(appdata, 'logs', 'stderr.log'),
)
reactor.stop() # @UndefinedVariable
try:
from twisted.internet import reactor # @UnresolvedImport
# from interface.command_line import run_url_command
# d = run_url_command('?action=restart', False)
# from interface import cmd_line
# d = cmd_line.call_xmlrpc_method('restart', ui)
from interface import cmd_line_json
d = cmd_line_json.call_websocket_method('process_restart',
websocket_timeout=5)
d.addCallback(done)
d.addErrback(failed)
reactor.run() # @UndefinedVariable
bpio.shutdown()
if opts.coverage:
cov.stop()
cov.save()
if opts.coverage_report:
cov.report(file=open(opts.coverage_report, 'w'))
return 0
except:
lg.exc()
bpio.shutdown()
if opts.coverage:
cov.stop()
cov.save()
if opts.coverage_report:
cov.report(file=open(opts.coverage_report, 'w'))
return 1
else:
ui = ''
if cmd == 'restart':
ui = 'show'
try:
ret = run(ui, opts, args, overDict, executable_path)
except:
lg.exc()
ret = 1
bpio.shutdown()
if opts.coverage:
cov.stop()
cov.save()
if opts.coverage_report:
cov.report(file=open(opts.coverage_report, 'w'))
return ret
#---show---
elif cmd == 'show' or cmd == 'open':
if not bpio.isGUIpossible():
print_text('BitDust GUI is turned OFF\n', nl='')
bpio.shutdown()
return 0
if bpio.Linux() and not bpio.X11_is_running():
print_text('this operating system not supporting X11 interface\n',
nl='')
bpio.shutdown()
return 0
appList = bpio.find_main_process(
pid_file_path=os.path.join(appdata, 'metadata', 'processid'))
if len(appList) == 0:
try:
ret = run('show', opts, args, overDict, executable_path)
except:
lg.exc()
ret = 1
bpio.shutdown()
return ret
# print_text('found main BitDust process: %s, start the GUI\n' % str(appList))
# ret = show()
bpio.shutdown()
return ret
#---stop---
elif cmd == 'stop' or cmd == 'kill' or cmd == 'shutdown':
if cmd == 'kill':
ret = kill()
bpio.shutdown()
if opts.coverage:
cov.stop()
cov.save()
if opts.coverage_report:
cov.report(file=open(opts.coverage_report, 'w'))
return ret
appList = bpio.find_main_process(pid_file_path=os.path.join(
appdata, 'metadata', 'processid'), )
if len(appList) > 0:
if cmd == 'kill':
print_text(
'found main BitDust process: %s, about to kill running process ... '
% appList,
nl='')
ret = kill()
bpio.shutdown()
if opts.coverage:
cov.stop()
cov.save()
if opts.coverage_report:
cov.report(file=open(opts.coverage_report, 'w'))
return ret
try:
from twisted.internet import reactor # @UnresolvedImport
from twisted.python.failure import Failure
def _stopped(x):
if _Debug:
if isinstance(x, Failure):
print_text('finished with: %s\n' %
x.getErrorMessage(),
nl='')
else:
print_text('finished with: %s\n' % x, nl='')
else:
print_text('finished successfully\n', nl='')
reactor.stop() # @UndefinedVariable
bpio.shutdown()
print_text('found main BitDust process: %s ... ' % appList,
nl='')
from interface import cmd_line_json
cmd_line_json.call_websocket_method(
'process_stop', websocket_timeout=2).addBoth(_stopped)
reactor.run() # @UndefinedVariable
if opts.coverage:
cov.stop()
cov.save()
if opts.coverage_report:
cov.report(file=open(opts.coverage_report, 'w'))
return 0
except:
lg.exc()
ret = kill()
bpio.shutdown()
if opts.coverage:
cov.stop()
cov.save()
if opts.coverage_report:
cov.report(file=open(opts.coverage_report, 'w'))
return ret
else:
appListAllChilds = bpio.find_main_process(
check_processid_file=False,
extra_lookups=[],
)
if len(appListAllChilds) > 0:
print_text(
'BitDust child processes found: %s, performing "kill process" action ...\n'
% appListAllChilds,
nl='')
ret = kill()
if opts.coverage:
cov.stop()
cov.save()
if opts.coverage_report:
cov.report(file=open(opts.coverage_report, 'w'))
return ret
print_text('BitDust is not running at the moment\n', nl='')
bpio.shutdown()
if opts.coverage:
cov.stop()
cov.save()
if opts.coverage_report:
cov.report(file=open(opts.coverage_report, 'w'))
return 0
#---command_line---
from interface import cmd_line_json as cmdln
ret = cmdln.run(opts, args, pars, overDict, executable_path)
if ret == 2:
print_text(usage_text())
bpio.shutdown()
#---coverage report---
if opts.coverage:
cov.stop()
cov.save()
if opts.coverage_report:
cov.report(file=open(opts.coverage_report, 'w'))
return ret