def doInitInterfaces(self, *args, **kwargs):
if bpio.Android():
lg.close_intercepted_log_file()
lg.open_intercepted_log_file(
os.path.join(settings.LogsDir(), 'android.log'))
if _Debug:
lg.out(_DebugLevel, 'initializer.doInitInterfaces')
if settings.enableFTPServer():
try:
from interface import ftp_server
ftp_server.init()
except:
lg.exc()
if settings.enableJsonRPCServer():
try:
from interface import api_jsonrpc_server
api_jsonrpc_server.init()
except:
lg.exc()
if settings.enableRESTHTTPServer():
try:
from interface import api_rest_http_server
api_rest_http_server.init(
port=settings.getRESTHTTPServerPort())
except:
lg.exc()
if settings.enableWebSocketServer():
try:
from interface import api_web_socket
api_web_socket.init(port=settings.getWebSocketServerPort())
except:
lg.exc()
reactor.callLater(0, self.automat,
'init-interfaces-done') # @UndefinedVariable
def doStartProcess(self, *args, **kwargs):
"""
Action method.
"""
ncpus = bpio.detect_number_of_cpu_cores()
if ncpus > 1:
# do not use all CPU cors at once
# need to keep at least one for all other operations
# even decided to use only half of CPUs at the moment
# TODO: make an option in the software settings
ncpus = int(ncpus / 2.0)
if bpio.Android() or not config.conf().getBool(
'services/rebuilding/child-processes-enabled'):
self.processor = ThreadedRaidProcessor()
else:
os.environ['PYTHONUNBUFFERED'] = '1'
from parallelp import pp
self.processor = pp.Server(
secret='bitdust',
ncpus=ncpus,
loglevel=lg.get_loging_level(_DebugLevel),
logfile=settings.ParallelPLogFilename(),
)
# else:
# from raid import worker
# self.processor = worker.Manager(ncpus=ncpus)
self.automat('process-started')
def doInitLocal(self, *args, **kwargs):
"""
"""
self.flagGUI = args[0].strip() == 'show'
if _Debug:
lg.out(_DebugLevel, 'initializer.doInitLocal flagGUI=%s' % self.flagGUI)
self._init_local()
if bpio.Android():
self.automat('init-local-done')
else:
reactor.callWhenRunning(self.automat, 'init-local-done') # @UndefinedVariable
def doInitServices(self, *args, **kwargs):
"""
Action method.
"""
if bpio.Android():
lg.close_intercepted_log_file()
lg.open_intercepted_log_file('/storage/emulated/0/Android/data/org.bitdust_io.bitdust1/files/Documents/.bitdust/logs/android.log', mode='a')
if _Debug:
lg.dbg(_DebugLevel, 'log file "android.log" re-opened')
if _Debug:
lg.out(_DebugLevel, 'initializer.doInitServices')
driver.init()
d = driver.start()
d.addBoth(lambda x: self.automat('init-services-done'))
def _do_start_archive_backup(self):
local_path = self.local_data_callback(self.queue_id,
self.latest_sequence_id)
supplier_path_id = os.path.join(self.archive_folder_path,
strng.to_text(self.latest_sequence_id))
dataID = misc.NewBackupID()
backup_id = packetid.MakeBackupID(
customer=self.queue_owner_id,
path_id=supplier_path_id,
key_alias=self.queue_alias,
version=dataID,
)
backup_fs.MakeLocalDir(settings.getLocalBackupsDir(), backup_id)
if bpio.Android():
compress_mode = 'none'
else:
compress_mode = 'bz2'
arcname = os.path.basename(local_path)
backupPipe = backup_tar.backuptarfile_thread(local_path,
arcname=arcname,
compress=compress_mode)
self.backup_job = backup.backup(
backupID=backup_id,
pipe=backupPipe,
blockResultCallback=self._on_archive_backup_block_result,
finishCallback=self._on_archive_backup_done,
blockSize=1024 * 1024 * 10,
sourcePath=local_path,
keyID=self.group_key_id,
ecc_map=eccmap.eccmap(self.ecc_map),
creatorIDURL=self.queue_owner_idurl,
)
self.backup_job.automat('start')
if _Debug:
lg.args(_DebugLevel,
job=self.backup_job,
backup_id=backup_id,
local_path=local_path,
group_key_id=self.group_key_id)
def run(self):
"""
Runs a new ``Job`` from that ``Task``.
"""
iter_and_path = backup_fs.WalkByID(self.remotePath,
iterID=backup_fs.fsID(
self.customerIDURL))
if iter_and_path is None:
if _Debug:
lg.out(
_DebugLevel,
'backup_control.Task.run ERROR %s not found in the index' %
self.remotePath)
err = 'remote path "%s" not found in the catalog' % self.remotePath
OnTaskFailed(self.pathID, err)
return err
itemInfo, sourcePath = iter_and_path
if isinstance(itemInfo, dict):
try:
itemInfo = itemInfo[backup_fs.INFO_KEY]
except:
lg.exc()
err = 'catalog item related to "%s" is broken' % self.remotePath
OnTaskFailed(self.pathID, err)
return err
if not self.localPath:
self.localPath = sourcePath
lg.out(
'backup_control.Task.run local path was populated from catalog: %s'
% self.localPath)
if self.localPath != sourcePath:
if _Debug:
lg.out(
_DebugLevel,
' local path different in the catalog: %s != %s' %
(self.localPath, sourcePath))
if not bpio.pathExist(self.localPath):
lg.warn('path not exist: %s' % self.localPath)
err = 'local path "%s" not exist' % self.localPath
OnTaskFailed(self.pathID, err)
return err
dataID = misc.NewBackupID()
if itemInfo.has_version(dataID):
# ups - we already have same version
# let's add 1,2,3... to the end to make absolutely unique version ID
i = 1
while itemInfo.has_version(dataID + str(i)):
i += 1
dataID += str(i)
self.backupID = packetid.MakeBackupID(
customer=self.fullCustomerID,
path_id=self.remotePath,
version=dataID,
)
if self.backupID in jobs():
lg.warn('backup job %s already started' % self.backupID)
return 'backup job %s already started' % self.backupID
try:
backup_fs.MakeLocalDir(settings.getLocalBackupsDir(),
self.backupID)
except:
lg.exc()
if _Debug:
lg.out(
_DebugLevel,
'backup_control.Task.run ERROR creating destination folder for %s'
% self.pathID)
err = 'failed creating destination folder for "%s"' % self.backupID
return OnTaskFailed(self.backupID, err)
if bpio.Android():
compress_mode = 'none'
else:
compress_mode = 'bz2' # 'none' # 'gz'
arcname = os.path.basename(sourcePath)
from storage import backup_tar
if bpio.pathIsDir(self.localPath):
backupPipe = backup_tar.backuptardir_thread(self.localPath,
arcname=arcname,
compress=compress_mode)
else:
backupPipe = backup_tar.backuptarfile_thread(
self.localPath, arcname=arcname, compress=compress_mode)
job = backup.backup(
self.backupID,
backupPipe,
finishCallback=OnJobDone,
blockResultCallback=OnBackupBlockReport,
notifyNewDataCallback=OnNewDataPrepared,
blockSize=settings.getBackupBlockSize(),
sourcePath=self.localPath,
keyID=self.keyID or itemInfo.key_id,
)
jobs()[self.backupID] = job
itemInfo.add_version(dataID)
if itemInfo.type == backup_fs.DIR:
dirsize.ask(self.localPath, OnFoundFolderSize,
(self.pathID, dataID))
else:
sz = os.path.getsize(self.localPath)
jobs()[self.backupID].totalSize = sz
itemInfo.set_size(sz)
backup_fs.Calculate()
Save()
jobs()[self.backupID].automat('start')
reactor.callLater(0, FireTaskStartedCallbacks, self.pathID,
dataID) # @UndefinedVariable
if _Debug:
lg.out(
_DebugLevel,
'backup_control.Task-%d.run [%s/%s], size=%d, %s' %
(self.number, self.pathID, dataID, itemInfo.size,
self.localPath))
return None
def init(UI='', options=None, args=None, overDict=None, executablePath=None):
"""
In the method ``main()`` program firstly checks the command line arguments
and then calls this method to start the whole process.
This initialize some low level modules and finally create an
instance of ``initializer()`` state machine and send it an event
"run".
"""
global AppDataDir
from logs import lg
if _Debug:
lg.out(_DebugLevel, 'bpmain.init UI="%s"' % UI)
from system import bpio
if _Debug:
lg.out(_DebugLevel, 'bpmain.init ostype=%r' % bpio.ostype())
#---settings---
from main import settings
if overDict:
settings.override_dict(overDict)
settings.init(AppDataDir)
if not options or options.debug is None:
lg.set_debug_level(settings.getDebugLevel())
from main import config
config.conf().addConfigNotifier(
'logs/debug-level', lambda p, value, o, r: lg.set_debug_level(value))
#---USE_TRAY_ICON---
if os.path.isfile(settings.LocalIdentityFilename()) and os.path.isfile(
settings.KeyFileName()):
try:
from system.tray_icon import USE_TRAY_ICON
if bpio.Mac() or not bpio.isGUIpossible():
if _Debug:
lg.out(_DebugLevel, ' GUI is not possible')
USE_TRAY_ICON = False
if USE_TRAY_ICON:
from twisted.internet import wxreactor
wxreactor.install()
if _Debug:
lg.out(_DebugLevel, ' wxreactor installed')
except:
USE_TRAY_ICON = False
lg.exc()
else:
if _Debug:
lg.out(_DebugLevel, ' local identity or key file is not ready')
USE_TRAY_ICON = False
if _Debug:
lg.out(_DebugLevel, ' USE_TRAY_ICON=' + str(USE_TRAY_ICON))
if USE_TRAY_ICON:
from system import tray_icon
icons_path = bpio.portablePath(
os.path.join(bpio.getExecutableDir(), 'icons'))
if _Debug:
lg.out(_DebugLevel,
'bpmain.init call tray_icon.init(%s)' % icons_path)
tray_icon.init(icons_path)
def _tray_control_func(cmd):
if cmd == 'exit':
from . import shutdowner
shutdowner.A('stop', 'exit')
tray_icon.SetControlFunc(_tray_control_func)
#---OS Windows init---
if bpio.Windows():
try:
from win32event import CreateMutex # @UnresolvedImport
mutex = CreateMutex(None, False, "BitDust")
if _Debug:
lg.out(_DebugLevel,
'bpmain.init created a Mutex: %s' % str(mutex))
except:
lg.exc()
#---twisted reactor---
if _Debug:
lg.out(_DebugLevel,
'bpmain.init want to import twisted.internet.reactor')
try:
from twisted.internet import reactor # @UnresolvedImport
except:
lg.exc()
sys.exit('Error initializing reactor in bpmain.py\n')
# #---logfile----
# if (lg.logs_enabled() and lg.log_file()) and not bpio.Android():
# lg.out(2, 'bpmain.run want to switch log files')
# if bpio.Windows() and bpio.isFrozen():
# lg.stdout_stop_redirecting()
# lg.close_log_file()
# lg.open_log_file(settings.MainLogFilename())
# # lg.open_log_file(settings.MainLogFilename() + '-' + time.strftime('%y%m%d%H%M%S') + '.log')
# if bpio.Windows() and bpio.isFrozen():
# lg.stdout_start_redirecting()
#---memdebug---
if config.conf().getBool('logs/memdebug-enabled'):
try:
from logs import memdebug
memdebug_port = int(config.conf().getData('logs/memdebug-port'))
memdebug.start(memdebug_port)
reactor.addSystemEventTrigger('before', 'shutdown',
memdebug.stop) # @UndefinedVariable
if _Debug:
lg.out(
_DebugLevel,
'bpmain.init memdebug web server started on port %d' %
memdebug_port)
except:
lg.exc()
#---process ID---
try:
pid = os.getpid()
pid_file_path = os.path.join(settings.MetaDataDir(), 'processid')
bpio.WriteTextFile(pid_file_path, str(pid))
if _Debug:
lg.out(
_DebugLevel,
'bpmain.init wrote process id [%s] in the file %s' %
(str(pid), pid_file_path))
except:
lg.exc()
# #---reactor.callLater patch---
if _Debug:
patchReactorCallLater(reactor)
monitorDelayedCalls(reactor)
if _Debug:
lg.out(_DebugLevel, " python executable is: %s" % sys.executable)
lg.out(_DebugLevel, " python version is:\n%s" % sys.version)
lg.out(
_DebugLevel, " python sys.path is:\n %s" %
('\n '.join(sys.path)))
lg.out(_DebugLevel, "bpmain.init UI=[%s]" % UI)
if lg.is_debug(12):
lg.out(_DebugLevel, '\n' + bpio.osinfofull())
if _Debug:
lg.out(_DebugLevel, 'bpmain.init going to import automats')
#---START!---
from automats import automat
automat.LifeBegins(lg.when_life_begins())
automat.SetGlobalLogEvents(
config.conf().getBool('logs/automat-events-enabled'))
automat.SetGlobalLogTransitions(
config.conf().getBool('logs/automat-transitions-enabled'))
automat.OpenLogFile(settings.AutomatsLog())
from main import events
events.init()
from main import initializer
IA = initializer.A()
if _Debug:
lg.out(_DebugLevel,
'bpmain.init is sending event "run" to initializer()')
if bpio.Android():
IA.automat('run', UI)
else:
reactor.callWhenRunning(IA.automat, 'run', UI) # @UndefinedVariable
return IA
def main(executable_path=None, start_reactor=True):
"""
THE ENTRY POINT
"""
global AppDataDir
pars = parser()
(opts, args) = pars.parse_args()
if opts.coverage:
import coverage # @UnresolvedImport
cov = coverage.Coverage(config_file=opts.coverage_config)
cov.start()
overDict = override_options(opts, args)
cmd = ''
if len(args) > 0:
cmd = args[0].lower()
try:
from system import deploy
except:
dirpath = os.path.dirname(os.path.abspath(sys.argv[0]))
sys.path.insert(0, os.path.abspath(os.path.join(dirpath, '..')))
from distutils.sysconfig import get_python_lib
sys.path.append(os.path.join(get_python_lib(), 'bitdust'))
try:
from system import deploy
except:
print_text('ERROR! can not import working code. Python Path:')
print_text('\n'.join(sys.path))
return 1
#---install---
if cmd in [
'deploy',
'install',
'venv',
'virtualenv',
]:
from system import deploy
return deploy.run(args)
if opts.appdir:
appdata = opts.appdir
AppDataDir = appdata
else:
curdir = os.getcwd()
appdatafile = os.path.join(curdir, 'appdata')
defaultappdata = deploy.default_base_dir_portable()
appdata = defaultappdata
if os.path.isfile(appdatafile):
try:
appdata = os.path.abspath(
open(appdatafile, 'rb').read().strip())
except:
appdata = defaultappdata
if not os.path.isdir(appdata):
appdata = defaultappdata
AppDataDir = appdata
#---BitDust Home
deploy.init_base_dir(base_dir=AppDataDir)
from logs import lg
#---init IO module
from system import bpio
bpio.init()
appList = bpio.find_main_process(
pid_file_path=os.path.join(appdata, 'metadata', 'processid'))
if bpio.Android():
lg.close_intercepted_log_file()
lg.open_intercepted_log_file(
'/storage/emulated/0/Android/data/org.bitdust_io.bitdust1/files/Documents/.bitdust/logs/android.log'
)
# sys.excepthook = lg.exception_hook
#---init logging
from twisted.internet.defer import setDebugging
if _Debug:
if bpio.isFrozen():
setDebugging(False)
else:
setDebugging(True)
else:
setDebugging(False)
from twisted.logger import globalLogPublisher, LogLevel
tw_log_observer = TwistedUnhandledErrorsObserver(level=LogLevel.critical)
globalLogPublisher.addObserver(tw_log_observer)
#---life begins!
# ask logger to count time for each log line from that moment, not absolute time
lg.life_begins()
# try to read debug level value at the early stage - no problem if fail here
try:
if cmd == '' or cmd == 'start' or cmd == 'go' or cmd == 'show' or cmd == 'open':
lg.set_debug_level(
int(
bpio.ReadTextFile(
os.path.abspath(
os.path.join(appdata, 'config', 'logs',
'debug-level')))))
except:
pass
if opts.no_logs:
lg.disable_logs()
if opts.debug or str(opts.debug) == '0':
lg.set_debug_level(int(opts.debug))
#---logpath---
logpath = None
if opts.output:
logpath = opts.output
else:
try:
os.makedirs(os.path.join(appdata, 'logs'), exist_ok=True)
except:
pass
logpath = os.path.join(appdata, 'logs', 'stdout.log')
need_redirecting = False
if bpio.Windows() and not bpio.isConsoled():
need_redirecting = True
if logpath:
if not appList:
if cmd not in [
'detach',
'daemon',
'stop',
'kill',
'shutdown',
'restart',
'reboot',
'reconnect',
'show',
'open',
]:
lg.open_log_file(logpath)
if bpio.Windows() and bpio.isFrozen():
need_redirecting = True
if bpio.Android():
need_redirecting = True
if opts.quite and not opts.verbose:
lg.disable_output()
else:
if need_redirecting:
lg.stdout_start_redirecting()
lg.stderr_start_redirecting()
#---start---
if cmd == '' or cmd == 'start' or cmd == 'go':
if appList:
print_text('BitDust already started, found another process: %s\n' %
str(appList),
nl='')
bpio.shutdown()
return 0
UI = ''
try:
ret = run(UI, opts, args, overDict, executable_path, start_reactor)
except:
lg.exc()
ret = 1
bpio.shutdown()
if opts.coverage:
cov.stop()
cov.save()
if opts.coverage_report:
cov.report(file=open(opts.coverage_report, 'w'))
return ret
#---daemon---
elif cmd == 'detach' or cmd == 'daemon':
appList = bpio.find_main_process(
pid_file_path=os.path.join(appdata, 'metadata', 'processid'))
if len(appList) > 0:
print_text('main BitDust process already started: %s\n' %
str(appList),
nl='')
bpio.shutdown()
if opts.coverage:
cov.stop()
cov.save()
if opts.coverage_report:
cov.report(file=open(opts.coverage_report, 'w'))
return 0
from lib import misc
print_text('new BitDust process will be started in daemon mode\n',
nl='')
result = misc.DoRestart(
detach=True,
# std_out=os.path.join(appdata, 'logs', 'stdout.log'),
# std_err=os.path.join(appdata, 'logs', 'stderr.log'),
)
if result is not None:
try:
result = int(result)
except:
try:
result = result.pid
except:
pass
bpio.shutdown()
if opts.coverage:
cov.stop()
cov.save()
if opts.coverage_report:
cov.report(file=open(opts.coverage_report, 'w'))
return 0
#---restart---
elif cmd == 'restart' or cmd == 'reboot':
appList = bpio.find_main_process(
pid_file_path=os.path.join(appdata, 'metadata', 'processid'))
ui = False
if len(appList) > 0:
print_text('found main BitDust process: %r ... ' % appList, nl='')
def done(x):
print_text('finished successfully\n', nl='')
from twisted.internet import reactor # @UnresolvedImport
if reactor.running and not reactor._stopped: # @UndefinedVariable
reactor.stop() # @UndefinedVariable
def failed(x):
if isinstance(x, Failure):
print_text('finished with: %s\n' % x.getErrorMessage(),
nl='')
else:
print_text('finished successfully\n', nl='')
ok = str(x).count('Connection was closed cleanly') > 0
from twisted.internet import reactor # @UnresolvedImport
if ok and reactor.running and not reactor._stopped: # @UndefinedVariable
# print_text('DONE\n', '')
reactor.stop() # @UndefinedVariable
return
print_text('forcing previous process shutdown\n', nl='')
try:
kill()
except:
lg.exc()
from lib import misc
reactor.addSystemEventTrigger( # @UndefinedVariable
'after',
'shutdown',
misc.DoRestart,
param='show' if ui else '',
detach=True,
# std_out=os.path.join(appdata, 'logs', 'stdout.log'),
# std_err=os.path.join(appdata, 'logs', 'stderr.log'),
)
reactor.stop() # @UndefinedVariable
try:
from twisted.internet import reactor # @UnresolvedImport
# from interface.command_line import run_url_command
# d = run_url_command('?action=restart', False)
# from interface import cmd_line
# d = cmd_line.call_xmlrpc_method('restart', ui)
from interface import cmd_line_json
d = cmd_line_json.call_websocket_method('process_restart',
websocket_timeout=5)
d.addCallback(done)
d.addErrback(failed)
reactor.run() # @UndefinedVariable
bpio.shutdown()
if opts.coverage:
cov.stop()
cov.save()
if opts.coverage_report:
cov.report(file=open(opts.coverage_report, 'w'))
return 0
except:
lg.exc()
bpio.shutdown()
if opts.coverage:
cov.stop()
cov.save()
if opts.coverage_report:
cov.report(file=open(opts.coverage_report, 'w'))
return 1
else:
ui = ''
if cmd == 'restart':
ui = 'show'
try:
ret = run(ui, opts, args, overDict, executable_path)
except:
lg.exc()
ret = 1
bpio.shutdown()
if opts.coverage:
cov.stop()
cov.save()
if opts.coverage_report:
cov.report(file=open(opts.coverage_report, 'w'))
return ret
#---show---
elif cmd == 'show' or cmd == 'open':
if not bpio.isGUIpossible():
print_text('BitDust GUI is turned OFF\n', nl='')
bpio.shutdown()
return 0
if bpio.Linux() and not bpio.X11_is_running():
print_text('this operating system not supporting X11 interface\n',
nl='')
bpio.shutdown()
return 0
appList = bpio.find_main_process(
pid_file_path=os.path.join(appdata, 'metadata', 'processid'))
if len(appList) == 0:
try:
ret = run('show', opts, args, overDict, executable_path)
except:
lg.exc()
ret = 1
bpio.shutdown()
return ret
# print_text('found main BitDust process: %s, start the GUI\n' % str(appList))
# ret = show()
bpio.shutdown()
return ret
#---stop---
elif cmd == 'stop' or cmd == 'kill' or cmd == 'shutdown':
if cmd == 'kill':
ret = kill()
bpio.shutdown()
if opts.coverage:
cov.stop()
cov.save()
if opts.coverage_report:
cov.report(file=open(opts.coverage_report, 'w'))
return ret
appList = bpio.find_main_process(pid_file_path=os.path.join(
appdata, 'metadata', 'processid'), )
if len(appList) > 0:
if cmd == 'kill':
print_text(
'found main BitDust process: %s, about to kill running process ... '
% appList,
nl='')
ret = kill()
bpio.shutdown()
if opts.coverage:
cov.stop()
cov.save()
if opts.coverage_report:
cov.report(file=open(opts.coverage_report, 'w'))
return ret
try:
from twisted.internet import reactor # @UnresolvedImport
from twisted.python.failure import Failure
def _stopped(x):
if _Debug:
if isinstance(x, Failure):
print_text('finished with: %s\n' %
x.getErrorMessage(),
nl='')
else:
print_text('finished with: %s\n' % x, nl='')
else:
print_text('finished successfully\n', nl='')
reactor.stop() # @UndefinedVariable
bpio.shutdown()
print_text('found main BitDust process: %s ... ' % appList,
nl='')
from interface import cmd_line_json
cmd_line_json.call_websocket_method(
'process_stop', websocket_timeout=2).addBoth(_stopped)
reactor.run() # @UndefinedVariable
if opts.coverage:
cov.stop()
cov.save()
if opts.coverage_report:
cov.report(file=open(opts.coverage_report, 'w'))
return 0
except:
lg.exc()
ret = kill()
bpio.shutdown()
if opts.coverage:
cov.stop()
cov.save()
if opts.coverage_report:
cov.report(file=open(opts.coverage_report, 'w'))
return ret
else:
appListAllChilds = bpio.find_main_process(
check_processid_file=False,
extra_lookups=[],
)
if len(appListAllChilds) > 0:
print_text(
'BitDust child processes found: %s, performing "kill process" action ...\n'
% appListAllChilds,
nl='')
ret = kill()
if opts.coverage:
cov.stop()
cov.save()
if opts.coverage_report:
cov.report(file=open(opts.coverage_report, 'w'))
return ret
print_text('BitDust is not running at the moment\n', nl='')
bpio.shutdown()
if opts.coverage:
cov.stop()
cov.save()
if opts.coverage_report:
cov.report(file=open(opts.coverage_report, 'w'))
return 0
#---command_line---
from interface import cmd_line_json as cmdln
ret = cmdln.run(opts, args, pars, overDict, executable_path)
if ret == 2:
print_text(usage_text())
bpio.shutdown()
#---coverage report---
if opts.coverage:
cov.stop()
cov.save()
if opts.coverage_report:
cov.report(file=open(opts.coverage_report, 'w'))
return ret