def download_file(url, file): if url not in download_progress: download_progress[url] = {} download_progress[url]["status"] = "downloading" else: if download_progress[url]["status"] == "downloading": launcher_log.error("url in downloading, %s", url) return False try: launcher_log.info("download %s to %s", url, file) opener = get_opener() req = opener.open(url) download_progress[url]["size"] = int( req.headers.get('content-length') or 0) CHUNK = 16 * 1024 downloaded = 0 with open(file, 'wb') as fp: while True: chunk = req.read(CHUNK) if not chunk: break fp.write(chunk) downloaded += len(chunk) download_progress[url]["downloaded"] = downloaded download_progress[url]["status"] = "finished" return True except Exception as e: download_progress[url]["status"] = "fail" launcher_log.exception("download %s to %s fail:%r", url, file, e) return False
def download_file(url, file): if url not in download_progress: download_progress[url] = {} download_progress[url]["status"] = "downloading" else: if download_progress[url]["status"] == "downloading": launcher_log.error("url in downloading, %s", url) return False try: launcher_log.info("download %s to %s", url, file) opener = get_opener() req = opener.open(url) download_progress[url]["size"] = int(req.headers.get('content-length') or 0) CHUNK = 16 * 1024 downloaded = 0 with open(file, 'wb') as fp: while True: chunk = req.read(CHUNK) if not chunk: break fp.write(chunk) downloaded += len(chunk) download_progress[url]["downloaded"] = downloaded download_progress[url]["status"] = "finished" return True except Exception as e: download_progress[url]["status"] = "fail" launcher_log.exception("download %s to %s fail:%r", url, file, e) return False
def start(module): if not os.path.isdir(os.path.join(root_path, module)): return try: if not module in config.config["modules"]: launcher_log.error("module not exist %s", module) raise if module in proc_handler: launcher_log.error("module %s is running", module) return "module is running" if module not in proc_handler: proc_handler[module] = {} if module == 'ossftp': masquerade_address = config.get( ["modules", "ossftp", "masquerade_address"], "") address = config.get(["modules", "ossftp", "address"], "127.0.0.1") port = config.get(["modules", "ossftp", "port"], 2048) passive_ports_start = config.get( ["modules", "ossftp", "passive_ports_start"], 51000) passive_ports_end = config.get( ["modules", "ossftp", "passive_ports_end"], 53000) is_internal = config.get(["modules", "ossftp", "internal"], None) log_level = config.get(["modules", "ossftp", "log_level"], "INFO") bucket_endpoints = config.get( ["modules", "ossftp", "bucket_endpoints"], "") script_path = os.path.join(root_path, 'ossftp', 'ftpserver.py') if not os.path.isfile(script_path): launcher_log.critical("start module script not exist:%s", script_path) return "fail" cmd = [ sys.executable, script_path, "--listen_address=%s" % address, "--port=%d" % port, "--passive_ports_start=%d" % passive_ports_start, "--passive_ports_end=%d" % passive_ports_end, "--loglevel=%s" % log_level, "--bucket_endpoints=%s" % bucket_endpoints ] print(cmd) proc_handler[module]["proc"] = subprocess.Popen( cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) else: raise ValueError("Wrong module: %s" % module) launcher_log.info("%s started", module) except Exception as e: launcher_log.exception("start module %s fail:%s", module, e) return "Except:%s" % e return "start success."
def general_gtk_callback(widget=None, data=None): args = data.split('|') if len(args) != 3: launcher_log.error("general_gtk_callback data:%s", data) return module = args[0] new_version = args[1] action = args[2] if action == "download": download_module(module, new_version) elif action == "install": install_module(module, new_version) elif action == "ignore": ignore_module(module, new_version)
def stop(module): try: if not module in proc_handler: launcher_log.error("module %s not running", module) return proc_handler[module]["proc"].terminate() # Sends SIGTERM proc_handler[module]["proc"].wait() #proc_handler[module]["proc"].stop() del proc_handler[module] launcher_log.info("module %s stopped", module) except Exception as e: launcher_log.exception("stop module %s fail:%s", module, e) return "Except:%s" % e return "stop success."
def install_module(module, new_version): import module_init import os, subprocess, sys current_path = os.path.dirname(os.path.abspath(__file__)) new_module_version_path = os.path.abspath( os.path.join(current_path, os.pardir, os.pardir, module, new_version)) #check path exist if not os.path.isdir(new_module_version_path): launcher_log.error("install module %s dir %s not exist", module, new_module_version_path) return #call setup.py setup_script = os.path.join(new_module_version_path, "setup.py") if not os.path.isfile(setup_script): launcher_log.warn("update %s fail. setup script %s not exist", module, setup_script) return config.set(["modules", module, "current_version"], str(new_version)) config.save() if module == "launcher": module_init.stop_all() import web_control web_control.stop() subprocess.Popen([sys.executable, setup_script], shell=False) os._exit(0) else: launcher_log.info("Setup %s version %s ...", module, new_version) try: module_init.stop(module) subprocess.call([sys.executable, setup_script], shell=False) launcher_log.info("Finished new version setup.") launcher_log.info("Restarting new version ...") module_init.start(module) except Exception as e: launcher_log.error("install module %s %s fail:%s", module, new_version, e)
def start(module): if not os.path.isdir(os.path.join(root_path, module)): return try: if module not in config.config["modules"]: launcher_log.error("module not exist %s", module) raise if module in proc_handler: launcher_log.error("module %s is running", module) return "module is running" if module not in proc_handler: proc_handler[module] = {} if os.path.isfile(os.path.join(root_path, module, "__init__.py")): if "imp" not in proc_handler[module]: proc_handler[module]["imp"] = __import__( module, globals(), locals(), ['local', 'start'], -1) _start = proc_handler[module]["imp"].start p = threading.Thread(target=_start.main) p.daemon = True p.start() proc_handler[module]["proc"] = p while not _start.client.ready: time.sleep(0.1) else: script_path = os.path.join(root_path, module, 'start.py') if not os.path.isfile(script_path): launcher_log.warn("start module script not exist:%s", script_path) return "fail" proc_handler[module]["proc"] = subprocess.Popen( [sys.executable, script_path], shell=False) launcher_log.info("%s started", module) except Exception as e: launcher_log.exception("start module %s fail:%s", module, e) return "Except:%s" % e return "start success."
def start(module): if not os.path.isdir(os.path.join(root_path, module)): return try: if not module in config.config["modules"]: launcher_log.error("module not exist %s", module) raise if module in proc_handler: launcher_log.error("module %s is running", module) return "module is running" if module not in proc_handler: proc_handler[module] = {} if os.path.isfile(os.path.join(root_path, module, "__init__.py")): if "imp" not in proc_handler[module]: proc_handler[module]["imp"] = __import__(module, globals(), locals(), ['local', 'start'], -1) _start = proc_handler[module]["imp"].start p = threading.Thread(target = _start.main) p.daemon = True p.start() proc_handler[module]["proc"] = p while not _start.client.ready: time.sleep(0.1) else: script_path = os.path.join(root_path, module, 'start.py') if not os.path.isfile(script_path): launcher_log.warn("start module script not exist:%s", script_path) return "fail" proc_handler[module]["proc"] = subprocess.Popen([sys.executable, script_path], shell=False) launcher_log.info("%s started", module) except Exception as e: launcher_log.exception("start module %s fail:%s", module, e) return "Except:%s" % e return "start success."
def start(module): if not os.path.isdir(os.path.join(root_path, module)): return try: if not module in config.config["modules"]: launcher_log.error("module not exist %s", module) raise if module in proc_handler: launcher_log.error("module %s is running", module) return "module is running" if module not in proc_handler: proc_handler[module] = {} if module == 'ossftp': masquerade_address = config.get(["modules", "ossftp", "masquerade_address"], "") address = config.get(["modules", "ossftp", "address"], "127.0.0.1") port = config.get(["modules", "ossftp", "port"], 2048) is_internal = config.get(["modules", "ossftp", "internal"], None) log_level = config.get(["modules", "ossftp", "log_level"], "INFO") bucket_endpoints = config.get(["modules", "ossftp", "bucket_endpoints"], "") script_path = os.path.join(root_path, 'ossftp', 'ftpserver.py') if not os.path.isfile(script_path): launcher_log.critical("start module script not exist:%s", script_path) return "fail" cmd = [sys.executable, script_path, "--listen_address=%s"%address, "--port=%d"%port, "--loglevel=%s"%log_level, "--bucket_endpoints=%s"%bucket_endpoints] print cmd proc_handler[module]["proc"] = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) #t = FTPd(masquerade_address, address, port, bucket_endpoints, is_internal, log_level) #t.start() #proc_handler[module]["proc"] = t else: raise ValueError("Wrong module: %s" % module) launcher_log.info("%s started", module) except Exception as e: launcher_log.exception("start module %s fail:%s", module, e) return "Except:%s" % e return "start success."
def confirm_module_ready(port): if port == 0: launcher_log.error("confirm_module_ready with port 0") time.sleep(1) return False for i in range(200): req = http_request("http://127.0.0.1:%d/is_ready" % port) if req == False: time.sleep(1) continue content = req.read(1024) req.close() #logging.debug("cert_import_ready return:%s", content) if content == "True": return True else: time.sleep(1) return False
def stop(module): try: if not module in proc_handler: launcher_log.error("module %s not running", module) return if os.path.isfile(os.path.join(root_path, module, "__init__.py")): _start = proc_handler[module]["imp"].start _start.client.terminate() launcher_log.debug("module %s stopping", module) while _start.client.ready: time.sleep(0.1) else: proc_handler[module]["proc"].terminate() # Sends SIGTERM proc_handler[module]["proc"].wait() del proc_handler[module] launcher_log.info("module %s stopped", module) except Exception as e: launcher_log.exception("stop module %s fail:%s", module, e) return "Except:%s" % e return "stop success."
def stop(module): try: if not module in proc_handler: launcher_log.error("module %s not running", module) return if os.path.isfile(os.path.join(root_path, module, "__init__.py")): _start = proc_handler[module]["imp"].start _start.client.config.keep_run = False launcher_log.debug("module %s stopping", module) while _start.client.ready: time.sleep(0.1) else: proc_handler[module]["proc"].terminate() # Sends SIGTERM proc_handler[module]["proc"].wait() del proc_handler[module] launcher_log.info("module %s stopped", module) except Exception as e: launcher_log.exception("stop module %s fail:%s", module, e) return "Except:%s" % e return "stop success."
def notify_general(self, msg="msg", title="Title", buttons={}, timeout=3600): launcher_log.error("Mac notify_general not implemented.")
def download_module(module, new_version): import os global update_content, update_dict current_path = os.path.dirname(os.path.abspath(__file__)) download_path = os.path.abspath( os.path.join(current_path, os.pardir, os.pardir, 'data', 'downloads')) if not os.path.isdir(download_path): os.mkdir(download_path) try: for source in update_dict["modules"][module]["versions"][new_version][ "sources"]: url = source["url"] filename = module + "-" + new_version + ".zip" file_path = os.path.join(download_path, filename) if os.path.isfile(file_path) and sha1_file( file_path) == update_dict["modules"][module]["versions"][ new_version]["sha1"]: pass elif not download_file(url, file_path): launcher_log.warn("download %s fail", url) continue sha1 = sha1_file(file_path) if update_dict["modules"][module]["versions"][new_version][ "sha1"] != sha1: launcher_log.warn("download %s sha1 wrong", url) continue module_path = os.path.abspath( os.path.join(current_path, os.pardir, os.pardir, module)) if not os.path.isdir(module_path): os.path.mkdir(module_path, "755") version_path = os.path.join(module_path, new_version) if os.path.isdir(version_path): launcher_log.error("module dir exist:%s, download exist.", version_path) return with zipfile.ZipFile(file_path, "r") as dz: dz.extractall(module_path) dz.close() import shutil unzip_path = os.path.abspath( os.path.join(module_path, module + "-" + new_version)) tag_path = os.path.abspath(os.path.join(module_path, new_version)) shutil.move(unzip_path, tag_path) msg = "Module %s new version %s downloaded, Install?" % ( module, new_version) if sys.platform == "linux" or sys.platform == "linux2": from gtk_tray import sys_tray data_install = "%s|%s|install" % (module, new_version) data_ignore = "%s|%s|ignore" % (module, new_version) buttons = { 1: { "data": data_install, "label": "Install", 'callback': general_gtk_callback }, 2: { "data": data_ignore, "label": "Ignore", 'callback': general_gtk_callback } } sys_tray.notify_general(msg=msg, title="Install", buttons=buttons) elif sys.platform == "win32": from win_tray import sys_tray if sys_tray.dialog_yes_no(msg, u"Install", None, None) == 1: install_module(module, new_version) else: ignore_module(module, new_version) elif sys.platform == "darwin": from mac_tray import sys_tray if sys_tray.dialog_yes_no(msg, u"Install", None, None) == 1: install_module(module, new_version) else: ignore_module(module, new_version) else: install_module(module, new_version) break except Exception as e: launcher_log.warn("get gae_proxy source fail, content:%s err:%s", update_content, e)
def req_config_handler(self): req = urlparse.urlparse(self.path).query reqs = urlparse.parse_qs(req, keep_blank_values=True) data = '' if reqs['cmd'] == ['get_config']: config.load() data = '{ "popup_webui": %d, "show_systray": %d, "auto_start": %d, "ossftp_address": "%s", "ossftp_port": %d, "ossftp_loglevel": "%s", "ossftp_bucketendpoints": "%s" }' %\ (config.get(["modules", "launcher", "popup_webui"], 1) , config.get(["modules", "launcher", "show_systray"], 1) , config.get(["modules", "launcher", "auto_start"], 0) , config.get(["modules", "ossftp", "address"], '127.0.0.1') , config.get(["modules", "ossftp", "port"], 2048) , config.get(["modules", "ossftp", "log_level"], 'INFO') , config.get(["modules", "ossftp", "bucket_endpoints"], '')) elif reqs['cmd'] == ['set_config']: success = True popup_webui = config.get(["modules", "launcher", "popup_webui"], 1) auto_start = config.get(["modules", "launcher", "auto_start"], 0) show_systray = config.get(["modules", "launcher", "show_systray"], 1) ossftp_address = config.get(["modules", "ossftp", "address"], "127.0.0.1") ossftp_port = config.get(["modules", "ossftp", "port"], 2048) ossftp_loglevel = config.get(["modules", "ossftp", "log_level"], 'INFO') ossftp_bucketendpoints = config.get(["modules", "ossftp", "bucket_endpoints"], '') data = '{"res":"fail"}' if success and 'popup_webui' in reqs : popup_webui = int(reqs['popup_webui'][0]) if popup_webui != 0 and popup_webui != 1: success = False data = '{"res":"fail, popup_webui:%s"}' % popup_webui if success and 'show_systray' in reqs : show_systray = int(reqs['show_systray'][0]) if show_systray != 0 and show_systray != 1: success = False data = '{"res":"fail, show_systray:%s"}' % show_systray if success and 'auto_start' in reqs : auto_start = int(reqs['auto_start'][0]) if auto_start != 0 and auto_start != 1: success = False data = '{"res":"fail, auto_start:%s"}' % auto_start if success and 'ossftp_address' in reqs: ossftp_address = reqs['ossftp_address'][0].strip() if not self.ip_check(ossftp_address): success = False data = '{"res":"fail, ilegal ossftp address: %s"}' % ossftp_address if success and 'ossftp_port' in reqs: ossftp_port = int(reqs['ossftp_port'][0]) if ossftp_port < 0: success = False data = '{"res":"fail, ilegal ossftp port: %d"}' % ossftp_port if success and 'ossftp_loglevel' in reqs: ossftp_loglevel = reqs['ossftp_loglevel'][0].strip().upper() if (ossftp_loglevel not in ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']): success = False data = '{"res":"fail, illegal ossftp log level: %s. Must be: DEBUG, INFO, WARNING, ERROR, CRITICAL"}' % ossftp_loglevel if success and 'ossftp_bucketendpoints' in reqs: ossftp_bucketendpoints = reqs['ossftp_bucketendpoints'][0].strip() if success: config.set(["modules", "launcher", "popup_webui"], popup_webui) config.set(["modules", "launcher", "show_systray"], show_systray) config.set(["modules", "launcher", "auto_start"], auto_start) config.set(["modules", "ossftp", "address"], ossftp_address) config.set(["modules", "ossftp", "port"], ossftp_port) config.set(["modules", "ossftp", "log_level"], ossftp_loglevel) config.set(["modules", "ossftp", "bucket_endpoints"], ossftp_bucketendpoints) config.save() if auto_start: autorun.enable() else: autorun.disable() data = '{"res":"success"}' launcher_log.info('Set config: %s', json.dumps(config.config, sort_keys=True, separators=(',',':'), indent=2)) else: launcher_log.error(data) self.send_response('text/html', data)
def download_module(module, new_version): import os global update_content, update_dict current_path = os.path.dirname(os.path.abspath(__file__)) download_path = os.path.abspath( os.path.join(current_path, os.pardir, os.pardir, 'data', 'downloads')) if not os.path.isdir(download_path): os.mkdir(download_path) try: for source in update_dict["modules"][module]["versions"][new_version]["sources"]: url = source["url"] filename = module + "-" + new_version + ".zip" file_path = os.path.join(download_path, filename) if os.path.isfile(file_path) and sha1_file(file_path) == update_dict["modules"][module]["versions"][new_version]["sha1"]: pass elif not download_file(url, file_path): launcher_log.warn("download %s fail", url) continue sha1 = sha1_file(file_path) if update_dict["modules"][module]["versions"][new_version]["sha1"] != sha1: launcher_log.warn("download %s sha1 wrong", url) continue module_path = os.path.abspath( os.path.join(current_path, os.pardir, os.pardir, module)) if not os.path.isdir(module_path): os.path.mkdir(module_path, "755") version_path = os.path.join(module_path, new_version) if os.path.isdir(version_path): launcher_log.error("module dir exist:%s, download exist.", version_path) return with zipfile.ZipFile(file_path, "r") as dz: dz.extractall(module_path) dz.close() import shutil unzip_path = os.path.abspath(os.path.join(module_path, module + "-" + new_version)) tag_path = os.path.abspath(os.path.join(module_path, new_version)) shutil.move(unzip_path, tag_path) msg = "Module %s new version %s downloaded, Install?" % (module, new_version) if sys.platform == "linux" or sys.platform == "linux2": from gtk_tray import sys_tray data_install = "%s|%s|install" % (module, new_version) data_ignore = "%s|%s|ignore" % (module, new_version) buttons = {1: {"data":data_install, "label":"Install", 'callback':general_gtk_callback}, 2: {"data":data_ignore, "label":"Ignore", 'callback':general_gtk_callback}} sys_tray.notify_general(msg=msg, title="Install", buttons=buttons) elif sys.platform == "win32": from win_tray import sys_tray if sys_tray.dialog_yes_no(msg, u"Install", None, None) == 1: install_module(module, new_version) else: ignore_module(module, new_version) elif sys.platform == "darwin": from mac_tray import sys_tray if sys_tray.dialog_yes_no(msg, u"Install", None, None) == 1: install_module(module, new_version) else: ignore_module(module, new_version) else: install_module(module, new_version) break except Exception as e: launcher_log.warn("get gae_proxy source fail, content:%s err:%s", update_content, e)
def req_config_handler(self): req = urlparse(self.path).query reqs = parse_qs(req, keep_blank_values=True) data = '' if reqs['cmd'] == ['get_config']: config.load() data = '{ "popup_webui": %d, "show_systray": %d, "auto_start": %d, "language": "%s", "ossftp_address": "%s", "ossftp_port": %d, "ossftp_loglevel": "%s", "ossftp_bucketendpoints": "%s", "passive_ports_start":%d, "passive_ports_end":%d}' %\ (config.get(["modules", "launcher", "popup_webui"], 1) , config.get(["modules", "launcher", "show_systray"], 1) , config.get(["modules", "launcher", "auto_start"], 0) , config.get(["modules", "launcher", "language"], "cn") , config.get(["modules", "ossftp", "address"], '127.0.0.1') , config.get(["modules", "ossftp", "port"], 2048) , config.get(["modules", "ossftp", "log_level"], 'INFO') , config.get(["modules", "ossftp", "bucket_endpoints"], '') , config.get(["modules", "ossftp", "passive_ports_start"], 51000) , config.get(["modules", "ossftp", "passive_ports_end"], 53000)) elif reqs['cmd'] == ['set_config']: success = True popup_webui = config.get(["modules", "launcher", "popup_webui"], 1) auto_start = config.get(["modules", "launcher", "auto_start"], 0) show_systray = config.get(["modules", "launcher", "show_systray"], 1) language = config.get(["modules", "launcher", "language"], "cn") ossftp_address = config.get(["modules", "ossftp", "address"], "127.0.0.1") ossftp_port = config.get(["modules", "ossftp", "port"], 2048) ossftp_loglevel = config.get(["modules", "ossftp", "log_level"], 'INFO') ossftp_bucketendpoints = config.get( ["modules", "ossftp", "bucket_endpoints"], '') passive_ports_start = config.get( ["modules", "ossftp", "passive_ports_start"], 51000) passive_ports_end = config.get( ["modules", "ossftp", "passive_ports_end"], 53000) data = '{"res":"fail"}' if success and 'language' in reqs: language = reqs['language'][0] if language != 'en' and language != 'cn': success = False data = '{"res":"fail, language:%s"}' % language if success and 'popup_webui' in reqs: popup_webui = int(reqs['popup_webui'][0]) if popup_webui != 0 and popup_webui != 1: success = False data = '{"res":"fail, popup_webui:%s"}' % popup_webui if success and 'show_systray' in reqs: show_systray = int(reqs['show_systray'][0]) if show_systray != 0 and show_systray != 1: success = False data = '{"res":"fail, show_systray:%s"}' % show_systray if success and 'auto_start' in reqs: auto_start = int(reqs['auto_start'][0]) if auto_start != 0 and auto_start != 1: success = False data = '{"res":"fail, auto_start:%s"}' % auto_start if success and 'ossftp_address' in reqs: ossftp_address = reqs['ossftp_address'][0].strip() if not self.ip_check(ossftp_address): success = False data = '{"res":"fail, ilegal ossftp address: %s"}' % ossftp_address if success and 'ossftp_port' in reqs: ossftp_port = int(reqs['ossftp_port'][0]) if ossftp_port < 0: success = False data = '{"res":"fail, ilegal ossftp port: %d"}' % ossftp_port if success and 'ossftp_loglevel' in reqs: ossftp_loglevel = reqs['ossftp_loglevel'][0].strip().upper() if (ossftp_loglevel not in [ 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL' ]): success = False data = '{"res":"fail, illegal ossftp log level: %s. Must be: DEBUG, INFO, WARNING, ERROR, CRITICAL"}' % ossftp_loglevel if success and 'ossftp_bucketendpoints' in reqs: ossftp_bucketendpoints = reqs['ossftp_bucketendpoints'][ 0].strip() if success and 'passive_ports_start' in reqs: passive_ports_start = int(reqs['passive_ports_start'][0]) if passive_ports_end < 0: success = False data = '{"res":"fail, illegal ossftp passive_ports_start: %d"}' % passive_ports_start if success and 'passive_ports_end' in reqs: passive_ports_end = int(reqs['passive_ports_end'][0]) if passive_ports_end < 0: success = False data = '{"res":"fail, illegal ossftp passive_ports_end: %d"}' % passive_ports_end if success: config.set(["modules", "launcher", "popup_webui"], popup_webui) config.set(["modules", "launcher", "show_systray"], show_systray) config.set(["modules", "launcher", "auto_start"], auto_start) config.set(["modules", "launcher", "language"], language) config.set(["modules", "ossftp", "address"], ossftp_address) config.set(["modules", "ossftp", "port"], ossftp_port) config.set(["modules", "ossftp", "log_level"], ossftp_loglevel) config.set(["modules", "ossftp", "bucket_endpoints"], ossftp_bucketendpoints) config.set(["modules", "ossftp", "passive_ports_start"], passive_ports_start) config.set(["modules", "ossftp", "passive_ports_end"], passive_ports_end) config.save() if auto_start: autorun.enable() else: autorun.disable() data = '{"res":"success"}' import copy tmp_config = copy.deepcopy(config.config.copy()) del tmp_config['modules']['accounts'] launcher_log.info( 'Set config: %s', json.dumps(tmp_config, sort_keys=True, separators=(',', ':'), indent=2)) else: launcher_log.error(data) self.send_response('text/html', data)