def download_file(url, file):
if url not in download_progress:
download_progress[url] = {}
download_progress[url]["status"] = "downloading"
else:
if download_progress[url]["status"] == "downloading":
launcher_log.error("url in downloading, %s", url)
return False
try:
launcher_log.info("download %s to %s", url, file)
opener = get_opener()
req = opener.open(url)
download_progress[url]["size"] = int(
req.headers.get('content-length') or 0)
CHUNK = 16 * 1024
downloaded = 0
with open(file, 'wb') as fp:
while True:
chunk = req.read(CHUNK)
if not chunk: break
fp.write(chunk)
downloaded += len(chunk)
download_progress[url]["downloaded"] = downloaded
download_progress[url]["status"] = "finished"
return True
except Exception as e:
download_progress[url]["status"] = "fail"
launcher_log.exception("download %s to %s fail:%r", url, file, e)
return False
def req_init_module_handler(self):
req = urlparse.urlparse(self.path).query
reqs = urlparse.parse_qs(req, keep_blank_values=True)
data = ''
try:
module = reqs['module'][0]
config.load()
if reqs['cmd'] == ['start']:
result = module_init.start(module)
data = '{ "module": "%s", "cmd": "start", "result": "%s" }' % (
module, result)
elif reqs['cmd'] == ['stop']:
result = module_init.stop(module)
data = '{ "module": "%s", "cmd": "stop", "result": "%s" }' % (
module, result)
elif reqs['cmd'] == ['restart']:
result_stop = module_init.stop(module)
result_start = module_init.start(module)
data = '{ "module": "%s", "cmd": "restart", "stop_result": "%s", "start_result": "%s" }' % (
module, result_stop, result_start)
except Exception as e:
launcher_log.exception("init_module except:%s", e)
self.send_response("text/html", data)
def download_file(url, file):
if url not in download_progress:
download_progress[url] = {}
download_progress[url]["status"] = "downloading"
else:
if download_progress[url]["status"] == "downloading":
launcher_log.error("url in downloading, %s", url)
return False
try:
launcher_log.info("download %s to %s", url, file)
opener = get_opener()
req = opener.open(url)
download_progress[url]["size"] = int(req.headers.get('content-length') or 0)
CHUNK = 16 * 1024
downloaded = 0
with open(file, 'wb') as fp:
while True:
chunk = req.read(CHUNK)
if not chunk: break
fp.write(chunk)
downloaded += len(chunk)
download_progress[url]["downloaded"] = downloaded
download_progress[url]["status"] = "finished"
return True
except Exception as e:
download_progress[url]["status"] = "fail"
launcher_log.exception("download %s to %s fail:%r", url, file, e)
return False
def check_update():
try:
update_rule = config.get(["update", "check_update"], "stable")
if update_rule == "dont-check":
return
check_push_update()
if update_rule != "stable" and update_rule != "test":
return
versions = update_from_github.get_github_versions()
current_version = update_from_github.current_version()
if update_rule == "test":
if LooseVersion(current_version) < LooseVersion(versions[0][1]):
launcher_log.info("update to test version %s", versions[0][1])
update_from_github.update_version(versions[0][1])
elif update_rule == "stable":
if LooseVersion(current_version) < LooseVersion(versions[1][1]):
launcher_log.info("update to stable version %s", versions[1][1])
update_from_github.update_version(versions[1][1])
except IOError as e:
launcher_log.warn("check update fail:%r", e)
except Exception as e:
launcher_log.exception("check_update fail:%r", e)
def check_update():
try:
update_rule = config.get(["update", "check_update"], "stable")
if update_rule == "dont-check":
return
check_push_update()
if update_rule != "stable" and update_rule != "test":
return
versions = update_from_github.get_github_versions()
current_version = update_from_github.current_version()
if update_rule == "test":
if LooseVersion(current_version) < LooseVersion(versions[0][1]):
launcher_log.info("update to test version %s", versions[0][1])
update_from_github.update_version(versions[0][1])
elif update_rule == "stable":
if LooseVersion(current_version) < LooseVersion(versions[1][1]):
launcher_log.info("update to stable version %s",
versions[1][1])
update_from_github.update_version(versions[1][1])
except IOError as e:
launcher_log.warn("check update fail:%r", e)
except Exception as e:
launcher_log.exception("check_update fail:%r", e)
def update_version(version):
try:
download_overwrite_new_version(version)
update_config(version)
restart_xxnet()
except Exception as e:
launcher_log.exception("update version %s fail:%r", version, e)
def req_init_module_handler(self):
req = urlparse.urlparse(self.path).query
reqs = urlparse.parse_qs(req, keep_blank_values=True)
data = ""
try:
module = reqs["module"][0]
config.load()
if reqs["cmd"] == ["start"]:
result = module_init.start(module)
data = '{ "module": "%s", "cmd": "start", "result": "%s" }' % (module, result)
elif reqs["cmd"] == ["stop"]:
result = module_init.stop(module)
data = '{ "module": "%s", "cmd": "stop", "result": "%s" }' % (module, result)
elif reqs["cmd"] == ["restart"]:
result_stop = module_init.stop(module)
result_start = module_init.start(module)
data = '{ "module": "%s", "cmd": "restart", "stop_result": "%s", "start_result": "%s" }' % (
module,
result_stop,
result_start,
)
except Exception as e:
launcher_log.exception("init_module except:%s", e)
self.send_response("text/html", data)
def start(module):
if not os.path.isdir(os.path.join(root_path, module)):
return
try:
if not module in config.config["modules"]:
launcher_log.error("module not exist %s", module)
raise
if module in proc_handler:
launcher_log.error("module %s is running", module)
return "module is running"
if module not in proc_handler:
proc_handler[module] = {}
if module == 'ossftp':
masquerade_address = config.get(
["modules", "ossftp", "masquerade_address"], "")
address = config.get(["modules", "ossftp", "address"], "127.0.0.1")
port = config.get(["modules", "ossftp", "port"], 2048)
passive_ports_start = config.get(
["modules", "ossftp", "passive_ports_start"], 51000)
passive_ports_end = config.get(
["modules", "ossftp", "passive_ports_end"], 53000)
is_internal = config.get(["modules", "ossftp", "internal"], None)
log_level = config.get(["modules", "ossftp", "log_level"], "INFO")
bucket_endpoints = config.get(
["modules", "ossftp", "bucket_endpoints"], "")
script_path = os.path.join(root_path, 'ossftp', 'ftpserver.py')
if not os.path.isfile(script_path):
launcher_log.critical("start module script not exist:%s",
script_path)
return "fail"
cmd = [
sys.executable, script_path,
"--listen_address=%s" % address,
"--port=%d" % port,
"--passive_ports_start=%d" % passive_ports_start,
"--passive_ports_end=%d" % passive_ports_end,
"--loglevel=%s" % log_level,
"--bucket_endpoints=%s" % bucket_endpoints
]
print(cmd)
proc_handler[module]["proc"] = subprocess.Popen(
cmd,
shell=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
else:
raise ValueError("Wrong module: %s" % module)
launcher_log.info("%s started", module)
except Exception as e:
launcher_log.exception("start module %s fail:%s", module, e)
return "Except:%s" % e
return "start success."
def get_xxnet_url_version(readme_file):
try:
fd = open(readme_file, "r")
lines = fd.readlines()
p = re.compile(r'https://codeload.github.com/XX-net/XX-Net/zip/([0-9]+)\.([0-9]+)\.([0-9]+)')
for line in lines:
m = p.match(line)
if m:
version = m.group(1) + "." + m.group(2) + "." + m.group(3)
return m.group(0), version
except Exception as e:
launcher_log.exception("xxnet_version fail:%s", e)
raise "get_version_fail:" % readme_file
def get_xxnet_url_version(readme_file):
versions = []
try:
fd = open(readme_file, "r")
lines = fd.readlines()
p = re.compile(r'https://codeload.github.com/XX-net/XX-Net/zip/([0-9]+)\.([0-9]+)\.([0-9]+)')
for line in lines:
m = p.match(line)
if m:
version = m.group(1) + "." + m.group(2) + "." + m.group(3)
versions.append([m.group(0), version])
if len(versions) == 2:
return versions
except Exception as e:
launcher_log.exception("xxnet_version fail:%r", e)
raise "get_version_fail:" % readme_file
def current_version():
readme_file = os.path.join(root_path, "README.md")
try:
fd = open(readme_file, "r")
lines = fd.readlines()
import re
p = re.compile(r'https://codeload.github.com/XX-net/XX-Net/zip/([0-9]+)\.([0-9]+)\.([0-9]+)') #zip/([0-9]+).([0-9]+).([0-9]+)
#m = p.match(content)
for line in lines:
m = p.match(line)
if m:
version = m.group(1) + "." + m.group(2) + "." + m.group(3)
return version
except Exception as e:
launcher_log.exception("xxnet_version fail")
return "get_version_fail"
def stop(module):
try:
if not module in proc_handler:
launcher_log.error("module %s not running", module)
return
proc_handler[module]["proc"].terminate() # Sends SIGTERM
proc_handler[module]["proc"].wait()
#proc_handler[module]["proc"].stop()
del proc_handler[module]
launcher_log.info("module %s stopped", module)
except Exception as e:
launcher_log.exception("stop module %s fail:%s", module, e)
return "Except:%s" % e
return "stop success."
def stop(module):
try:
if not module in proc_handler:
launcher_log.error("module %s not running", module)
return
proc_handler[module]["proc"].terminate() # Sends SIGTERM
proc_handler[module]["proc"].wait()
#proc_handler[module]["proc"].stop()
del proc_handler[module]
launcher_log.info("module %s stopped", module)
except Exception as e:
launcher_log.exception("stop module %s fail:%s", module, e)
return "Except:%s" % e
return "stop success."
def download_file(url, file):
if url not in download_progress:
download_progress[url] = {}
download_progress[url]["status"] = "downloading"
else:
if download_progress[url]["status"] == "downloading":
launcher_log.warn("url in downloading, %s", url)
return False
for i in range(0, 2):
try:
launcher_log.info("download %s to %s, retry:%d", url, file, i)
opener = get_opener(i)
req = opener.open(url)
download_progress[url]["size"] = int(
req.headers.get('content-length') or 0)
CHUNK = 16 * 1024
downloaded = 0
with open(file, 'wb') as fp:
while True:
chunk = req.read(CHUNK)
if not chunk:
break
fp.write(chunk)
downloaded += len(chunk)
download_progress[url]["downloaded"] = downloaded
if downloaded != download_progress[url]["size"]:
launcher_log.warn(
"download size:%d, need size:%d, download fail.",
downloaded, download_progress[url]["size"])
continue
else:
download_progress[url]["status"] = "finished"
return True
except urllib2.URLError as e:
launcher_log.warn("download %s to %s URL fail:%r", url, file, e)
continue
except Exception as e:
launcher_log.exception("download %s to %s fail:%r", url, file, e)
continue
download_progress[url]["status"] = "failed"
return False
def start(module):
if not os.path.isdir(os.path.join(root_path, module)):
return
try:
if module not in config.config["modules"]:
launcher_log.error("module not exist %s", module)
raise
if module in proc_handler:
launcher_log.error("module %s is running", module)
return "module is running"
if module not in proc_handler:
proc_handler[module] = {}
if os.path.isfile(os.path.join(root_path, module, "__init__.py")):
if "imp" not in proc_handler[module]:
proc_handler[module]["imp"] = __import__(
module, globals(), locals(), ['local', 'start'], -1)
_start = proc_handler[module]["imp"].start
p = threading.Thread(target=_start.main)
p.daemon = True
p.start()
proc_handler[module]["proc"] = p
while not _start.client.ready:
time.sleep(0.1)
else:
script_path = os.path.join(root_path, module, 'start.py')
if not os.path.isfile(script_path):
launcher_log.warn("start module script not exist:%s",
script_path)
return "fail"
proc_handler[module]["proc"] = subprocess.Popen(
[sys.executable, script_path], shell=False)
launcher_log.info("%s started", module)
except Exception as e:
launcher_log.exception("start module %s fail:%s", module, e)
return "Except:%s" % e
return "start success."
def start(module):
if not os.path.isdir(os.path.join(root_path, module)):
return
try:
if not module in config.config["modules"]:
launcher_log.error("module not exist %s", module)
raise
if module in proc_handler:
launcher_log.error("module %s is running", module)
return "module is running"
if module not in proc_handler:
proc_handler[module] = {}
if os.path.isfile(os.path.join(root_path, module, "__init__.py")):
if "imp" not in proc_handler[module]:
proc_handler[module]["imp"] = __import__(module, globals(), locals(), ['local', 'start'], -1)
_start = proc_handler[module]["imp"].start
p = threading.Thread(target = _start.main)
p.daemon = True
p.start()
proc_handler[module]["proc"] = p
while not _start.client.ready:
time.sleep(0.1)
else:
script_path = os.path.join(root_path, module, 'start.py')
if not os.path.isfile(script_path):
launcher_log.warn("start module script not exist:%s", script_path)
return "fail"
proc_handler[module]["proc"] = subprocess.Popen([sys.executable, script_path], shell=False)
launcher_log.info("%s started", module)
except Exception as e:
launcher_log.exception("start module %s fail:%s", module, e)
return "Except:%s" % e
return "start success."
def download_file(url, file):
if url not in download_progress:
download_progress[url] = {}
download_progress[url]["status"] = "downloading"
else:
if download_progress[url]["status"] == "downloading":
launcher_log.warn("url in downloading, %s", url)
return False
for i in range(0, 2):
try:
launcher_log.info("download %s to %s, retry:%d", url, file, i)
opener = get_opener(i)
req = opener.open(url)
download_progress[url]["size"] = int(req.headers.get('content-length') or 0)
CHUNK = 16 * 1024
downloaded = 0
with open(file, 'wb') as fp:
while True:
chunk = req.read(CHUNK)
if not chunk:
break
fp.write(chunk)
downloaded += len(chunk)
download_progress[url]["downloaded"] = downloaded
if downloaded != download_progress[url]["size"]:
launcher_log.warn("download size:%d, need size:%d, download fail.", downloaded, download_progress[url]["size"])
continue
else:
download_progress[url]["status"] = "finished"
return True
except urllib2.URLError as e:
launcher_log.warn("download %s to %s URL fail:%r", url, file, e)
continue
except Exception as e:
launcher_log.exception("download %s to %s fail:%r", url, file, e)
continue
download_progress[url]["status"] = "failed"
return False
def start(module):
if not os.path.isdir(os.path.join(root_path, module)):
return
try:
if not module in config.config["modules"]:
launcher_log.error("module not exist %s", module)
raise
if module in proc_handler:
launcher_log.error("module %s is running", module)
return "module is running"
if module not in proc_handler:
proc_handler[module] = {}
if module == 'ossftp':
masquerade_address = config.get(["modules", "ossftp", "masquerade_address"], "")
address = config.get(["modules", "ossftp", "address"], "127.0.0.1")
port = config.get(["modules", "ossftp", "port"], 2048)
is_internal = config.get(["modules", "ossftp", "internal"], None)
log_level = config.get(["modules", "ossftp", "log_level"], "INFO")
bucket_endpoints = config.get(["modules", "ossftp", "bucket_endpoints"], "")
script_path = os.path.join(root_path, 'ossftp', 'ftpserver.py')
if not os.path.isfile(script_path):
launcher_log.critical("start module script not exist:%s", script_path)
return "fail"
cmd = [sys.executable, script_path, "--listen_address=%s"%address, "--port=%d"%port, "--loglevel=%s"%log_level, "--bucket_endpoints=%s"%bucket_endpoints]
print cmd
proc_handler[module]["proc"] = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
#t = FTPd(masquerade_address, address, port, bucket_endpoints, is_internal, log_level)
#t.start()
#proc_handler[module]["proc"] = t
else:
raise ValueError("Wrong module: %s" % module)
launcher_log.info("%s started", module)
except Exception as e:
launcher_log.exception("start module %s fail:%s", module, e)
return "Except:%s" % e
return "start success."
def stop(module):
try:
if not module in proc_handler:
launcher_log.error("module %s not running", module)
return
if os.path.isfile(os.path.join(root_path, module, "__init__.py")):
_start = proc_handler[module]["imp"].start
_start.client.terminate()
launcher_log.debug("module %s stopping", module)
while _start.client.ready:
time.sleep(0.1)
else:
proc_handler[module]["proc"].terminate() # Sends SIGTERM
proc_handler[module]["proc"].wait()
del proc_handler[module]
launcher_log.info("module %s stopped", module)
except Exception as e:
launcher_log.exception("stop module %s fail:%s", module, e)
return "Except:%s" % e
return "stop success."
def stop(module):
try:
if not module in proc_handler:
launcher_log.error("module %s not running", module)
return
if os.path.isfile(os.path.join(root_path, module, "__init__.py")):
_start = proc_handler[module]["imp"].start
_start.client.config.keep_run = False
launcher_log.debug("module %s stopping", module)
while _start.client.ready:
time.sleep(0.1)
else:
proc_handler[module]["proc"].terminate() # Sends SIGTERM
proc_handler[module]["proc"].wait()
del proc_handler[module]
launcher_log.info("module %s stopped", module)
except Exception as e:
launcher_log.exception("stop module %s fail:%s", module, e)
return "Except:%s" % e
return "stop success."
def check_setup(): #40ms
if is_winxp():
try:
copy_VCR_files()
except Exception as e:
launcher_log.exception("setup win python except:%s", e)
def check_push_update():
global update_content, update_dict
try:
opener = get_opener()
req_url = update_url + "?uuid=" + get_uuid() + "&version=" + update_from_github.current_version()
try:
update_content = opener.open(req_url).read()
except Exception as e:
launcher_log.warn("check_update fail:%r", e)
return False
update_dict = json.loads(update_content)
return True
for module in update_dict["modules"]:
new_version = str(update_dict["modules"][module]["last_version"])
describe = update_dict["modules"][module]["versions"][new_version]["describe"]
if update_dict["modules"][module]["versions"][new_version]["notify"] != "true":
continue
if not module in config.config["modules"]:
ignore_version = 0
current_version = 0
config.config["modules"][module] = {}
config.config["modules"][module]["current_version"] = '0.0.0'
else:
current_version = config.get(["modules", module, "current_version"])
if "ignore_version" in config.config["modules"][module]:
ignore_version = config.config["modules"][module]["ignore_version"]
else:
ignore_version = current_version
if version_to_bin(new_version) <= version_to_bin(ignore_version):
continue
if version_to_bin(new_version) > version_to_bin(current_version):
launcher_log.info("new %s version:%s", module, new_version)
if sys.platform == "linux" or sys.platform == "linux2":
from gtk_tray import sys_tray
msg = "Module %s new version: %s, Download?\nNew:%s" % (module, new_version, describe)
data_download = "%s|%s|download" % (module, new_version)
data_ignore = "%s|%s|ignore" % (module, new_version)
buttons = {1: {"data":data_download, "label":"Download", 'callback':general_gtk_callback},
2: {"data":data_ignore, "label":"Ignore", 'callback':general_gtk_callback}}
sys_tray.notify_general(msg=msg, title="New Version", buttons=buttons)
elif sys.platform == "win32":
from win_tray import sys_tray
msg = "Module %s new version: %s, Download?" % (module, new_version)
if sys_tray.dialog_yes_no(msg, u"Download", None, None) == 1:
download_module(module, new_version)
else:
ignore_module(module, new_version)
elif sys.platform == "darwin":
from mac_tray import sys_tray
msg = "Module %s new version: %s, Download?" % (module, new_version)
if sys_tray.dialog_yes_no(msg, u"Download", None, None) == 1:
download_module(module, new_version)
else:
ignore_module(module, new_version)
else:
download_module(module, new_version)
except Exception as e:
launcher_log.exception("check_update except:%s", e)
return
def check_push_update():
global update_content, update_dict
try:
opener = get_opener()
req_url = update_url + "?uuid=" + get_uuid(
) + "&version=" + update_from_github.current_version()
try:
update_content = opener.open(req_url).read()
except Exception as e:
launcher_log.warn("check_update fail:%r", e)
return False
update_dict = json.loads(update_content)
return True
for module in update_dict["modules"]:
new_version = str(update_dict["modules"][module]["last_version"])
describe = update_dict["modules"][module]["versions"][new_version][
"describe"]
if update_dict["modules"][module]["versions"][new_version][
"notify"] != "true":
continue
if not module in config.config["modules"]:
ignore_version = 0
current_version = 0
config.config["modules"][module] = {}
config.config["modules"][module]["current_version"] = '0.0.0'
else:
current_version = config.get(
["modules", module, "current_version"])
if "ignore_version" in config.config["modules"][module]:
ignore_version = config.config["modules"][module][
"ignore_version"]
else:
ignore_version = current_version
if version_to_bin(new_version) <= version_to_bin(ignore_version):
continue
if version_to_bin(new_version) > version_to_bin(current_version):
launcher_log.info("new %s version:%s", module, new_version)
if sys.platform == "linux" or sys.platform == "linux2":
from gtk_tray import sys_tray
msg = "Module %s new version: %s, Download?\nNew:%s" % (
module, new_version, describe)
data_download = "%s|%s|download" % (module, new_version)
data_ignore = "%s|%s|ignore" % (module, new_version)
buttons = {
1: {
"data": data_download,
"label": "Download",
'callback': general_gtk_callback
},
2: {
"data": data_ignore,
"label": "Ignore",
'callback': general_gtk_callback
}
}
sys_tray.notify_general(msg=msg,
title="New Version",
buttons=buttons)
elif sys.platform == "win32":
from win_tray import sys_tray
msg = "Module %s new version: %s, Download?" % (
module, new_version)
if sys_tray.dialog_yes_no(msg, u"Download", None,
None) == 1:
download_module(module, new_version)
else:
ignore_module(module, new_version)
elif sys.platform == "darwin":
from mac_tray import sys_tray
msg = "Module %s new version: %s, Download?" % (
module, new_version)
if sys_tray.dialog_yes_no(msg, u"Download", None,
None) == 1:
download_module(module, new_version)
else:
ignore_module(module, new_version)
else:
download_module(module, new_version)
except Exception as e:
launcher_log.exception("check_update except:%s", e)
return
