def __init__ (self, dyadic_obj):
Debug.__init__(self)
self.dyadic = dyadic_obj
# From Verilog code
self.control_width = 31
self.origin_enabled = BitArray("0b1")
self.origin_disabled = BitArray("0b0")
self.predict_taken = BitArray("0b1")
self.predict_not_taken = BitArray("0b0")
self.predict_enabled = BitArray("0b1")
self.predict_disabled = BitArray("0b0")
self.a_negative = BitArray("0b00")
self.a_carryout = BitArray("0b01")
self.a_sentinel = BitArray("0b10")
self.a_external = BitArray("0b11")
self.b_lessthan = BitArray("0b00")
self.b_counter = BitArray("0b01")
self.b_sentinel = BitArray("0b10")
self.b_external = BitArray("0b11")
self.origin_width = 10
self.origin_enable_width = 1
self.destination_width = 10
self.predict_taken_width = 1
self.predict_enable_width = 1
self.a_width = 2
self.b_width = 2
self.ab_operator_width = self.dyadic.operator_width
self.condition_width = self.a_width + self.b_width + self.ab_operator_width
assert (self.origin_width + self.origin_enable_width + self.destination_width + self.predict_taken_width + self.predict_enable_width + self.condition_width) == self.control_width, "ERROR: Branch Detector control word width and sum of control bits widths do not agree"
def actionStreamFile(self, params):
site = self.sites.get(params["site"])
if not site or not site.settings["serving"]: # Site unknown or not serving
self.response({"error": "Unknown site"})
return False
try:
if config.debug_socket:
self.log.debug("Opening file: %s" % params["inner_path"])
with site.storage.open(params["inner_path"]) as file:
file.seek(params["location"])
stream_bytes = min(FILE_BUFF, os.fstat(file.fileno()).st_size-params["location"])
back = {
"size": os.fstat(file.fileno()).st_size,
"location": min(file.tell() + FILE_BUFF, os.fstat(file.fileno()).st_size),
"stream_bytes": stream_bytes
}
if config.debug_socket:
self.log.debug(
"Sending file %s from position %s to %s" %
(params["inner_path"], params["location"], back["location"])
)
self.response(back)
self.sendRawfile(file, read_bytes=FILE_BUFF)
if config.debug_socket:
self.log.debug("File %s sent" % params["inner_path"])
# Add peer to site if not added before
connected_peer = site.addPeer(self.connection.ip, self.connection.port)
if connected_peer: # Just added
connected_peer.connect(self.connection) # Assign current connection to peer
except Exception, err:
self.log.debug("GetFile read error: %s" % Debug.formatException(err))
self.response({"error": "File read error: %s" % Debug.formatException(err)})
return False
def __init__ (self, data, code, configuration, operators):
Debug.__init__(self)
self.operators = operators
self.init_mems = []
# Do all these first. Instructions and data depend on them
self.OD = Opcode_Decoder(configuration.filename_od, self.operators, code, configuration)
self.init_mems.append(self.OD)
self.PC = Program_Counter(configuration.filename_pc, code, configuration)
self.PC_prev = Program_Counter(configuration.filename_pc_prev, code, configuration)
self.init_mems.append(self.PC)
self.init_mems.append(self.PC_prev)
self.DO = Default_Offset(configuration.filename_do, configuration)
self.init_mems.append(self.DO)
self.BD = Branch_Detector(self.operators.branch_detector, code, configuration)
self.PO = Programmed_Offset(data, configuration)
# Now all Code and Data have been resolved and generated
# We can now create the Data and Instruction Memories
self.A = Data_Memory(configuration.filename_data_A, configuration.data_A_label, data, code, configuration)
self.B = Data_Memory(configuration.filename_data_B, configuration.data_B_label, data, code, configuration)
self.init_mems.append(self.A)
self.init_mems.append(self.B)
self.I = Instruction_Memory(configuration.filename_I, code, configuration)
self.init_mems.append(self.I)
def actionGetFile(self, params):
site = self.sites.get(params["site"])
if not site or not site.settings["serving"]: # Site unknown or not serving
self.response({"error": "Unknown site"})
return False
try:
file_path = site.storage.getPath(params["inner_path"])
with StreamingMsgpack.FilePart(file_path, "rb") as file:
file.seek(params["location"])
file.read_bytes = FILE_BUFF
file_size = os.fstat(file.fileno()).st_size
assert params["location"] <= file_size, "Bad file location"
back = {
"body": file,
"size": file_size,
"location": min(file.tell() + FILE_BUFF, file_size)
}
self.response(back, streaming=True)
bytes_sent = min(FILE_BUFF, file_size - params["location"]) # Number of bytes we going to send
site.settings["bytes_sent"] = site.settings.get("bytes_sent", 0) + bytes_sent
if config.debug_socket:
self.log.debug("File %s at position %s sent %s bytes" % (file_path, params["location"], bytes_sent))
# Add peer to site if not added before
connected_peer = site.addPeer(self.connection.ip, self.connection.port)
if connected_peer: # Just added
connected_peer.connect(self.connection) # Assign current connection to peer
except Exception, err:
self.log.debug("GetFile read error: %s" % Debug.formatException(err))
self.response({"error": "File read error: %s" % Debug.formatException(err)})
return False
def actionGetFile(self, params):
site = self.sites.get(params["site"])
if not site or not site.settings["serving"]: # Site unknown or not serving
self.response({"error": "Unknown site"})
return False
try:
file_path = site.storage.getPath(params["inner_path"])
if config.debug_socket: self.log.debug("Opening file: %s" % file_path)
file = open(file_path, "rb")
file.seek(params["location"])
back = {}
back["body"] = file.read(FILE_BUFF)
back["location"] = file.tell()
back["size"] = os.fstat(file.fileno()).st_size
if config.debug_socket: self.log.debug("Sending file %s from position %s to %s" % (file_path, params["location"], back["location"]))
self.response(back)
if config.debug_socket: self.log.debug("File %s sent" % file_path)
# Add peer to site if not added before
connected_peer = site.addPeer(self.connection.ip, self.connection.port)
if connected_peer: # Just added
connected_peer.connect(self.connection) # Assign current connection to peer
except Exception, err:
self.log.debug("GetFile read error: %s" % Debug.formatException(err))
self.response({"error": "File read error: %s" % Debug.formatException(err)})
return False
def doTest(*args, **kwargs): Debug.prt(kwargs) ## コマンドライン引数をセットする if isinstance(kwargs, dict): env.cExecConfig.setConfig(kwargs) ## jmxが指定されていればアップロード if 'jmx' in kwargs: local_jmx_file = kwargs['jmx'] Debug.prt(kwargs['jmx'],'jmx') #jmxが指定されている時のみ、config=autoを使用出来る if 'config' in kwargs and kwargs['config'] == 'auto': env.cExecConfig.setupAuto() env.cExecConfig.jmx_file = os.path.basename(local_jmx_file) put(local_jmx_file, "%s/%s" % ( env.cExecConfig.jmx_dir, env.cExecConfig.jmx_file )) try: # テストを実行 cExecJmeter = ExecJmeter() cExecJmeter.execTest( env.cExecConfig ) except ClassException, e: print e.message
def __init__ (self, dyadic_obj):
Debug.__init__(self)
self.dyadic = dyadic_obj
# From Verilog code
self.control_width = 20
self.select_r = BitArray("0b00")
self.select_r_zero = BitArray("0b01")
self.select_r_neg = BitArray("0b10")
self.select_s = BitArray("0b11")
self.simple = BitArray("0b0")
self.dual = BitArray("0b1")
self.addsub_a_plus_b = BitArray("0b00")
self.addsub_minus_a_plus_b = BitArray("0b01")
self.addsub_a_minus_b = BitArray("0b10")
self.addsub_minus_a_minus_b = BitArray("0b11")
self.shift_none = BitArray("0b00")
self.shift_right = BitArray("0b01")
self.shift_right_signed = BitArray("0b10")
self.shift_left = BitArray("0b11")
self.split_no = BitArray("0b0")
self.split_yes = BitArray("0b1")
self.select_width = 2
self.dyadic1_width = self.dyadic.operator_width
self.dyadic2_width = self.dyadic.operator_width
self.dual_width = 1
self.addsub_width = 2
self.dyadic3_width = self.dyadic.operator_width
self.shift_width = 2
self.split_width = 1
assert (self.select_width + self.dyadic1_width + self.dyadic2_width + self.dual_width + self.addsub_width + self.dyadic3_width + self.shift_width + self.split_width) == self.control_width, "ERROR: ALU control word width and sum of control bits widths do not agree"
def asyncErrorWatcher(func, *args, **kwargs):
try:
func(*args, **kwargs)
except Exception, err:
if config.debug: # Allow websocket errors to appear on /Debug
sys.modules["main"].DebugHook.handleError()
self.log.error("WebSocket handleRequest error: %s" % Debug.formatException(err))
self.cmd("error", "Internal error: %s" % Debug.formatException(err, "html"))
def actionSiteSign(self, to, privatekey=None, inner_path="content.json", remove_missing_optional=False, update_changed_files=False, response_ok=True):
self.log.debug("Signing: %s" % inner_path)
site = self.site
extend = {} # Extended info for signing
# Change to the file's content.json
file_info = site.content_manager.getFileInfo(inner_path)
if not inner_path.endswith("content.json"):
if not file_info:
raise Exception("Invalid content.json file: %s" % inner_path)
inner_path = file_info["content_inner_path"]
# Add certificate to user files
is_user_content = file_info and ("cert_signers" in file_info or "cert_signers_pattern" in file_info)
if is_user_content and privatekey is None:
cert = self.user.getCert(self.site.address)
extend["cert_auth_type"] = cert["auth_type"]
extend["cert_user_id"] = self.user.getCertUserId(site.address)
extend["cert_sign"] = cert["cert_sign"]
self.log.debug("Extending content.json with cert %s" % extend["cert_user_id"])
if not self.hasFilePermission(inner_path):
self.log.error("SiteSign error: you don't own this site & site owner doesn't allow you to do so.")
return self.response(to, {"error": "Forbidden, you can only modify your own sites"})
if privatekey == "stored": # Get privatekey from sites.json
privatekey = self.user.getSiteData(self.site.address).get("privatekey")
if not privatekey: # Get privatekey from users.json auth_address
privatekey = self.user.getAuthPrivatekey(self.site.address)
# Signing
# Reload content.json, ignore errors to make it up-to-date
site.content_manager.loadContent(inner_path, add_bad_files=False, force=True)
# Sign using private key sent by user
try:
site.content_manager.sign(inner_path, privatekey, extend=extend, update_changed_files=update_changed_files, remove_missing_optional=remove_missing_optional)
except (VerifyError, SignError) as err:
self.cmd("notification", ["error", _["Content signing failed"] + "<br><small>%s</small>" % err])
self.response(to, {"error": "Site sign failed: %s" % err})
self.log.error("Site sign failed: %s: %s" % (inner_path, Debug.formatException(err)))
return
except Exception as err:
self.cmd("notification", ["error", _["Content signing error"] + "<br><small>%s</small>" % Debug.formatException(err)])
self.response(to, {"error": "Site sign error: %s" % Debug.formatException(err)})
self.log.error("Site sign error: %s: %s" % (inner_path, Debug.formatException(err)))
return
site.content_manager.loadContent(inner_path, add_bad_files=False) # Load new content.json, ignore errors
if update_changed_files:
self.site.updateWebsocket(file_done=inner_path)
if response_ok:
self.response(to, "ok")
else:
return inner_path
def __init__ (self, code, data, configuration, operators):
Debug.__init__(self)
self.operators = operators
self.code = code
self.data = data
self.configuration = configuration
self.defined_opcodes = {} # {label:opcode_obj}
# One set of initial/current opcodes per thread
self.initial_opcodes = [[None for entry in range(self.configuration.opcode_count)] for thread in range(self.configuration.thread_count)]
self.current_opcodes = [[None for entry in range(self.configuration.opcode_count)] for thread in range(self.configuration.thread_count)]
def __init__ (self, label = None, address = None, opcode = None, D = None, DA = None, DB = None, A = None, B = None):
Debug.__init__(self)
self.label = label
self.address = address
self.opcode = opcode
self.D = D
self.DA = DA
self.DB = DB
self.A = A
self.B = B
def __init__ (self, data, code, label = None, destination = None,):
Debug.__init__(self)
self.label = label
self.destination = destination
self.instructions = []
self.init_data = []
self.data = data
self.code = code
# keep any init instructions added later in order,
# so add list of init instructions to global list
self.code.instructions.append(self.instructions)
def __init__ (self, label, split, shift, dyadic3, addsub, dual, dyadic2, dyadic1, select, operators):
Debug.__init__(self)
self.label = label
self.split = split
self.shift = shift
self.dyadic3 = dyadic3
self.addsub = addsub
self.dual = dual
self.dyadic2 = dyadic2
self.dyadic1 = dyadic1
self.select = select
self.binary = self.to_binary(operators)
def __init__ (self, data, configuration, operators):
Debug.__init__(self)
self.operators = operators
self.data = data
self.configuration = configuration
self.usage = Usage(configuration)
self.init_loads = []
self.instructions = []
self.opcodes = Opcode_Manager(self, data, configuration, operators)
self.conditions = []
self.branches = []
self.initial_pc = []
def publisher(self, inner_path, peers, published, limit, diffs={}, event_done=None, cb_progress=None):
file_size = self.storage.getSize(inner_path)
content_json_modified = self.content_manager.contents[inner_path]["modified"]
body = self.storage.read(inner_path)
while 1:
if not peers or len(published) >= limit:
if event_done:
event_done.set(True)
break # All peers done, or published engouht
peer = peers.pop()
if peer in published:
continue
if peer.last_content_json_update == content_json_modified:
self.log.debug("%s already received this update for %s, skipping" % (peer, inner_path))
continue
if peer.connection and peer.connection.last_ping_delay: # Peer connected
# Timeout: 5sec + size in kb + last_ping
timeout = 5 + int(file_size / 1024) + peer.connection.last_ping_delay
else: # Peer not connected
# Timeout: 10sec + size in kb
timeout = 10 + int(file_size / 1024)
result = {"exception": "Timeout"}
for retry in range(2):
try:
with gevent.Timeout(timeout, False):
result = peer.request("update", {
"site": self.address,
"inner_path": inner_path,
"body": body,
"diffs": diffs
})
if result:
break
except Exception, err:
self.log.error("Publish error: %s" % Debug.formatException(err))
result = {"exception": Debug.formatException(err)}
if result and "ok" in result:
published.append(peer)
if cb_progress and len(published) <= limit:
cb_progress(len(published), limit)
self.log.info("[OK] %s: %s %s/%s" % (peer.key, result["ok"], len(published), limit))
else:
if result == {"exception": "Timeout"}:
peer.onConnectionError("Publish timeout")
self.log.info("[FAILED] %s: %s" % (peer.key, result))
time.sleep(0.01)
def find_largest_contour(seq, exclude = None): max_area = 0 largest = None try: while seq: if seq != exclude: area = cv.ContourArea(seq) if max_area < area: max_area = area largest = seq seq = seq.h_next() return largest except: Debug.print_stacktrace() return None
def connect(self, connection=None):
if self.connection:
self.log("Getting connection (Closing %s)..." % self.connection)
self.connection.close()
else:
self.log("Getting connection...")
if connection: # Connection specified
self.connection = connection
else: # Try to find from connection pool or create new connection
self.connection = None
try:
if self.connection_server:
self.connection = self.connection_server.getConnection(self.ip, self.port, site=self.site)
elif self.site:
self.connection = self.site.connection_server.getConnection(self.ip, self.port, site=self.site)
else:
self.connection = sys.modules["main"].file_server.getConnection(self.ip, self.port, site=self.site)
except Exception, err:
self.onConnectionError()
self.log("Getting connection error: %s (connection_error: %s, hash_failed: %s)" %
(Debug.formatException(err), self.connection_error, self.hash_failed))
self.connection = None
def send(self, message, streaming=False):
if config.debug_socket:
self.log(
"Send: %s, to: %s, streaming: %s, site: %s, inner_path: %s, req_id: %s"
% (
message.get("cmd"),
message.get("to"),
streaming,
message.get("params", {}).get("site"),
message.get("params", {}).get("inner_path"),
message.get("req_id"),
)
)
self.last_send_time = time.time()
try:
if streaming:
bytes_sent = StreamingMsgpack.stream(message, self.sock.sendall)
message = None
self.bytes_sent += bytes_sent
self.server.bytes_sent += bytes_sent
else:
data = msgpack.packb(message)
message = None
self.bytes_sent += len(data)
self.server.bytes_sent += len(data)
self.sock.sendall(data)
except Exception, err:
self.log("Send errror: %s" % Debug.formatException(err))
self.close()
return False
def request(self, cmd, params = {}):
if not self.connection or self.connection.closed:
self.connect()
if not self.connection:
self.onConnectionError()
return None # Connection failed
#if cmd != "ping" and self.last_response and time.time() - self.last_response > 20*60: # If last response if older than 20 minute, ping first to see if still alive
# if not self.ping(): return None
for retry in range(1,3): # Retry 3 times
#if config.debug_socket: self.log.debug("sendCmd: %s %s" % (cmd, params.get("inner_path")))
try:
response = self.connection.request(cmd, params)
if not response: raise Exception("Send error")
#if config.debug_socket: self.log.debug("Got response to: %s" % cmd)
if "error" in response:
self.log.debug("%s error: %s" % (cmd, response["error"]))
self.onConnectionError()
else: # Successful request, reset connection error num
self.connection_error = 0
self.last_response = time.time()
return response
except Exception, err:
if type(err).__name__ == "Notify": # Greenlet kill by worker
self.log.debug("Peer worker got killed: %s, aborting cmd: %s" % (err.message, cmd))
break
else:
self.onConnectionError()
self.log.debug("%s (connection_error: %s, hash_failed: %s, retry: %s)" % (Debug.formatException(err), self.connection_error, self.hash_failed, retry))
time.sleep(1*retry)
self.connect()
def request(self, cmd, params={}, stream_to=None):
if not self.connection or self.connection.closed:
self.connect()
if not self.connection:
self.onConnectionError()
return None # Connection failed
self.log("Send request: %s %s" % (params.get("site", ""), cmd))
for retry in range(1, 4): # Retry 3 times
try:
res = self.connection.request(cmd, params, stream_to)
if not res:
raise Exception("Send error")
if "error" in res:
self.log("%s error: %s" % (cmd, res["error"]))
self.onConnectionError()
else: # Successful request, reset connection error num
self.connection_error = 0
self.time_response = time.time()
return res
except Exception, err:
if type(err).__name__ == "Notify": # Greenlet killed by worker
self.log("Peer worker got killed: %s, aborting cmd: %s" % (err.message, cmd))
break
else:
self.onConnectionError()
self.log(
"%s (connection_error: %s, hash_failed: %s, retry: %s)" %
(Debug.formatException(err), self.connection_error, self.hash_failed, retry)
)
time.sleep(1 * retry)
self.connect()
def createBroadcastSocket(self):
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
if hasattr(socket, 'SO_REUSEPORT'):
try:
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
except Exception as err:
self.log.warning("Error setting SO_REUSEPORT: %s" % err)
binded = False
for retry in range(3):
try:
sock.bind((self.listen_ip, self.listen_port))
binded = True
break
except Exception as err:
self.log.error(
"Socket bind to %s:%s error: %s, retry #%s" %
(self.listen_ip, self.listen_port, Debug.formatException(err), retry)
)
time.sleep(retry)
if binded:
return sock
else:
return False
def start(self): # Listens for discover requests
self.sock = self.createBroadcastSocket()
if not self.sock:
self.log.error("Unable to listen on port %s" % self.listen_port)
return
self.log.debug("Started on port %s" % self.listen_port)
self.running = True
while self.running:
try:
data, addr = self.sock.recvfrom(8192)
except Exception as err:
if self.running:
self.log.error("Listener receive error: %s" % err)
continue
if not self.running:
break
try:
message = msgpack.unpackb(data)
response_addr, message = self.handleMessage(addr, message)
if message:
self.send(response_addr, message)
except Exception as err:
self.log.error("Handlemessage error: %s" % Debug.formatException(err))
self.log.debug("Stopped listening on port %s" % self.listen_port)
def openport(self, port=None, check=True):
if not port:
port = self.port
if self.port_opened:
return True # Port already opened
if check: # Check first if its already opened
time.sleep(1) # Wait for port open
if self.testOpenport(port, use_alternative=False)["result"] is True:
return True # Port already opened
if config.tor == "always": # Port opening won't work in Tor mode
return False
self.log.info("Trying to open port using UpnpPunch...")
try:
UpnpPunch.ask_to_open_port(self.port, 'ZeroNet', retries=3, protos=["TCP"])
except (UpnpPunch.UpnpError, UpnpPunch.IGDError, socket.error) as err:
self.log.error("UpnpPunch run error: %s" %
Debug.formatException(err))
return False
if self.testOpenport(port)["result"] is True:
self.upnp_port_opened = True
return True
self.log.info("Upnp mapping failed :( Please forward port %s on your router to your ipaddress" % port)
return False
def publisher(self, inner_path, peers, published, limit, event_done=None):
file_size = self.storage.getSize(inner_path)
body = self.storage.read(inner_path)
while 1:
if not peers or len(published) >= limit:
if event_done: event_done.set(True)
break # All peers done, or published engouht
peer = peers.pop(0)
if peer.connection and peer.connection.last_ping_delay: # Peer connected
timeout = timeout = 5+int(file_size/1024)+peer.connection.last_ping_delay # Timeout: 5sec + size in kb + last_ping
else:
timeout = timeout = 5+int(file_size/1024) # Timeout: 5sec + size in kb
result = {"exception": "Timeout"}
for retry in range(2):
try:
with gevent.Timeout(timeout, False):
result = peer.request("update", {
"site": self.address,
"inner_path": inner_path,
"body": body,
"peer": (config.ip_external, config.fileserver_port)
})
if result: break
except Exception, err:
result = {"exception": Debug.formatException(err)}
if result and "ok" in result:
published.append(peer)
self.log.info("[OK] %s: %s" % (peer.key, result["ok"]))
else:
if result == {"exception": "Timeout"}: peer.onConnectionError()
self.log.info("[FAILED] %s: %s" % (peer.key, result))
def startTor(self):
if sys.platform.startswith("win"):
try:
if not os.path.isfile(self.tor_exe):
self.downloadTor()
self.log.info("Starting Tor client %s..." % self.tor_exe)
tor_dir = os.path.dirname(self.tor_exe)
startupinfo = subprocess.STARTUPINFO()
startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
cmd = r"%s -f torrc --defaults-torrc torrc-defaults --ignore-missing-torrc" % self.tor_exe
if config.tor_use_bridges:
cmd += " --UseBridges 1"
self.tor_process = subprocess.Popen(cmd, cwd=tor_dir, close_fds=True, startupinfo=startupinfo)
for wait in range(1, 10): # Wait for startup
time.sleep(wait * 0.5)
self.enabled = True
if self.connect():
if self.isSubprocessRunning():
self.request("TAKEOWNERSHIP") # Shut down Tor client when controll connection closed
break
# Terminate on exit
atexit.register(self.stopTor)
except Exception, err:
self.log.error(u"Error starting Tor client: %s" % Debug.formatException(str(err).decode("utf8", "ignore")))
self.enabled = False
def siteSign(self, address, privatekey=None, inner_path="content.json", publish=False, remove_missing_optional=False):
from Site import Site
from Site import SiteManager
from Debug import Debug
SiteManager.site_manager.load()
logging.info("Signing site: %s..." % address)
site = Site(address, allow_create=False)
if not privatekey: # If no privatekey defined
from User import UserManager
user = UserManager.user_manager.get()
if user:
site_data = user.getSiteData(address)
privatekey = site_data.get("privatekey")
else:
privatekey = None
if not privatekey:
# Not found in users.json, ask from console
import getpass
privatekey = getpass.getpass("Private key (input hidden):")
diffs = site.content_manager.getDiffs(inner_path)
try:
succ = site.content_manager.sign(inner_path=inner_path, privatekey=privatekey, update_changed_files=True, remove_missing_optional=remove_missing_optional)
except Exception, err:
logging.error("Sign error: %s" % Debug.formatException(err))
succ = False
def __init__ (self, configuration):
Debug.__init__(self)
self.configuration = configuration
self.po_in_use = dict()
for operand in ["A", "B", "DA", "DB"]:
self.po_in_use[operand] = self.init_flags(self.configuration.memory_map.po[operand])
self.sentinel_in_use = dict()
self.mask_in_use = dict()
for operand in ["A", "B"]:
self.sentinel_in_use[operand] = self.init_flags(self.configuration.memory_map.sentinel[operand])
self.mask_in_use[operand] = self.init_flags(self.configuration.memory_map.mask[operand])
self.bc_in_use = self.init_flags(self.configuration.memory_map.bc)
self.bd_in_use = self.init_flags(self.configuration.memory_map.bd)
def actionFileWrite(self, to, inner_path, content_base64):
if (
not self.site.settings["own"] and
self.user.getAuthAddress(self.site.address) not in self.site.content_manager.getValidSigners(inner_path)
):
return self.response(to, {"error": "Forbidden, you can only modify your own files"})
try:
import base64
content = base64.b64decode(content_base64)
# Save old file to generate patch later
if (
inner_path.endswith(".json") and not inner_path.endswith("content.json") and
self.site.storage.isFile(inner_path) and not self.site.storage.isFile(inner_path + "-old")
):
try:
self.site.storage.rename(inner_path, inner_path + "-old")
except Exception:
# Rename failed, fall back to standard file write
f_old = self.site.storage.open(inner_path, "rb")
f_new = self.site.storage.open(inner_path + "-old", "wb")
shutil.copyfileobj(f_old, f_new)
self.site.storage.write(inner_path, content)
except Exception, err:
return self.response(to, {"error": "Write error: %s" % Debug.formatException(err)})
def actionSidebarGetPeers(self, to):
permissions = self.getPermissions(to)
if "ADMIN" not in permissions:
return self.response(to, "You don't have permission to run this command")
try:
peer_locations = self.getPeerLocations(self.site.peers)
globe_data = []
ping_times = [
peer_location["ping"]
for peer_location in peer_locations
if peer_location["ping"]
]
if ping_times:
ping_avg = sum(ping_times) / float(len(ping_times))
else:
ping_avg = 0
for peer_location in peer_locations:
if peer_location["ping"] == 0: # Me
height = -0.135
elif peer_location["ping"]:
height = min(0.20, math.log(1 + peer_location["ping"] / ping_avg, 300))
else:
height = -0.03
globe_data += [peer_location["lat"], peer_location["lon"], height]
self.response(to, globe_data)
except Exception, err:
self.log.debug("sidebarGetPeers error: %s" % Debug.formatException(err))
self.response(to, {"error": err})
def messageLoop(self):
if not self.sock:
self.log("Socket error: No socket found")
return False
self.protocol = "v2"
self.updateName()
self.connected = True
self.unpacker = msgpack.Unpacker()
sock = self.sock
try:
while True:
buff = sock.recv(16*1024)
if not buff: break # Connection closed
self.last_recv_time = time.time()
self.incomplete_buff_recv += 1
self.bytes_recv += len(buff)
self.server.bytes_recv += len(buff)
if not self.unpacker:
self.unpacker = msgpack.Unpacker()
self.unpacker.feed(buff)
for message in self.unpacker:
self.incomplete_buff_recv = 0
self.handleMessage(message)
message = None
buff = None
except Exception, err:
if not self.closed: self.log("Socket error: %s" % Debug.formatException(err))
def toBinaryTest():
Debug.debug(QuantumMath.toBinary(5, 3), DebugLevel.Result)
Debug.debug(QuantumMath.toBinary(5, 5), DebugLevel.Result)
Debug.debug(QuantumMath.toBinary(16, 6), DebugLevel.Result)
Debug.debug(QuantumMath.toBinary(31, 6), DebugLevel.Result)
def specialTest():
dic = DictionarySearcher(2)
matrix = np.matrix([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0],
[0, 0, 0, -1]])
id = AQP.generateSpecial(matrix=matrix, name="prueba1", dic=dic)
Debug.debug(Visualizer.printPath(id, dic, deepEnd=False),
DebugLevel.Result)
Debug.debug(Visualizer.printCompress(id, dic), DebugLevel.Result)
Debug.debug(
Visualizer.printCompress(id,
dic,
deepEnd=True,
deep=1,
autoReference=True), DebugLevel.Result)
Debug.debug("\n" + str(dic.getGate(id).getMatrix()) + "\n\n",
DebugLevel.Result)
matrix = np.matrix([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0],
[0, 0, 0, 1j]])
id = AQP.generateSpecial(matrix=matrix, name="prueba2", dic=dic)
Debug.debug(Visualizer.printPath(id, dic, deepEnd=False),
DebugLevel.Result)
Debug.debug(Visualizer.printCompress(id, dic), DebugLevel.Result)
Debug.debug(
Visualizer.printCompress(id,
dic,
deepEnd=True,
deep=1,
autoReference=True), DebugLevel.Result)
Debug.debug("\n" + str(dic.getGate(id).getMatrix()) + "\n\n",
DebugLevel.Result)
matrix = np.matrix([[-1, 0, 0, 0], [0, 1, 0, 0],
[0, 0, (1 / math.sqrt(2)) * (1 + 1j), 0],
[0, 0, 0, 1j]])
id = AQP.generateSpecial(matrix=matrix, name="prueba3", dic=dic)
Debug.debug(Visualizer.printPath(id, dic, deepEnd=False),
DebugLevel.Result)
Debug.debug(Visualizer.printCompress(id, dic), DebugLevel.Result)
Debug.debug(
Visualizer.printCompress(id,
dic,
deepEnd=True,
deep=1,
autoReference=True), DebugLevel.Result)
Debug.debug("\n" + str(dic.getGate(id).getMatrix()) + "\n\n",
DebugLevel.Result)
matrix = np.matrix([[1j, 0, 0, 0],
[0, (1 / math.sqrt(2)), (-1 / math.sqrt(2)), 0],
[0, (1 / math.sqrt(2)), (1 / math.sqrt(2)), 0],
[0, 0, 0, -1]])
id = AQP.generateSpecial(matrix=matrix, name="prueba4", dic=dic)
Debug.debug(Visualizer.printPath(id, dic, deepEnd=False),
DebugLevel.Result)
Debug.debug(Visualizer.printCompress(id, dic), DebugLevel.Result)
Debug.debug(
Visualizer.printCompress(id,
dic,
deepEnd=True,
deep=1,
autoReference=True), DebugLevel.Result)
Debug.debug("\n" + str(dic.getGate(id).getMatrix()) + "\n\n",
DebugLevel.Result)
"""
Debug.debug("\n" + str(dic.getGate(67).getMatrix()) + "\n\n", DebugLevel.Result)
Debug.debug("\n" + str(dic.getGate(68).getMatrix()) + "\n\n", DebugLevel.Result)
Debug.debug("\n" + str(dic.getGate(78).getMatrix()) + "\n\n", DebugLevel.Result)
Debug.debug("\n" + str(dic.getGate(68).getMatrix()) + "\n\n", DebugLevel.Result)
Debug.debug("\n" + str(dic.getGate(67).getMatrix()) + "\n\n", DebugLevel.Result)
Debug.debug("\n" + str(AQP.generateMatrixFromPath([68,67,78,67,68], dic)) + "\n\n", DebugLevel.Result)
"""
matrix = np.matrix([[-1, 1, 1, 1], [1, -1, 1, 1], [1, 1, -1, 1],
[1, 1, 1, -1]]) * (1 / 2)
id = AQP.generateSpecial(matrix=matrix, name="prueba5", dic=dic)
Debug.debug(Visualizer.printPath(id, dic, deepEnd=False),
DebugLevel.Result)
Debug.debug(Visualizer.printCompress(id, dic), DebugLevel.Result)
Debug.debug(
Visualizer.printCompress(id,
dic,
deepEnd=True,
deep=1,
autoReference=True), DebugLevel.Result)
Debug.debug("\n" + str(dic.getGate(id).getMatrix()) + "\n\n",
DebugLevel.Result)
dic = DictionarySearcher(3)
matrix = np.eye(2**3, dtype=complex)
matrix[7, 7] = -1
id = AQP.generateSpecial(matrix=matrix, name="prueba6", dic=dic)
Debug.debug(Visualizer.printPath(id, dic, deepEnd=False),
DebugLevel.Result)
Debug.debug(Visualizer.printCompress(id, dic), DebugLevel.Result)
Debug.debug(
Visualizer.printCompress(id,
dic,
deepEnd=True,
deep=1,
autoReference=True), DebugLevel.Result)
Debug.debug("\n" + str(dic.getGate(id).getMatrix()) + "\n\n",
DebugLevel.Result)
matrix[5, 6] = 1
matrix[6, 5] = 1
matrix[5, 5] = 0
matrix[6, 6] = 0
id = AQP.generateSpecial(matrix=matrix, name="prueba67", dic=dic)
Debug.debug(Visualizer.printPath(id, dic, deepEnd=False),
DebugLevel.Result)
Debug.debug(Visualizer.printCompress(id, dic), DebugLevel.Result)
Debug.debug(
Visualizer.printCompress(id,
dic,
deepEnd=True,
deep=1,
autoReference=True), DebugLevel.Result)
Debug.debug("\n" + str(dic.getGate(id).getMatrix()) + "\n\n",
DebugLevel.Result)
def verifyFile(self, inner_path, file, ignore_same=True):
if inner_path.endswith("content.json"): # content.json: Check using sign
from Crypt import CryptBitcoin
try:
if type(file) is dict:
new_content = file
else:
new_content = json.load(file)
if inner_path in self.contents:
old_content = self.contents.get(inner_path, {"modified": 0})
# Checks if its newer the ours
if old_content["modified"] == new_content["modified"] and ignore_same: # Ignore, have the same content.json
return None
elif old_content["modified"] > new_content["modified"]: # We have newer
raise VerifyError(
"We have newer (Our: %s, Sent: %s)" %
(old_content["modified"], new_content["modified"])
)
if new_content["modified"] > time.time() + 60 * 60 * 24: # Content modified in the far future (allow 1 day+)
raise VerifyError("Modify timestamp is in the far future!")
if self.isArchived(inner_path, new_content["modified"]):
if inner_path in self.site.bad_files:
del self.site.bad_files[inner_path]
raise VerifyError("This file is archived!")
# Check sign
sign = new_content.get("sign")
signs = new_content.get("signs", {})
if "sign" in new_content:
del(new_content["sign"]) # The file signed without the sign
if "signs" in new_content:
del(new_content["signs"]) # The file signed without the signs
sign_content = json.dumps(new_content, sort_keys=True) # Dump the json to string to remove whitepsace
# Fix float representation error on Android
modified = new_content["modified"]
if config.fix_float_decimals and type(modified) is float and not str(modified).endswith(".0"):
modified_fixed = "{:.6f}".format(modified).strip("0.")
sign_content = sign_content.replace(
'"modified": %s' % repr(modified),
'"modified": %s' % modified_fixed
)
self.verifyContent(inner_path, new_content)
if signs: # New style signing
valid_signers = self.getValidSigners(inner_path, new_content)
signs_required = self.getSignsRequired(inner_path, new_content)
if inner_path == "content.json" and len(valid_signers) > 1: # Check signers_sign on root content.json
signers_data = "%s:%s" % (signs_required, ",".join(valid_signers))
if not CryptBitcoin.verify(signers_data, self.site.address, new_content["signers_sign"]):
raise VerifyError("Invalid signers_sign!")
if inner_path != "content.json" and not self.verifyCert(inner_path, new_content): # Check if cert valid
raise VerifyError("Invalid cert!")
valid_signs = 0
for address in valid_signers:
if address in signs:
valid_signs += CryptBitcoin.verify(sign_content, address, signs[address])
if valid_signs >= signs_required:
break # Break if we has enough signs
if valid_signs < signs_required:
raise VerifyError("Valid signs: %s/%s" % (valid_signs, signs_required))
else:
return True
else: # Old style signing
if CryptBitcoin.verify(sign_content, self.site.address, sign):
return True
else:
raise VerifyError("Invalid old-style sign")
except Exception, err:
self.log.warning("%s: verify sign error: %s" % (inner_path, Debug.formatException(err)))
raise err
def getConnection(self,
ip=None,
port=None,
peer_id=None,
create=True,
site=None):
if ip.endswith(
".onion"
) and self.tor_manager.start_onions and site: # Site-unique connection for Tor
site_onion = self.tor_manager.getOnion(site.address)
key = ip + site_onion
elif ip.endswith(
".i2p"
) and self.i2p_manager.start_onions and site: # Site-unique connection for I2P
site_i2p = self.i2p_manager.getOnion(site.address)
key = ip + site_i2p
else:
key = ip
# Find connection by ip
if key in self.ips:
connection = self.ips[key]
if not peer_id or connection.handshake.get(
"peer_id") == peer_id: # Filter by peer_id
if not connection.connected and create:
succ = connection.event_connected.get(
) # Wait for connection
if not succ:
raise Exception("Connection event return error")
return connection
# Recover from connection pool
for connection in self.connections:
if connection.ip == ip:
if peer_id and connection.handshake.get(
"peer_id") != peer_id: # Does not match
continue
if ip.endswith(
".onion"
) and self.tor_manager.start_onions and ip.replace(
".onion", "") != connection.target_onion:
# For different site
continue
if ip.endswith(
".i2p"
) and self.i2p_manager.start_onions and ip.replace(
".i2p", "") != connection.target_i2p:
# For different site
continue
if not connection.connected and create:
succ = connection.event_connected.get(
) # Wait for connection
if not succ:
raise Exception("Connection event return error")
return connection
# No connection found
if create: # Allow to create new connection if not found
if port == 0:
raise Exception("This peer is not connectable")
if (ip, port) in self.peer_blacklist:
raise Exception("This peer is blacklisted")
try:
if ip.endswith(
".onion"
) and self.tor_manager.start_onions and site: # Lock connection to site
connection = Connection(self,
ip,
port,
target_onion=site_onion,
target_i2p=None)
elif ip.endswith(
".i2p"
) and self.i2p_manager.start_onions and site: # Lock connection to site
connection = Connection(self,
ip,
port,
target_onion=None,
target_i2p=site_i2p)
else:
connection = Connection(self, ip, port)
self.ips[key] = connection
self.connections.append(connection)
succ = connection.connect()
if not succ:
connection.close("Connection event return error")
raise Exception("Connection event return error")
except Exception, err:
connection.close("%s Connect error: %s" %
(ip, Debug.formatException(err)))
raise err
if len(self.connections) > config.global_connected_limit:
gevent.spawn(self.checkMaxConnections)
return connection
def rowReverseGateTest():
dic = DictionarySearcher(2)
id = AQP.generateRowReverse(target1=0, target2=3, dic=dic)
Debug.debug(Visualizer.printPath(id, dic, deepEnd=False),
DebugLevel.Result)
Debug.debug(
Visualizer.printCompress(id,
dic,
deepEnd=True,
deep=1,
autoReference=True), DebugLevel.Result)
Debug.debug(
"\n" + Visualizer.printIntegerMatrix(dic.getGate(id).getMatrix()) +
"\n\n", DebugLevel.Result)
id = AQP.generateRowReverse(target1=2, target2=1, dic=dic)
Debug.debug(Visualizer.printPath(id, dic, deepEnd=False),
DebugLevel.Result)
Debug.debug(
Visualizer.printCompress(id,
dic,
deepEnd=True,
deep=1,
autoReference=True), DebugLevel.Result)
Debug.debug(
"\n" + Visualizer.printIntegerMatrix(dic.getGate(id).getMatrix()) +
"\n\n", DebugLevel.Result)
id = AQP.generateRowReverse(target1=1, target2=2, dic=dic)
Debug.debug(Visualizer.printPath(id, dic, deepEnd=False),
DebugLevel.Result)
Debug.debug(
Visualizer.printCompress(id,
dic,
deepEnd=True,
deep=1,
autoReference=True), DebugLevel.Result)
Debug.debug(
"\n" + Visualizer.printIntegerMatrix(dic.getGate(id).getMatrix()) +
"\n\n", DebugLevel.Result)
dic = DictionarySearcher(3)
id = AQP.generateRowReverse(target1=0, target2=3, dic=dic)
Debug.debug(Visualizer.printPath(id, dic, deepEnd=False),
DebugLevel.Result)
Debug.debug(
Visualizer.printCompress(id,
dic,
deepEnd=True,
deep=1,
autoReference=True), DebugLevel.Result)
Debug.debug(
"\n" + Visualizer.printIntegerMatrix(dic.getGate(id).getMatrix()) +
"\n\n", DebugLevel.Result)
id = AQP.generateRowReverse(target1=1, target2=7, dic=dic)
Debug.debug(Visualizer.printPath(id, dic, deepEnd=False),
DebugLevel.Result)
Debug.debug(
Visualizer.printCompress(id,
dic,
deepEnd=True,
deep=1,
autoReference=True), DebugLevel.Result)
Debug.debug(
"\n" + Visualizer.printIntegerMatrix(dic.getGate(id).getMatrix()) +
"\n\n", DebugLevel.Result)
id = AQP.generateRowReverse(target1=4, target2=2, dic=dic)
Debug.debug(Visualizer.printPath(id, dic, deepEnd=False),
DebugLevel.Result)
Debug.debug(
Visualizer.printCompress(id,
dic,
deepEnd=True,
deep=1,
autoReference=True), DebugLevel.Result)
Debug.debug(
"\n" + Visualizer.printIntegerMatrix(dic.getGate(id).getMatrix()) +
"\n\n", DebugLevel.Result)
def actionFeedQuery(self, to, limit=10, day_limit=3):
if "ADMIN" not in self.site.settings["permissions"]:
return self.response(to, "FeedQuery not allowed")
from Site import SiteManager
rows = []
stats = []
total_s = time.time()
num_sites = 0
for address, site_data in self.user.sites.items():
feeds = site_data.get("follow")
if not feeds:
continue
if type(feeds) is not dict:
self.log.debug("Invalid feed for site %s" % address)
continue
num_sites += 1
for name, query_set in feeds.iteritems():
site = SiteManager.site_manager.get(address)
if not site or not site.storage.has_db:
continue
s = time.time()
try:
query_raw, params = query_set
query_parts = re.split(r"UNION(?:\s+ALL|)", query_raw)
for i, query_part in enumerate(query_parts):
db_query = DbQuery(query_part)
if day_limit:
where = " WHERE %s > strftime('%%s', 'now', '-%s day')" % (
db_query.fields.get("date_added",
"date_added"), day_limit)
if "WHERE" in query_part:
query_part = re.sub(
"WHERE (.*?)(?=$| GROUP BY)",
where + " AND (\\1)", query_part)
else:
query_part += where
query_parts[i] = query_part
query = " UNION ".join(query_parts)
if ":params" in query:
query = query.replace(":params",
",".join(["?"] * len(params)))
res = site.storage.query(
query +
" ORDER BY date_added DESC LIMIT %s" % limit,
params * query_raw.count(":params"))
else:
res = site.storage.query(
query +
" ORDER BY date_added DESC LIMIT %s" % limit)
except Exception as err: # Log error
self.log.error("%s feed query %s error: %s" %
(address, name, Debug.formatException(err)))
stats.append({
"site": site.address,
"feed_name": name,
"error": str(err),
"query": query
})
continue
for row in res:
row = dict(row)
if not isinstance(row["date_added"],
(int, long, float, complex)):
self.log.debug("Invalid date_added from site %s: %r" %
(address, row["date_added"]))
continue
if row["date_added"] > 1000000000000: # Formatted as millseconds
row["date_added"] = row["date_added"] / 1000
if "date_added" not in row or row[
"date_added"] > time.time() + 120:
self.log.debug(
"Newsfeed item from the future from from site %s" %
address)
continue # Feed item is in the future, skip it
row["site"] = address
row["feed_name"] = name
rows.append(row)
stats.append({
"site": site.address,
"feed_name": name,
"taken": round(time.time() - s, 3)
})
time.sleep(0.0001)
return self.response(
to, {
"rows": rows,
"stats": stats,
"num": len(rows),
"sites": num_sites,
"taken": round(time.time() - total_s, 3)
})
def publisher(self,
inner_path,
peers,
published,
limit,
event_done=None,
diffs={}):
file_size = self.storage.getSize(inner_path)
content_json_modified = self.content_manager.contents[inner_path][
"modified"]
body = self.storage.read(inner_path)
# Find out my ip and port
tor_manager = self.connection_server.tor_manager
if tor_manager and tor_manager.enabled and tor_manager.start_onions:
my_ip = tor_manager.getOnion(self.address)
if my_ip:
my_ip += ".onion"
my_port = config.fileserver_port
else:
my_ip = config.ip_external
if self.connection_server.port_opened:
my_port = config.fileserver_port
else:
my_port = 0
while 1:
if not peers or len(published) >= limit:
if event_done:
event_done.set(True)
break # All peers done, or published engouht
peer = peers.pop(0)
if peer in peers: # Remove duplicate
peers.remove(peer)
if peer in published:
continue
if peer.last_content_json_update == content_json_modified:
self.log.debug(
"%s already received this update for %s, skipping" %
(peer, inner_path))
continue
if peer.connection and peer.connection.last_ping_delay: # Peer connected
# Timeout: 5sec + size in kb + last_ping
timeout = 5 + int(
file_size / 1024) + peer.connection.last_ping_delay
else: # Peer not connected
# Timeout: 10sec + size in kb
timeout = 10 + int(file_size / 1024)
result = {"exception": "Timeout"}
for retry in range(2):
try:
with gevent.Timeout(timeout, False):
result = peer.request(
"update", {
"site": self.address,
"inner_path": inner_path,
"body": body,
"diffs": diffs,
"peer": (my_ip, my_port)
})
if result:
break
except Exception, err:
self.log.error("Publish error: %s" %
Debug.formatException(err))
result = {"exception": Debug.formatException(err)}
if result and "ok" in result:
published.append(peer)
self.log.info("[OK] %s: %s %s/%s" %
(peer.key, result["ok"], len(published), limit))
else:
if result == {"exception": "Timeout"}:
peer.onConnectionError()
self.log.info("[FAILED] %s: %s" % (peer.key, result))
time.sleep(0.01)
if self.site.storage.isFile(
include_inner_path
): # Content.json exists, load it
success = self.loadContent(include_inner_path,
add_bad_files=add_bad_files)
if success: changed += success # Add changed files
else: # Content.json not exits, add to changed files
self.log.debug("Missing include: %s" %
include_inner_path)
changed += [include_inner_path]
# Update the content
self.contents[content_inner_path] = new_content
except Exception, err:
self.log.error("Content.json parse error: %s" %
Debug.formatException(err))
return False # Content.json parse error
# Add changed files to bad files
if add_bad_files:
for inner_path in changed:
self.site.bad_files[inner_path] = True
if new_content["modified"] > self.site.settings.get("modified", 0):
self.site.settings["modified"] = min(
time.time() + 60 * 10, new_content["modified"]
) # Dont store modifications in the far future (more than 10 minute)
return changed
# Get total size of site
def start(self):
ws = self.ws
if self.site.address == config.homepage and not self.site.page_requested:
# Add open fileserver port message or closed port error to homepage at first request after start
self.site.page_requested = True # Dont add connection notification anymore
file_server = sys.modules["main"].file_server
if file_server.port_opened is None or file_server.tor_manager.start_onions is None:
self.site.page_requested = False # Not ready yet, check next time
elif file_server.port_opened is True:
self.site.notifications.append([
"done",
_["Congratulation, your port <b>{0}</b> is opened.<br>You are full member of ZeroNet network!"]
.format(config.fileserver_port), 10000
])
elif config.tor == "always" and file_server.tor_manager.start_onions:
self.site.notifications.append([
"done",
_(u"""
{_[Tor mode active, every connection using Onion route.]}<br>
{_[Successfully started Tor onion hidden services.]}
"""), 10000
])
elif config.tor == "always" and file_server.tor_manager.start_onions is not False:
self.site.notifications.append([
"error",
_(u"""
{_[Tor mode active, every connection using Onion route.]}<br>
{_[Unable to start hidden services, please check your config.]}
"""), 0
])
elif file_server.port_opened is False and file_server.tor_manager.start_onions:
self.site.notifications.append([
"done",
_(u"""
{_[Successfully started Tor onion hidden services.]}<br>
{_[For faster connections open <b>{0}</b> port on your router.]}
""").format(config.fileserver_port), 10000
])
else:
self.site.notifications.append([
"error",
_(u"""
{_[Your connection is restricted. Please, open <b>{0}</b> port on your router]}<br>
{_[or configure Tor to become full member of ZeroNet network.]}
""").format(config.fileserver_port), 0
])
for notification in self.site.notifications: # Send pending notification messages
self.cmd("notification", notification)
self.site.notifications = []
while True:
try:
message = ws.receive()
except Exception, err:
return "Bye." # Close connection
if message:
try:
self.handleRequest(message)
except Exception, err:
if config.debug: # Allow websocket errors to appear on /Debug
sys.modules["main"].DebugHook.handleError()
self.log.error("WebSocket handleRequest error: %s" %
Debug.formatException(err))
self.cmd(
"error", "Internal error: %s" %
Debug.formatException(err, "html"))
def assemblerGetPathTest():
dic = DictionarySearcher(2)
matrix = QuantumMath.getOracleMatrix(2, [0])
id = AQP.generateSpecial(matrix=matrix, name="asmPath1", dic=dic)
Debug.debug(Visualizer.printCompress(
id,
dic,
), DebugLevel.Result)
Debug.debug(
Visualizer.printCompress(id,
dic,
deepEnd=True,
deep=1,
autoReference=True), DebugLevel.Result)
Debug.debug(
Visualizer.printCompress(id,
dic,
deepEnd=True,
deep=2,
autoReference=True), DebugLevel.Debug)
Debug.debug("\n" + str(dic.getGate(id).getMatrix()) + "\n\n",
DebugLevel.Result)
Debug.debug(WriterAsm.getPath(id, dic), DebugLevel.Result)
dic = DictionarySearcher(2)
matrix = QuantumMath.getAmplitudAmplifier(2)
id = AQP.generateSpecial(matrix=matrix, name="asmPath2", dic=dic)
Debug.debug(Visualizer.printCompress(
id,
dic,
), DebugLevel.Result)
Debug.debug(
Visualizer.printCompress(id,
dic,
deepEnd=True,
deep=1,
autoReference=True), DebugLevel.Result)
Debug.debug(
Visualizer.printCompress(id,
dic,
deepEnd=True,
deep=2,
autoReference=True), DebugLevel.Debug)
Debug.debug("\n" + str(dic.getGate(id).getMatrix()) + "\n\n",
DebugLevel.Result)
Debug.debug(WriterAsm.getPath(id, dic), DebugLevel.Result)
def connect(self, connection = None):
if not self.log: self.log = logging.getLogger("Peer:%s:%s %s" % (self.ip, self.port, self.site))
if self.connection:
self.log.debug("Getting connection (Closing %s)..." % self.connection)
self.connection.close()
else:
self.log.debug("Getting connection...")
if connection: # Connection specificed
self.connection = connection
else: # Try to find from connection pool or create new connection
self.connection = None
try:
self.connection = self.connection_server.getConnection(self.ip, self.port)
except Exception, err:
self.onConnectionError()
self.log.debug("Getting connection error: %s (connection_error: %s, hash_failed: %s)" % (Debug.formatException(err), self.connection_error, self.hash_failed))
self.connection = None
def basicTurnGateTest():
dic = DictionarySearcher(2)
id = AQP.generateBasicTurn(target=1,
angle=math.pi,
turnType=TurnType.Z,
dic=dic)
Debug.debug(Visualizer.printPath(id, dic), DebugLevel.Result)
Debug.debug(Visualizer.printCompress(id, dic), DebugLevel.Result)
Debug.debug("\n" + str(dic.getGate(id).getMatrix()) + "\n\n",
DebugLevel.Result)
id = AQP.generateBasicTurn(target=0,
angle=math.pi / 2,
turnType=TurnType.Z,
dic=dic)
Debug.debug(Visualizer.printPath(id, dic), DebugLevel.Result)
Debug.debug(Visualizer.printCompress(id, dic), DebugLevel.Result)
Debug.debug("\n" + str(dic.getGate(id).getMatrix()) + "\n\n",
DebugLevel.Result)
id = AQP.generateBasicTurn(target=1,
angle=math.pi / 4,
turnType=TurnType.X,
dic=dic)
Debug.debug(Visualizer.printPath(id, dic), DebugLevel.Result)
Debug.debug(Visualizer.printCompress(id, dic), DebugLevel.Result)
Debug.debug("\n" + str(dic.getGate(id).getMatrix()) + "\n\n",
DebugLevel.Result)
dic = DictionarySearcher(4)
id = AQP.generateBasicTurn(target=3,
angle=math.pi / 4,
turnType=TurnType.Z,
dic=dic)
Debug.debug(Visualizer.printPath(id, dic), DebugLevel.Result)
Debug.debug(Visualizer.printCompress(id, dic), DebugLevel.Result)
Debug.debug("\n" + str(dic.getGate(id).getMatrix()) + "\n\n",
DebugLevel.Result)
id = AQP.generateBasicTurn(target=1,
angle=math.pi / 4,
turnType=TurnType.Z,
dic=dic)
Debug.debug(Visualizer.printPath(id, dic), DebugLevel.Result)
Debug.debug(Visualizer.printCompress(id, dic), DebugLevel.Result)
Debug.debug("\n" + str(dic.getGate(id).getMatrix()) + "\n\n",
DebugLevel.Result)
def codingDecodingTest():
aux = QuantumMath.codingQubits([0, 1, 2])
Debug.debug(aux, DebugLevel.Result)
Debug.debug(QuantumMath.decodingQubits(aux), DebugLevel.Result)
aux = QuantumMath.codingQubits([1, 4])
Debug.debug(aux, DebugLevel.Result)
Debug.debug(QuantumMath.decodingQubits(aux), DebugLevel.Result)
aux = QuantumMath.codingQubits([6])
Debug.debug(aux, DebugLevel.Result)
Debug.debug(QuantumMath.decodingQubits(aux), DebugLevel.Result)
def actionFeedSearch(self, to, search, limit=30, day_limit=30):
if "ADMIN" not in self.site.settings["permissions"]:
return self.response(to, "FeedSearch not allowed")
from Site import SiteManager
rows = []
stats = []
num_sites = 0
total_s = time.time()
search_text, filters = self.parseSearch(search)
for address, site in SiteManager.site_manager.list().iteritems():
if not site.storage.has_db:
continue
if "site" in filters:
if filters["site"].lower() not in [
site.address,
site.content_manager.contents["content.json"].get(
"title").lower()
]:
continue
if site.storage.db: # Database loaded
feeds = site.storage.db.schema.get("feeds")
else:
try:
feeds = site.storage.loadJson("dbschema.json").get("feeds")
except:
continue
if not feeds:
continue
num_sites += 1
for name, query in feeds.iteritems():
s = time.time()
try:
db_query = DbQuery(query)
params = []
# Filters
if search_text:
db_query.wheres.append("(%s LIKE ? OR %s LIKE ?)" %
(db_query.fields["body"],
db_query.fields["title"]))
search_like = "%" + search_text.replace(" ", "%") + "%"
params.append(search_like)
params.append(search_like)
if filters.get("type") and filters["type"] not in query:
continue
if day_limit:
db_query.wheres.append(
"%s > strftime('%%s', 'now', '-%s day')" %
(db_query.fields.get("date_added",
"date_added"), day_limit))
# Order
db_query.parts["ORDER BY"] = "date_added DESC"
db_query.parts["LIMIT"] = str(limit)
res = site.storage.query(str(db_query), params)
except Exception, err:
self.log.error("%s feed query %s error: %s" %
(address, name, Debug.formatException(err)))
stats.append({
"site": site.address,
"feed_name": name,
"error": str(err),
"query": query
})
continue
for row in res:
row = dict(row)
if row["date_added"] > time.time() + 120:
continue # Feed item is in the future, skip it
row["site"] = address
row["feed_name"] = name
rows.append(row)
stats.append({
"site": site.address,
"feed_name": name,
"taken": round(time.time() - s, 3)
})
include_inner_path = content_inner_dir + relative_dir + "/content.json"
if not self.site.storage.isFile(include_inner_path):
continue # Content.json not exist
include_changed, include_deleted = self.loadContent(
include_inner_path, add_bad_files=add_bad_files, delete_removed_files=delete_removed_files,
load_includes=False
)
if include_changed:
changed += include_changed # Add changed files
if include_deleted:
deleted += include_deleted # Add changed files
# Update the content
self.contents[content_inner_path] = new_content
except Exception, err:
self.log.error("Content.json parse error: %s" % Debug.formatException(err))
return [], [] # Content.json parse error
# Add changed files to bad files
if add_bad_files:
for inner_path in changed:
self.site.bad_files[inner_path] = self.site.bad_files.get(inner_path, 0) + 1
if new_content["modified"] > self.site.settings.get("modified", 0):
# Dont store modifications in the far future (more than 10 minute)
self.site.settings["modified"] = min(time.time() + 60 * 10, new_content["modified"])
return changed, deleted
# Get total size of site
# Return: 32819 (size of files in kb)
def multipleTurnGateTest():
dic = DictionarySearcher(2)
id = AQP.generateMultipleTurn(sources=[0],
target=1,
angle=math.pi,
turnType=TurnType.Z,
dic=dic)
Debug.debug(Visualizer.printPath(id, dic, deepEnd=False),
DebugLevel.Result)
Debug.debug(Visualizer.printCompress(id, dic), DebugLevel.Result)
Debug.debug(
Visualizer.printCompress(id, dic, deepEnd=False, autoReference=True),
DebugLevel.Result)
Debug.debug("\n" + str(dic.getGate(id).getMatrix()) + "\n\n",
DebugLevel.Result)
dic = DictionarySearcher(3)
id = AQP.generateMultipleTurn(sources=[0, 1],
target=2,
angle=math.pi,
turnType=TurnType.Z,
dic=dic)
Debug.debug(Visualizer.printPath(id, dic, deepEnd=False),
DebugLevel.Result)
Debug.debug(Visualizer.printCompress(id, dic), DebugLevel.Result)
Debug.debug(
Visualizer.printCompress(id,
dic,
deepEnd=True,
deep=1,
autoReference=True), DebugLevel.Result)
Debug.debug("\n" + str(dic.getGate(id).getMatrix()) + "\n\n",
DebugLevel.Result)
id = AQP.generateMultipleTurn(sources=[0, 1],
target=2,
angle=math.pi / 4,
turnType=TurnType.X,
dic=dic)
Debug.debug(Visualizer.printPath(id, dic, deepEnd=False),
DebugLevel.Result)
Debug.debug(Visualizer.printCompress(id, dic), DebugLevel.Result)
Debug.debug("\n" + str(dic.getGate(id).getMatrix()) + "\n\n",
DebugLevel.Result)
id = AQP.generateMultipleTurn(sources=[0, 2],
target=1,
angle=math.pi / 4,
turnType=TurnType.X,
dic=dic)
Debug.debug(Visualizer.printPath(id, dic, deepEnd=False),
DebugLevel.Result)
Debug.debug(Visualizer.printCompress(id, dic), DebugLevel.Result)
Debug.debug(
Visualizer.printCompress(id,
dic,
deepEnd=False,
deep=1,
autoReference=True), DebugLevel.Result)
Debug.debug("\n" + str(dic.getGate(id).getMatrix()) + "\n\n",
DebugLevel.Result)
Debug.debug("\n" + str(dic) + "\n\n", DebugLevel.Result)
dic = DictionarySearcher(4)
id = AQP.generateMultipleTurn(sources=[0, 1, 2],
target=3,
angle=math.pi / 2,
turnType=TurnType.X,
dic=dic)
Debug.debug(Visualizer.printPath(id, dic, deepEnd=False),
DebugLevel.Result)
Debug.debug(Visualizer.printCompress(id, dic), DebugLevel.Result)
Debug.debug(
Visualizer.printCompress(id,
dic,
deepEnd=True,
deep=1,
autoReference=True), DebugLevel.Result)
Debug.debug("\n" + str(dic.getGate(id).getMatrix()) + "\n\n",
DebugLevel.Result)
id = AQP.generateMultipleTurn(sources=[0, 1, 2],
target=3,
angle=math.pi,
turnType=TurnType.Z,
dic=dic)
Debug.debug(Visualizer.printPath(id, dic, deepEnd=False),
DebugLevel.Result)
Debug.debug(Visualizer.printCompress(id, dic), DebugLevel.Result)
Debug.debug("\n" + str(dic.getGate(id).getMatrix()) + "\n\n",
DebugLevel.Result)
id = AQP.generateMultipleTurn(sources=[2],
target=0,
angle=math.pi / 4,
turnType=TurnType.X,
dic=dic)
Debug.debug(Visualizer.printPath(id, dic, deepEnd=False),
DebugLevel.Result)
Debug.debug(Visualizer.printCompress(id, dic), DebugLevel.Result)
Debug.debug("\n" + str(dic.getGate(id).getMatrix()) + "\n\n",
DebugLevel.Result)
def specialTurnGateTest():
dic = DictionarySearcher(2)
id = AQP.generateSpecialTurn(a=-1, b=0, dic=dic)
Debug.debug(Visualizer.printPath(id, dic, deepEnd=False),
DebugLevel.Result)
Debug.debug(Visualizer.printCompress(id, dic), DebugLevel.Result)
Debug.debug("\n" + str(dic.getGate(id).getMatrix()) + "\n\n",
DebugLevel.Result)
id = AQP.generateSpecialTurn(a=(1 / (math.sqrt(2))),
b=(1 / (math.sqrt(2))),
dic=dic)
Debug.debug(Visualizer.printPath(id, dic, deepEnd=False),
DebugLevel.Result)
Debug.debug(Visualizer.printCompress(id, dic), DebugLevel.Result)
Debug.debug("\n" + str(dic.getGate(id).getMatrix()) + "\n\n",
DebugLevel.Result)
dic = DictionarySearcher(3)
id = AQP.generateSpecialTurn(a=1j, b=0, dic=dic)
Debug.debug(Visualizer.printPath(id, dic, deepEnd=False),
DebugLevel.Result)
Debug.debug(Visualizer.printCompress(id, dic), DebugLevel.Result)
Debug.debug(
Visualizer.printCompress(id,
dic,
deepEnd=True,
deep=1,
autoReference=True), DebugLevel.Result)
Debug.debug("\n" + str(dic.getGate(id).getMatrix()) + "\n\n",
DebugLevel.Result)
dic = DictionarySearcher(4)
id = AQP.generateSpecialTurn(a=(1 / (math.sqrt(2))),
b=(1j * (1 / (math.sqrt(2)))),
dic=dic)
Debug.debug(Visualizer.printPath(id, dic, deepEnd=False),
DebugLevel.Result)
Debug.debug(Visualizer.printCompress(id, dic), DebugLevel.Result)
Debug.debug(
Visualizer.printCompress(id,
dic,
deepEnd=True,
deep=1,
autoReference=True), DebugLevel.Result)
Debug.debug("\n" + str(dic.getGate(id).getMatrix()) + "\n\n",
DebugLevel.Result)
def toffoliGateTest():
dic = DictionarySearcher(3)
id = AQP.generateToffoli(target=2, sources=[0, 1], dic=dic)
Debug.debug(Visualizer.printPath(id, dic, deepEnd=False),
DebugLevel.Result)
Debug.debug(Visualizer.printCompress(id, dic), DebugLevel.Result)
Debug.debug(
"\n" + Visualizer.printIntegerMatrix(dic.getGate(id).getMatrix()) +
"\n\n", DebugLevel.Result)
id = AQP.generateToffoli(target=1, sources=[0, 2], dic=dic)
Debug.debug(Visualizer.printPath(id, dic, deepEnd=False),
DebugLevel.Result)
Debug.debug(Visualizer.printCompress(id, dic), DebugLevel.Result)
Debug.debug(
"\n" + Visualizer.printIntegerMatrix(dic.getGate(id).getMatrix()) +
"\n\n", DebugLevel.Result)
dic = DictionarySearcher(4)
id = AQP.generateToffoli(target=3, sources=[0, 1, 2], dic=dic)
Debug.debug(Visualizer.printPath(id, dic, deepEnd=False),
DebugLevel.Result)
Debug.debug(Visualizer.printCompress(id, dic), DebugLevel.Result)
Debug.debug(
Visualizer.printCompress(id,
dic,
deepEnd=True,
deep=2,
autoReference=True), DebugLevel.Result)
Debug.debug("\n" + str(dic.getGate(id).getMatrix()) + "\n\n",
DebugLevel.Result)
Debug.debug(
"\n" + Visualizer.printIntegerMatrix(dic.getGate(id).getMatrix()) +
"\n\n", DebugLevel.Result)
id = AQP.generateToffoli(target=3, sources=[0, 2], dic=dic)
Debug.debug(Visualizer.printPath(id, dic, deepEnd=False),
DebugLevel.Result)
Debug.debug(Visualizer.printCompress(id, dic), DebugLevel.Result)
Debug.debug(
"\n" + Visualizer.printIntegerMatrix(dic.getGate(id).getMatrix()) +
"\n\n", DebugLevel.Result)
def verifyFile(self, inner_path, file, ignore_same=True):
if inner_path.endswith("content.json"): # content.json: Check using sign
from Crypt import CryptBitcoin
try:
new_content = json.load(file)
if inner_path in self.contents:
old_content = self.contents.get(inner_path)
# Checks if its newer the ours
if old_content["modified"] == new_content["modified"] and ignore_same: # Ignore, have the same content.json
return None
elif old_content["modified"] > new_content["modified"]: # We have newer
self.log.debug(
"We have newer %s (Our: %s, Sent: %s)" %
(inner_path, old_content["modified"], new_content["modified"])
)
gevent.spawn(self.site.publish, inner_path=inner_path) # Try to fix the broken peers
return False
if new_content["modified"] > time.time() + 60 * 60 * 24: # Content modified in the far future (allow 1 day+)
self.log.error("%s modify is in the future!" % inner_path)
return False
# Check sign
sign = new_content.get("sign")
signs = new_content.get("signs", {})
if "sign" in new_content:
del(new_content["sign"]) # The file signed without the sign
if "signs" in new_content:
del(new_content["signs"]) # The file signed without the signs
sign_content = json.dumps(new_content, sort_keys=True) # Dump the json to string to remove whitepsace
if not self.verifyContent(inner_path, new_content):
return False # Content not valid (files too large, invalid files)
if signs: # New style signing
valid_signers = self.getValidSigners(inner_path, new_content)
signs_required = self.getSignsRequired(inner_path, new_content)
if inner_path == "content.json" and len(valid_signers) > 1: # Check signers_sign on root content.json
if not CryptBitcoin.verify(
"%s:%s" % (signs_required, ",".join(valid_signers)), self.site.address, new_content["signers_sign"]
):
self.log.error("%s invalid signers_sign!" % inner_path)
return False
if inner_path != "content.json" and not self.verifyCert(inner_path, new_content): # Check if cert valid
self.log.error("%s invalid cert!" % inner_path)
return False
valid_signs = 0
for address in valid_signers:
if address in signs:
valid_signs += CryptBitcoin.verify(sign_content, address, signs[address])
if valid_signs >= signs_required:
break # Break if we has enough signs
self.log.debug("%s: Valid signs: %s/%s" % (inner_path, valid_signs, signs_required))
return valid_signs >= signs_required
else: # Old style signing
return CryptBitcoin.verify(sign_content, self.site.address, sign)
except Exception, err:
self.log.error("Verify sign error: %s" % Debug.formatException(err))
return False
self.log.info("Importing data...")
try:
if len(db_files) > 100:
self.site.messageWebsocket(
_["Database rebuilding...<br>Imported {0} of {1} files..."]
.format("0000", len(db_files)), "rebuild", 0)
for file_inner_path, file_path in db_files:
try:
if self.updateDbFile(file_inner_path,
file=open(file_path, "rb"),
cur=cur):
found += 1
except Exception, err:
self.log.error(
"Error importing %s: %s" %
(file_inner_path, Debug.formatException(err)))
if found and found % 100 == 0:
self.site.messageWebsocket(
_["Database rebuilding...<br>Imported {0} of {1} files..."]
.format(found, len(db_files)), "rebuild",
int(float(found) / len(db_files) * 100))
time.sleep(0.000001) # Context switch to avoid UI block
finally:
cur.execute("END")
if len(db_files) > 100:
self.site.messageWebsocket(
_["Database rebuilding...<br>Imported {0} of {1} files..."]
.format(found, len(db_files)), "rebuild", 100)
self.log.info("Imported %s data file in %ss" %
(found, time.time() - s))
def basicGateTest():
dic = DictionarySearcher(2)
id = AQP.generateBasic(standarGate=StandarGate.X, targets=[0], dic=dic)
Debug.debug(Visualizer.printPath(id, dic), DebugLevel.Result)
Debug.debug(Visualizer.printCompress(id, dic), DebugLevel.Result)
Debug.debug("\n" + str(dic.getGate(id).getMatrix()) + "\n\n",
DebugLevel.Result)
id = AQP.generateBasic(standarGate=StandarGate.X, targets=[0, 1], dic=dic)
Debug.debug(Visualizer.printPath(id, dic), DebugLevel.Result)
Debug.debug(Visualizer.printCompress(id, dic), DebugLevel.Result)
Debug.debug("\n" + str(dic.getGate(id).getMatrix()) + "\n\n",
DebugLevel.Result)
id = AQP.generateBasic(standarGate=StandarGate.H, targets=[0, 1], dic=dic)
Debug.debug(Visualizer.printPath(id, dic), DebugLevel.Result)
Debug.debug(Visualizer.printCompress(id, dic), DebugLevel.Result)
Debug.debug("\n" + str(dic.getGate(id).getMatrix()) + "\n\n",
DebugLevel.Result)
dic = DictionarySearcher(4)
id = AQP.generateBasic(standarGate=StandarGate.X,
targets=[1, 2, 3],
dic=dic)
Debug.debug(Visualizer.printPath(id, dic), DebugLevel.Result)
Debug.debug(Visualizer.printCompress(id, dic), DebugLevel.Result)
Debug.debug("\n" + str(dic.getGate(id).getMatrix()) + "\n\n",
DebugLevel.Result)
id = AQP.generateBasic(standarGate=StandarGate.H, targets=[2], dic=dic)
Debug.debug(Visualizer.printPath(id, dic), DebugLevel.Result)
Debug.debug(Visualizer.printCompress(id, dic), DebugLevel.Result)
Debug.debug("\n" + str(dic.getGate(id).getMatrix()) + "\n\n",
DebugLevel.Result)
def actionSiteSign(self,
to,
privatekey=None,
inner_path="content.json",
remove_missing_optional=False,
update_changed_files=False,
response_ok=True):
self.log.debug("Signing: %s" % inner_path)
site = self.site
extend = {} # Extended info for signing
# Change to the file's content.json
file_info = site.content_manager.getFileInfo(inner_path)
if not inner_path.endswith("content.json"):
if not file_info:
raise Exception("Invalid content.json file: %s" % inner_path)
inner_path = file_info["content_inner_path"]
# Add certificate to user files
if file_info and "cert_signers" in file_info and privatekey is None:
cert = self.user.getCert(self.site.address)
extend["cert_auth_type"] = cert["auth_type"]
extend["cert_user_id"] = self.user.getCertUserId(site.address)
extend["cert_sign"] = cert["cert_sign"]
if not self.hasFilePermission(inner_path):
self.log.error(
"SiteSign error: you don't own this site & site owner doesn't allow you to do so."
)
return self.response(
to, {"error": "Forbidden, you can only modify your own sites"})
if privatekey == "stored": # Get privatekey from sites.json
privatekey = self.user.getSiteData(
self.site.address).get("privatekey")
if not privatekey: # Get privatekey from users.json auth_address
privatekey = self.user.getAuthPrivatekey(self.site.address)
# Signing
# Reload content.json, ignore errors to make it up-to-date
site.content_manager.loadContent(inner_path,
add_bad_files=False,
force=True)
# Sign using private key sent by user
try:
site.content_manager.sign(
inner_path,
privatekey,
extend=extend,
update_changed_files=update_changed_files,
remove_missing_optional=remove_missing_optional)
except (VerifyError, SignError) as err:
self.cmd("notification", [
"error",
_["Content signing failed"] + "<br><small>%s</small>" % err
])
self.response(to, {"error": "Site sign failed: %s" % err})
self.log.error("Site sign failed: %s: %s" %
(inner_path, Debug.formatException(err)))
return
except Exception as err:
self.cmd("notification", [
"error", _["Content signing error"] +
"<br><small>%s</small>" % Debug.formatException(err)
])
self.response(
to,
{"error": "Site sign error: %s" % Debug.formatException(err)})
self.log.error("Site sign error: %s: %s" %
(inner_path, Debug.formatException(err)))
return
site.content_manager.loadContent(
inner_path,
add_bad_files=False) # Load new content.json, ignore errors
if update_changed_files:
self.site.updateWebsocket(file_done=inner_path)
if response_ok:
self.response(to, "ok")
else:
return inner_path
if read_bytes <= 0:
break
buff = self.sock.recv(16 * 1024)
if not buff:
break
buff_len = len(buff)
read_bytes -= buff_len
file.write(buff)
# Statistics
self.last_recv_time = time.time()
self.incomplete_buff_recv += 1
self.bytes_recv += buff_len
self.server.bytes_recv += buff_len
except Exception, err:
self.log("Stream read error: %s" % Debug.formatException(err))
if config.debug_socket:
self.log("End stream %s" % message["to"])
self.incomplete_buff_recv = 0
self.waiting_requests[message["to"]].set(
message) # Set the response to event
del self.waiting_streams[message["to"]]
del self.waiting_requests[message["to"]]
# Send data to connection
def send(self, message, streaming=False):
if config.debug_socket:
self.log(
"Send: %s, to: %s, streaming: %s, site: %s, inner_path: %s, req_id: %s"
if include_changed:
changed += include_changed # Add changed files
if include_deleted:
deleted += include_deleted # Add changed files
# Save some memory
new_content["signs"] = None
if "cert_sign" in new_content:
new_content["cert_sign"] = None
if new_content.get("files_optional"):
self.has_optional_files = True
# Update the content
self.contents[content_inner_path] = new_content
except Exception, err:
self.log.warning("%s parse error: %s" % (content_inner_path, Debug.formatException(err)))
return [], [] # Content.json parse error
# Add changed files to bad files
if add_bad_files:
for inner_path in changed:
self.site.bad_files[inner_path] = self.site.bad_files.get(inner_path, 0) + 1
for inner_path in deleted:
if inner_path in self.site.bad_files:
del self.site.bad_files[inner_path]
if new_content.get("modified", 0) > self.site.settings.get("modified", 0):
# Dont store modifications in the far future (more than 10 minute)
self.site.settings["modified"] = min(time.time() + 60 * 10, new_content["modified"])
return changed, deleted
def messageLoop(self):
if not self.sock:
self.log("Socket error: No socket found")
return False
self.protocol = "v2"
self.updateName()
self.connected = True
buff_len = 0
req_len = 0
unpacker_bytes = 0
try:
while not self.closed:
buff = self.sock.recv(64 * 1024)
if not buff:
break # Connection closed
buff_len = len(buff)
# Statistics
self.last_recv_time = time.time()
self.incomplete_buff_recv += 1
self.bytes_recv += buff_len
self.server.bytes_recv += buff_len
req_len += buff_len
if not self.unpacker:
self.unpacker = msgpack.fallback.Unpacker()
unpacker_bytes = 0
self.unpacker.feed(buff)
unpacker_bytes += buff_len
for message in self.unpacker:
if not type(message) is dict:
raise Exception(
"Invalid message type: %s, content: %r, buffer: %r"
% (type(message), message, buff[0:16]))
# Stats
self.incomplete_buff_recv = 0
stat_key = message.get("cmd", "unknown")
if stat_key == "response" and "to" in message:
cmd_sent = self.waiting_requests.get(
message["to"], {"cmd": "unknown"})["cmd"]
stat_key = "response: %s" % cmd_sent
if stat_key == "update":
stat_key = "update: %s" % message["params"]["site"]
self.server.stat_recv[stat_key]["bytes"] += req_len
self.server.stat_recv[stat_key]["num"] += 1
if "stream_bytes" in message:
self.server.stat_recv[stat_key]["bytes"] += message[
"stream_bytes"]
req_len = 0
# Handle message
if "stream_bytes" in message:
buff_left = self.handleStream(message, self.unpacker,
buff, unpacker_bytes)
self.unpacker = msgpack.fallback.Unpacker()
self.unpacker.feed(buff_left)
unpacker_bytes = len(buff_left)
if config.debug_socket:
self.log("Start new unpacker with buff_left: %r" %
buff_left)
break
else:
self.handleMessage(message)
message = None
except Exception as err:
if not self.closed:
self.log("Socket error: %s" % Debug.formatException(err))
self.server.stat_recv["error: %s" % err]["bytes"] += req_len
self.server.stat_recv["error: %s" % err]["num"] += 1
self.close("MessageLoop ended (closed: %s)" %
self.closed) # MessageLoop ended, close connection
class UiWebsocket(object):
def __init__(self, ws, site, server, user, request):
self.ws = ws
self.site = site
self.user = user
self.log = site.log
self.request = request
self.permissions = []
self.server = server
self.next_message_id = 1
self.waiting_cb = {
} # Waiting for callback. Key: message_id, Value: function pointer
self.channels = [] # Channels joined to
self.state = {"sending": False} # Shared state of websocket connection
self.send_queue = [] # Messages to send to client
self.admin_commands = ("sitePause", "siteResume", "siteDelete",
"siteList", "siteSetLimit",
"channelJoinAllsite", "serverUpdate",
"serverPortcheck", "serverShutdown",
"serverShowdirectory", "certSet", "configSet",
"permissionAdd", "permissionRemove")
self.async_commands = ("fileGet", "fileList", "dirList", "fileNeed")
# Start listener loop
def start(self):
ws = self.ws
if self.site.address == config.homepage and not self.site.page_requested:
# Add open fileserver port message or closed port error to homepage at first request after start
self.site.page_requested = True # Dont add connection notification anymore
file_server = sys.modules["main"].file_server
if file_server.port_opened is None or file_server.tor_manager.start_onions is None:
self.site.page_requested = False # Not ready yet, check next time
else:
try:
self.addHomepageNotifications()
except Exception, err:
self.log.error("Uncaught Exception: " +
Debug.formatException(err))
for notification in self.site.notifications: # Send pending notification messages
# send via WebSocket
self.cmd("notification", notification)
# just in case, log them to terminal
if notification[0] == "error":
self.log.error("\n*** %s\n" % self.dedent(notification[1]))
self.site.notifications = []
while True:
try:
if ws.closed:
break
else:
message = ws.receive()
except Exception, err:
self.log.error("WebSocket receive error: %s" %
Debug.formatException(err))
break
if message:
try:
req = json.loads(message)
self.handleRequest(req)
except Exception, err:
if config.debug: # Allow websocket errors to appear on /Debug
sys.modules["main"].DebugHook.handleError()
self.log.error("WebSocket handleRequest error: %s \n %s" %
(Debug.formatException(err), message))
if not self.hasPlugin("Multiuser"):
self.cmd(
"error", "Internal error: %s" %
Debug.formatException(err, "html"))
def groverMatricesTest():
Debug.debug(QuantumMath.getOracleMatrix(2, [3]), DebugLevel.Result)
Debug.debug(QuantumMath.getOracleMatrix(3, [3, 4, 5]), DebugLevel.Result)
matrix = QuantumMath.getAmplitudAmplifier(2)
Debug.debug(matrix, DebugLevel.Result)
Debug.debug(QuantumMath.matrixDeterminant(matrix), DebugLevel.Info)
matrix = QuantumMath.getAmplitudAmplifier(3)
Debug.debug(matrix, DebugLevel.Result)
Debug.debug(QuantumMath.matrixDeterminant(matrix), DebugLevel.Info)
matrix = QuantumMath.getAmplitudAmplifier(4)
Debug.debug(matrix, DebugLevel.Result)
Debug.debug(QuantumMath.matrixDeterminant(matrix), DebugLevel.Info)
def __init__(self, data, code, configuration):
Utility.__init__(self)
Debug.__init__(self)
self.data = data
self.code = code
self.configuration = configuration
def actionSidebarGetPeers(self, to):
permissions = self.getPermissions(to)
if "ADMIN" not in permissions:
return self.response(
to, "You don't have permission to run this command")
try:
import maxminddb
db_path = config.data_dir + '/GeoLite2-City.mmdb'
if not os.path.isfile(db_path) or os.path.getsize(db_path) == 0:
if not self.downloadGeoLiteDb(db_path):
return False
geodb = maxminddb.open_database(db_path)
peers = self.site.peers.values()
# Find avg ping
ping_times = [
peer.connection.last_ping_delay for peer in peers
if peer.connection and peer.connection.last_ping_delay
and peer.connection.last_ping_delay
]
if ping_times:
ping_avg = sum(ping_times) / float(len(ping_times))
else:
ping_avg = 0
# Place bars
globe_data = []
placed = {} # Already placed bars here
for peer in peers:
# Height of bar
if peer.connection and peer.connection.last_ping_delay:
ping = min(
0.20,
math.log(
1 + peer.connection.last_ping_delay / ping_avg,
300))
else:
ping = -0.03
# Query and cache location
if peer.ip in loc_cache:
loc = loc_cache[peer.ip]
else:
try:
loc = geodb.get(peer.ip)
except:
loc = None
loc_cache[peer.ip] = loc
if not loc or "location" not in loc:
continue
# Create position array
lat, lon = (loc["location"]["latitude"],
loc["location"]["longitude"])
latlon = "%s,%s" % (lat, lon)
if latlon in placed: # Dont place more than 1 bar to same place, fake repos using ip address last two part
lat += float(128 - int(peer.ip.split(".")[-2])) / 50
lon += float(128 - int(peer.ip.split(".")[-1])) / 50
latlon = "%s,%s" % (lat, lon)
placed[latlon] = True
globe_data += (lat, lon, ping)
# Append myself
loc = geodb.get(config.ip_external)
if loc:
lat, lon = (loc["location"]["latitude"],
loc["location"]["longitude"])
globe_data += (lat, lon, -0.135)
self.response(to, globe_data)
except Exception, err:
self.log.debug("sidebarGetPeers error: %s" %
Debug.formatException(err))
self.response(to, {"error": err})
def handleGetFile(self, params, streaming=False):
site = self.sites.get(params["site"])
if not site or not site.settings[
"serving"]: # Site unknown or not serving
self.response({"error": "Unknown site"})
return False
try:
file_path = site.storage.getPath(params["inner_path"])
if streaming:
file_obj = site.storage.open(params["inner_path"])
else:
file_obj = StreamingMsgpack.FilePart(file_path, "rb")
with file_obj as file:
file.seek(params["location"])
read_bytes = params.get("read_bytes", FILE_BUFF)
file_size = os.fstat(file.fileno()).st_size
if file_size > read_bytes: # Check if file is readable at current position (for big files)
if not self.isReadable(site, params["inner_path"], file,
params["location"]):
raise RequestError(
"File not readable at position: %s" %
params["location"])
if not streaming:
file.read_bytes = read_bytes
if params.get(
"file_size") and params["file_size"] != file_size:
self.connection.badAction(2)
raise RequestError("File size does not match: %sB != %sB" %
(params["file_size"], file_size))
if params["location"] > file_size:
self.connection.badAction(5)
raise RequestError("Bad file location")
if streaming:
back = {
"size":
file_size,
"location":
min(file.tell() + read_bytes, file_size),
"stream_bytes":
min(read_bytes, file_size - params["location"])
}
self.response(back)
self.sendRawfile(file, read_bytes=read_bytes)
else:
back = {
"body": file,
"size": file_size,
"location": min(file.tell() + file.read_bytes,
file_size)
}
self.response(back, streaming=True)
bytes_sent = min(
read_bytes, file_size -
params["location"]) # Number of bytes we going to send
site.settings["bytes_sent"] = site.settings.get(
"bytes_sent", 0) + bytes_sent
if config.debug_socket:
self.log.debug("File %s at position %s sent %s bytes" %
(file_path, params["location"], bytes_sent))
# Add peer to site if not added before
connected_peer = site.addPeer(self.connection.ip,
self.connection.port)
if connected_peer: # Just added
connected_peer.connect(
self.connection) # Assign current connection to peer
return {
"bytes_sent": bytes_sent,
"file_size": file_size,
"location": params["location"]
}
except RequestError, err:
self.log.debug("GetFile %s %s request error: %s" %
(self.connection, params["inner_path"],
Debug.formatException(err)))
self.response({"error": "File read error: %s" % err})
