def command(): # get <domainTXT/domain:port/ip:port> [-l <logpath>] [-n <name>] [-p] [-f] [-d] args = parser.parse_args() default_logpath = os.path.join(defaults.dir, 'logs', args.name) logpath = args.logpath or default_logpath if logpath != '-': create_logdir(logpath, default_logpath, args.force) ip, *port = args.addr.split(':') try: ip = str(ipaddress.ip_address(ip)) except ValueError: nslookup = """nslookup -q=txt %s|grep 'text =' | sed -E 's/.*text = "([^"]+)".*/\\1/'""" % ip lookup = subprocess.Popen(nslookup, shell=True, stdout=subprocess.PIPE) lookup = lookup.stdout.read().decode().strip() if lookup: ip, *port = lookup.split(':') else: nslookup = "nslookup %s -q=a | grep Address|tail -n1 | awk {'print $2'}" % ip lookup = subprocess.Popen(nslookup, shell=True, stdout=subprocess.PIPE) lookup = lookup.stdout.read().decode().strip() if not '#' in lookup: ip = lookup if not port: port = get_port(args.name) + 2 else: port = int(port[0]) sys.stderr.write("Connecting to %s:%i\n" % (ip, port)) received = False if logpath == '-': for data in utils.netcat(ip, port, b'cGET'): sys.stdout.write(data) received = True else: try: with open(logpath, 'w') as handler: for data in utils.netcat(ip, port, b'cGET'): handler.write(data) received = True except ConnectionRefusedError: os.remove(logpath) sys.stderr.write("Error: connection refused, bad port?\n") sys.exit(3) if not received: sys.stderr.write("Error: nothing has been received, bad port?\n") sys.exit(2) elif logpath != '-': log = Log(logpath) try: log.load(validate=True) except exceptions.ValidationError as e: sys.stderr.write(str(e) + '\n') sys.stdout.write("%s log created\n" % logpath)
def command(): # get <domainTXT/domain:port/ip:port> [-l <logpath>] [-n <name>] [-p] [-f] [-d] args = parser.parse_args() default_logpath = os.path.join(defaults.dir, "logs", args.name) logpath = args.logpath or default_logpath if logpath != "-": create_logdir(logpath, default_logpath, args.force) ip, *port = args.addr.split(":") try: ip = str(ipaddress.ip_address(ip)) except ValueError: nslookup = """nslookup -q=txt %s|grep 'text =' | sed -E 's/.*text = "([^"]+)".*/\\1/'""" % ip lookup = subprocess.Popen(nslookup, shell=True, stdout=subprocess.PIPE) lookup = lookup.stdout.read().decode().strip() if lookup: ip, *port = lookup.split(":") else: nslookup = "nslookup %s -q=a | grep Address|tail -n1 | awk {'print $2'}" % ip lookup = subprocess.Popen(nslookup, shell=True, stdout=subprocess.PIPE) lookup = lookup.stdout.read().decode().strip() if not "#" in lookup: ip = lookup if not port: port = get_port(args.name) + 2 else: port = int(port[0]) sys.stderr.write("Connecting to %s:%i\n" % (ip, port)) received = False if logpath == "-": for data in utils.netcat(ip, port, b"cGET"): sys.stdout.write(data) received = True else: try: with open(logpath, "w") as handler: for data in utils.netcat(ip, port, b"cGET"): handler.write(data) received = True except ConnectionRefusedError: os.remove(logpath) sys.stderr.write("Error: connection refused, bad port?\n") sys.exit(3) if not received: sys.stderr.write("Error: nothing has been received, bad port?\n") sys.exit(2) elif logpath != "-": log = Log(logpath) try: log.load(validate=True) except exceptions.ValidationError as e: sys.stderr.write(str(e) + "\n") sys.stdout.write("%s log created\n" % logpath)
class HandlerTests(unittest.TestCase): def init(self): __, self.logpath = tempfile.mkstemp() __, self.logpath_b = tempfile.mkstemp() self.addCleanup(os.remove, self.logpath) self.addCleanup(os.remove, self.logpath_b) self.log, self.root_key = utils.bootstrap(self.logpath) shutil.copy2(self.logpath, self.logpath_b) self.log_b = Log(self.logpath_b) self.log.load() self.log_b.load() self.view = View(self.log, self.root_key) self.view_b = View(self.log_b, self.root_key) self.view.build() self.view_b.build() protocol = BasefsSyncProtocol(self.log) protocol_b = BasefsSyncProtocol(self.log_b) protocol.transport = utils.Socket() protocol_b.transport = utils.Socket() return protocol, protocol_b def test_sync(self): protocol, protocol_b = self.init() self.assertEqual(protocol.initial_request(), protocol_b.initial_request()) path = os.path.join(os.sep, 'home-' + utils.random_ascii()) node = self.view.mkdir(path) protocol.update_merkle(node.entry) self.assertNotEqual(protocol.initial_request(), protocol_b.initial_request()) # 1 request = protocol_b.initial_request() # 2 protocol.data_received(request) response = protocol.transport.read() protocol.transport = utils.Socket() self.assertEqual(protocol.LS, response.decode().splitlines()[0]) self.assertEqual('*/ ' + self.log.root.hash, response.decode().splitlines()[1]) # 3 protocol_b.data_received(response) response = protocol_b.transport.read() protocol_b.transport = utils.Socket() self.assertEqual(protocol.PATH_REQ, response.decode().splitlines()[0]) self.assertEqual(path, response.decode().splitlines()[1]) self.assertEqual(2, len(response.decode().splitlines())) # 4 protocol.data_received(response) response = protocol.transport.read() with self.assertRaises(ValueError): protocol.transport.read(close_check=True) protocol.transport = utils.Socket() self.assertEqual(protocol.ENTRIES, response.decode().splitlines()[0]) self.assertEqual(3, len(response.decode().splitlines())) self.assertEqual(protocol.CLOSE, response.decode().splitlines()[-1]) # 5 protocol_b.data_received(response) with self.assertRaises(ValueError): protocol_b.transport.read(close_check=True) self.assertEqual(self.log.print_tree(), self.log_b.print_tree()) self.assertDictEqual(protocol.merkle, protocol_b.merkle) # delete dnode = self.view.delete(path) protocol.update_merkle(dnode.entry) request = protocol_b.initial_request() # 2 protocol.data_received(request) response = protocol.transport.read() protocol.transport = utils.Socket() self.assertEqual(protocol.LS, response.decode().splitlines()[0]) self.assertEqual('*/ ' + self.log.root.hash, response.decode().splitlines()[1]) # 3 protocol_b.data_received(response) response = protocol_b.transport.read() protocol_b.transport = utils.Socket() self.assertEqual(protocol.LS, response.decode().splitlines()[0]) self.assertEqual('*%s %s' % (path, node.entry.hash), response.decode().splitlines()[1]) self.assertEqual(2, len(response.decode().splitlines())) # 4 protocol.data_received(response) response = protocol.transport.read() with self.assertRaises(ValueError): protocol.transport.read(close_check=True) protocol.transport = utils.Socket() self.assertEqual(protocol.ENTRIES, response.decode().splitlines()[0]) self.assertEqual(3, len(response.decode().splitlines())) self.assertEqual(protocol.CLOSE, response.decode().splitlines()[-1]) # 5 protocol_b.data_received(response) with self.assertRaises(ValueError): protocol_b.transport.read(close_check=True) self.assertEqual(self.log.print_tree(), self.log_b.print_tree()) self.assertDictEqual(protocol.merkle, protocol_b.merkle) def test_sync_reverse(self): # Reverse communication protocol, protocol_b = self.init() path = os.path.join(os.sep, 'home-' + utils.random_ascii()) node = self.view.mkdir(path) protocol.update_merkle(node.entry) # 1 request = protocol.initial_request() # 2 protocol_b.data_received(request) response = protocol_b.transport.read() protocol_b.transport = utils.Socket() self.assertEqual(protocol.LS, response.decode().splitlines()[0]) self.assertEqual('*/ ' + self.log.root.hash, response.decode().splitlines()[1]) # 3 protocol.data_received(response) with self.assertRaises(ValueError): protocol.transport.read(close_check=True) response = protocol.transport.read() self.assertEqual(protocol.ENTRIES, response.decode().splitlines()[0]) self.assertEqual(3, len(response.decode().splitlines())) self.assertEqual(protocol.CLOSE, response.decode().splitlines()[-1]) # 4 protocol_b.data_received(response) with self.assertRaises(ValueError): protocol_b.transport.read(close_check=True) with self.assertRaises(ValueError): protocol.transport.read(close_check=True) self.assertEqual(self.log.print_tree(), self.log_b.print_tree()) self.assertDictEqual(protocol.merkle, protocol_b.merkle) # delete dnode = self.view.delete(path) protocol.update_merkle(dnode.entry) request = protocol.initial_request() # 2 protocol_b.data_received(request) response = protocol_b.transport.read() protocol_b.transport = utils.Socket() self.assertEqual(protocol.LS, response.decode().splitlines()[0]) self.assertEqual('*/ ' + self.log.root.hash, response.decode().splitlines()[1]) # 3 protocol.data_received(response) response = protocol.transport.read() protocol.transport = utils.Socket() self.assertEqual(protocol.LS, response.decode().splitlines()[0]) self.assertEqual( '*%s %s %s' % (path, node.entry.hash, dnode.entry.hash), response.decode().splitlines()[1]) self.assertEqual(2, len(response.decode().splitlines())) # 4 protocol_b.data_received(response) response = protocol_b.transport.read() protocol_b.transport = utils.Socket() self.assertEqual(protocol.ENTRY_REQ, response.decode().splitlines()[0]) self.assertEqual(dnode.entry.hash, response.decode().splitlines()[1]) # 5 protocol.data_received(response) response = protocol.transport.read() with self.assertRaises(ValueError): protocol.transport.read(close_check=True) protocol.transport = utils.Socket() self.assertEqual(protocol.ENTRIES, response.decode().splitlines()[0]) self.assertEqual(protocol.CLOSE, response.decode().splitlines()[-1]) self.assertEqual(3, len(response.decode().splitlines())) # BAD request before = self.log.print_tree() protocol.data_received(response) self.assertEqual(before, self.log.print_tree()) # 6 protocol_b.data_received(response) with self.assertRaises(ValueError): protocol_b.transport.read(close_check=True) self.assertDictEqual(protocol.merkle, protocol_b.merkle) self.assertEqual(self.log.print_tree(), self.log_b.print_tree())
class HandlerTests(unittest.TestCase): def init(self): __, self.logpath = tempfile.mkstemp() __, self.logpath_b = tempfile.mkstemp() self.addCleanup(os.remove, self.logpath) self.addCleanup(os.remove, self.logpath_b) self.log, self.root_key = utils.bootstrap(self.logpath) shutil.copy2(self.logpath, self.logpath_b) self.log_b = Log(self.logpath_b) self.log.load() self.log_b.load() self.view = View(self.log, self.root_key) self.view_b = View(self.log_b, self.root_key) self.view.build() self.view_b.build() protocol = BasefsSyncProtocol(self.log) protocol_b = BasefsSyncProtocol(self.log_b) protocol.transport = utils.Socket() protocol_b.transport = utils.Socket() return protocol, protocol_b def test_sync(self): protocol, protocol_b = self.init() self.assertEqual(protocol.initial_request(), protocol_b.initial_request()) path = os.path.join(os.sep, 'home-' + utils.random_ascii()) node = self.view.mkdir(path) protocol.update_merkle(node.entry) self.assertNotEqual(protocol.initial_request(), protocol_b.initial_request()) # 1 request = protocol_b.initial_request() # 2 protocol.data_received(request) response = protocol.transport.read() protocol.transport = utils.Socket() self.assertEqual(protocol.LS, response.decode().splitlines()[0]) self.assertEqual('*/ ' + self.log.root.hash, response.decode().splitlines()[1]) # 3 protocol_b.data_received(response) response = protocol_b.transport.read() protocol_b.transport = utils.Socket() self.assertEqual(protocol.PATH_REQ, response.decode().splitlines()[0]) self.assertEqual(path, response.decode().splitlines()[1]) self.assertEqual(2, len(response.decode().splitlines())) # 4 protocol.data_received(response) response = protocol.transport.read() with self.assertRaises(ValueError): protocol.transport.read(close_check=True) protocol.transport = utils.Socket() self.assertEqual(protocol.ENTRIES, response.decode().splitlines()[0]) self.assertEqual(3, len(response.decode().splitlines())) self.assertEqual(protocol.CLOSE, response.decode().splitlines()[-1]) # 5 protocol_b.data_received(response) with self.assertRaises(ValueError): protocol_b.transport.read(close_check=True) self.assertEqual(self.log.print_tree(), self.log_b.print_tree()) self.assertDictEqual(protocol.merkle, protocol_b.merkle) # delete dnode = self.view.delete(path) protocol.update_merkle(dnode.entry) request = protocol_b.initial_request() # 2 protocol.data_received(request) response = protocol.transport.read() protocol.transport = utils.Socket() self.assertEqual(protocol.LS, response.decode().splitlines()[0]) self.assertEqual('*/ ' + self.log.root.hash, response.decode().splitlines()[1]) # 3 protocol_b.data_received(response) response = protocol_b.transport.read() protocol_b.transport = utils.Socket() self.assertEqual(protocol.LS, response.decode().splitlines()[0]) self.assertEqual('*%s %s' % (path, node.entry.hash), response.decode().splitlines()[1]) self.assertEqual(2, len(response.decode().splitlines())) # 4 protocol.data_received(response) response = protocol.transport.read() with self.assertRaises(ValueError): protocol.transport.read(close_check=True) protocol.transport = utils.Socket() self.assertEqual(protocol.ENTRIES, response.decode().splitlines()[0]) self.assertEqual(3, len(response.decode().splitlines())) self.assertEqual(protocol.CLOSE, response.decode().splitlines()[-1]) # 5 protocol_b.data_received(response) with self.assertRaises(ValueError): protocol_b.transport.read(close_check=True) self.assertEqual(self.log.print_tree(), self.log_b.print_tree()) self.assertDictEqual(protocol.merkle, protocol_b.merkle) def test_sync_reverse(self): # Reverse communication protocol, protocol_b = self.init() path = os.path.join(os.sep, 'home-' + utils.random_ascii()) node = self.view.mkdir(path) protocol.update_merkle(node.entry) # 1 request = protocol.initial_request() # 2 protocol_b.data_received(request) response = protocol_b.transport.read() protocol_b.transport = utils.Socket() self.assertEqual(protocol.LS, response.decode().splitlines()[0]) self.assertEqual('*/ ' + self.log.root.hash, response.decode().splitlines()[1]) # 3 protocol.data_received(response) with self.assertRaises(ValueError): protocol.transport.read(close_check=True) response = protocol.transport.read() self.assertEqual(protocol.ENTRIES, response.decode().splitlines()[0]) self.assertEqual(3, len(response.decode().splitlines())) self.assertEqual(protocol.CLOSE, response.decode().splitlines()[-1]) # 4 protocol_b.data_received(response) with self.assertRaises(ValueError): protocol_b.transport.read(close_check=True) with self.assertRaises(ValueError): protocol.transport.read(close_check=True) self.assertEqual(self.log.print_tree(), self.log_b.print_tree()) self.assertDictEqual(protocol.merkle, protocol_b.merkle) # delete dnode = self.view.delete(path) protocol.update_merkle(dnode.entry) request = protocol.initial_request() # 2 protocol_b.data_received(request) response = protocol_b.transport.read() protocol_b.transport = utils.Socket() self.assertEqual(protocol.LS, response.decode().splitlines()[0]) self.assertEqual('*/ ' + self.log.root.hash, response.decode().splitlines()[1]) # 3 protocol.data_received(response) response = protocol.transport.read() protocol.transport = utils.Socket() self.assertEqual(protocol.LS, response.decode().splitlines()[0]) self.assertEqual('*%s %s %s' % (path, node.entry.hash, dnode.entry.hash), response.decode().splitlines()[1]) self.assertEqual(2, len(response.decode().splitlines())) # 4 protocol_b.data_received(response) response = protocol_b.transport.read() protocol_b.transport = utils.Socket() self.assertEqual(protocol.ENTRY_REQ, response.decode().splitlines()[0]) self.assertEqual(dnode.entry.hash, response.decode().splitlines()[1]) # 5 protocol.data_received(response) response = protocol.transport.read() with self.assertRaises(ValueError): protocol.transport.read(close_check=True) protocol.transport = utils.Socket() self.assertEqual(protocol.ENTRIES, response.decode().splitlines()[0]) self.assertEqual(protocol.CLOSE, response.decode().splitlines()[-1]) self.assertEqual(3, len(response.decode().splitlines())) # BAD request before = self.log.print_tree() protocol.data_received(response) self.assertEqual(before, self.log.print_tree()) # 6 protocol_b.data_received(response) with self.assertRaises(ValueError): protocol_b.transport.read(close_check=True) self.assertDictEqual(protocol.merkle, protocol_b.merkle) self.assertEqual(self.log.print_tree(), self.log_b.print_tree())
def command(mount=False, arg_parser=None): if arg_parser is None: # Invoking from code, not basefs bin set_parser(parser) arg_parser = parser args = arg_parser.parse_args() context = get_context(args.logpath, defaults) logpath = context.fs.logpath if context.mount_info: mountpoint = context.mount_info.mountpoint sys.stderr.write("Error: log %s already mounted in %s\n" % (logpath, mountpoint)) sys.exit(4) ip, *port = args.bind.split(':') if port: port = int(port[0]) else: port = context.fs.port if args.iface: iface_ip = utils.get_ip_address(args.iface) if ip != '0.0.0.0' and ip != iface_ip: sys.stderr.write("-bind and -iface ip addresses do not match %s != %s\n" % (ip, iface_ip)) sys.exit(9) ip = iface_ip # logpath = args.logpath config = get_or_create_config(defaults) hostname = args.hostname section = config[context.fs.name] if not hostname: if context.fs.name in config: hostname = section['hostname'] else: hostname = defaults.hostname rpc_port = port+1 sync_port = port+2 logpath = os.path.normpath(logpath) keypath = os.path.normpath(args.keypath) logging.basicConfig( level=logging.DEBUG if args.debug else logging.INFO, format='%(asctime)-15s [%(levelname)s] %(name)s: %(message)s', ) logpath = os.path.normpath(logpath) log = Log(logpath) log.load() if keypath == defaults.keypath and not os.path.exists(keypath): view = View(log) else: key = Key.load(keypath) view = View(log, key) view.build() serf = None serf_agent = None if args.serf: join = args.join.split(',') if args.join else [] serf, serf_agent = gossip.run(section, view, ip, port, hostname, join) if args.watcher: handler = handlers.Handler(args.watcher, view.log, state=serf.blockstate) else: if args.watcher: handler = handlers.Handler(args.watcher, view.log) if mount: init_function = lambda: None if args.serf: # Eventloop needs to run on a separated thread when using FUSE init_function = lambda: loop.run_thread(view, serf, port+2, config=section) mountpoint = args.mountpoint sys.stdout.write('Mounting %s into %s\n' % (logpath, mountpoint)) fs = FileSystem(view, serf=serf, serf_agent=serf_agent, init_function=init_function) fsname = '%s:%i' % (logpath, sync_port) foreground = args.foreground or args.debug FUSE(fs, mountpoint, fsname=fsname, nothreads=False, foreground=foreground) else: try: loop.run(view, serf, port+2, config=section) except KeyboardInterrupt: pass finally: serf_agent.stop()