def command(): # bootsrap <name> -i <ip>[:<port>] [-l <logpath>] [-k <keypath>] [-f] args = parser.parse_args() default_logpath = os.path.join(defaults.dir, 'logs', args.name) logpath = args.logpath or default_logpath port = args.port or get_port(args.name) utils.create_logdir(logpath, default_logpath, args.force) keys = [] for keypath in args.keypaths.split(','): if not os.path.isfile(keypath): sys.stderr.write("Error: bootsraping keypath %s does not exist.\n" % keypath) sys.exit(2) keys.append(Key.load(keypath)) log = Log(logpath) # TODO, name=args.name) ips = [] for ip in args.ips.split(','): if ':' not in ip: ip += ':%i' % port ips.append(ip) log.bootstrap(keys, ips) config = get_or_create_config(defaults) if args.name not in config: config[args.name] = {} config[args.name].update({ 'logpath': logpath, 'port': str(port), }) config.save() sys.stdout.write('Created log file %s\n' % logpath) sys.stdout.write('Network bootstraping will happen at:\n %s\n' % '\n '.join(ips)) sys.exit(0)
def command(): # get <domainTXT/domain:port/ip:port> [-l <logpath>] [-n <name>] [-p] [-f] [-d] args = parser.parse_args() default_logpath = os.path.join(defaults.dir, 'logs', args.name) logpath = args.logpath or default_logpath if logpath != '-': create_logdir(logpath, default_logpath, args.force) ip, *port = args.addr.split(':') try: ip = str(ipaddress.ip_address(ip)) except ValueError: nslookup = """nslookup -q=txt %s|grep 'text =' | sed -E 's/.*text = "([^"]+)".*/\\1/'""" % ip lookup = subprocess.Popen(nslookup, shell=True, stdout=subprocess.PIPE) lookup = lookup.stdout.read().decode().strip() if lookup: ip, *port = lookup.split(':') else: nslookup = "nslookup %s -q=a | grep Address|tail -n1 | awk {'print $2'}" % ip lookup = subprocess.Popen(nslookup, shell=True, stdout=subprocess.PIPE) lookup = lookup.stdout.read().decode().strip() if not '#' in lookup: ip = lookup if not port: port = get_port(args.name) + 2 else: port = int(port[0]) sys.stderr.write("Connecting to %s:%i\n" % (ip, port)) received = False if logpath == '-': for data in utils.netcat(ip, port, b'cGET'): sys.stdout.write(data) received = True else: try: with open(logpath, 'w') as handler: for data in utils.netcat(ip, port, b'cGET'): handler.write(data) received = True except ConnectionRefusedError: os.remove(logpath) sys.stderr.write("Error: connection refused, bad port?\n") sys.exit(3) if not received: sys.stderr.write("Error: nothing has been received, bad port?\n") sys.exit(2) elif logpath != '-': log = Log(logpath) try: log.load(validate=True) except exceptions.ValidationError as e: sys.stderr.write(str(e) + '\n') sys.stdout.write("%s log created\n" % logpath)
def command(): # get <domainTXT/domain:port/ip:port> [-l <logpath>] [-n <name>] [-p] [-f] [-d] args = parser.parse_args() default_logpath = os.path.join(defaults.dir, "logs", args.name) logpath = args.logpath or default_logpath if logpath != "-": create_logdir(logpath, default_logpath, args.force) ip, *port = args.addr.split(":") try: ip = str(ipaddress.ip_address(ip)) except ValueError: nslookup = """nslookup -q=txt %s|grep 'text =' | sed -E 's/.*text = "([^"]+)".*/\\1/'""" % ip lookup = subprocess.Popen(nslookup, shell=True, stdout=subprocess.PIPE) lookup = lookup.stdout.read().decode().strip() if lookup: ip, *port = lookup.split(":") else: nslookup = "nslookup %s -q=a | grep Address|tail -n1 | awk {'print $2'}" % ip lookup = subprocess.Popen(nslookup, shell=True, stdout=subprocess.PIPE) lookup = lookup.stdout.read().decode().strip() if not "#" in lookup: ip = lookup if not port: port = get_port(args.name) + 2 else: port = int(port[0]) sys.stderr.write("Connecting to %s:%i\n" % (ip, port)) received = False if logpath == "-": for data in utils.netcat(ip, port, b"cGET"): sys.stdout.write(data) received = True else: try: with open(logpath, "w") as handler: for data in utils.netcat(ip, port, b"cGET"): handler.write(data) received = True except ConnectionRefusedError: os.remove(logpath) sys.stderr.write("Error: connection refused, bad port?\n") sys.exit(3) if not received: sys.stderr.write("Error: nothing has been received, bad port?\n") sys.exit(2) elif logpath != "-": log = Log(logpath) try: log.load(validate=True) except exceptions.ValidationError as e: sys.stderr.write(str(e) + "\n") sys.stdout.write("%s log created\n" % logpath)
def setUp(self): __, self.logpath = tempfile.mkstemp() __, self.logpath_b = tempfile.mkstemp() self.addCleanup(os.remove, self.logpath) self.addCleanup(os.remove, self.logpath_b) __, self.keypath = tempfile.mkstemp() self.addCleanup(os.remove, self.keypath) self.port = random.randint(40000, 50000-1) self.port_b = random.randint(50000, 60000) log = Log(self.logpath) root_key = Key.generate() log.bootstrap([root_key], ['127.0.0.1:%i' % self.port]) root_key.save(self.keypath) shutil.copy2(self.logpath, self.logpath_b) self.hostname = utils.random_ascii(10) self.hostname_b = utils.random_ascii(10) self.mountpath = tempfile.mkdtemp() self.mountpath_b = tempfile.mkdtemp() context = { 'mountpath': self.mountpath, 'logpath': self.logpath, 'keypath': self.keypath, 'port': self.port, 'hostname': self.hostname, } cmd = 'basefs mount %(logpath)s %(mountpath)s -k %(keypath)s -p %(port)s -n %(hostname)s' proc = subprocess.Popen(cmd % context, shell=True) self.addCleanup(proc.kill) time.sleep(1) self.addCleanup(proc.kill) context.update({ 'mountpath': self.mountpath_b, 'logpath': self.logpath_b, 'port': self.port_b, 'hostname': self.hostname_b, }) proc = subprocess.Popen(cmd % context, shell=True) self.addCleanup(proc.kill) self.addCleanup(time.sleep, 1) self.addCleanup(proc.kill) self.addCleanup(shutil.rmtree, self.mountpath) self.addCleanup(shutil.rmtree, self.mountpath_b) time.sleep(1)
def init(self): __, self.logpath = tempfile.mkstemp() __, self.logpath_b = tempfile.mkstemp() self.addCleanup(os.remove, self.logpath) self.addCleanup(os.remove, self.logpath_b) self.log, self.root_key = utils.bootstrap(self.logpath) shutil.copy2(self.logpath, self.logpath_b) self.log_b = Log(self.logpath_b) self.log.load() self.log_b.load() self.view = View(self.log, self.root_key) self.view_b = View(self.log_b, self.root_key) self.view.build() self.view_b.build() protocol = BasefsSyncProtocol(self.log) protocol_b = BasefsSyncProtocol(self.log_b) protocol.transport = utils.Socket() protocol_b.transport = utils.Socket() return protocol, protocol_b
def bootstrap(logpath): log = Log(logpath) root_key = Key.generate() ip = '127.0.0.1' log.bootstrap([root_key], [ip]) return log, root_key
class HandlerTests(unittest.TestCase): def init(self): __, self.logpath = tempfile.mkstemp() __, self.logpath_b = tempfile.mkstemp() self.addCleanup(os.remove, self.logpath) self.addCleanup(os.remove, self.logpath_b) self.log, self.root_key = utils.bootstrap(self.logpath) shutil.copy2(self.logpath, self.logpath_b) self.log_b = Log(self.logpath_b) self.log.load() self.log_b.load() self.view = View(self.log, self.root_key) self.view_b = View(self.log_b, self.root_key) self.view.build() self.view_b.build() protocol = BasefsSyncProtocol(self.log) protocol_b = BasefsSyncProtocol(self.log_b) protocol.transport = utils.Socket() protocol_b.transport = utils.Socket() return protocol, protocol_b def test_sync(self): protocol, protocol_b = self.init() self.assertEqual(protocol.initial_request(), protocol_b.initial_request()) path = os.path.join(os.sep, 'home-' + utils.random_ascii()) node = self.view.mkdir(path) protocol.update_merkle(node.entry) self.assertNotEqual(protocol.initial_request(), protocol_b.initial_request()) # 1 request = protocol_b.initial_request() # 2 protocol.data_received(request) response = protocol.transport.read() protocol.transport = utils.Socket() self.assertEqual(protocol.LS, response.decode().splitlines()[0]) self.assertEqual('*/ ' + self.log.root.hash, response.decode().splitlines()[1]) # 3 protocol_b.data_received(response) response = protocol_b.transport.read() protocol_b.transport = utils.Socket() self.assertEqual(protocol.PATH_REQ, response.decode().splitlines()[0]) self.assertEqual(path, response.decode().splitlines()[1]) self.assertEqual(2, len(response.decode().splitlines())) # 4 protocol.data_received(response) response = protocol.transport.read() with self.assertRaises(ValueError): protocol.transport.read(close_check=True) protocol.transport = utils.Socket() self.assertEqual(protocol.ENTRIES, response.decode().splitlines()[0]) self.assertEqual(3, len(response.decode().splitlines())) self.assertEqual(protocol.CLOSE, response.decode().splitlines()[-1]) # 5 protocol_b.data_received(response) with self.assertRaises(ValueError): protocol_b.transport.read(close_check=True) self.assertEqual(self.log.print_tree(), self.log_b.print_tree()) self.assertDictEqual(protocol.merkle, protocol_b.merkle) # delete dnode = self.view.delete(path) protocol.update_merkle(dnode.entry) request = protocol_b.initial_request() # 2 protocol.data_received(request) response = protocol.transport.read() protocol.transport = utils.Socket() self.assertEqual(protocol.LS, response.decode().splitlines()[0]) self.assertEqual('*/ ' + self.log.root.hash, response.decode().splitlines()[1]) # 3 protocol_b.data_received(response) response = protocol_b.transport.read() protocol_b.transport = utils.Socket() self.assertEqual(protocol.LS, response.decode().splitlines()[0]) self.assertEqual('*%s %s' % (path, node.entry.hash), response.decode().splitlines()[1]) self.assertEqual(2, len(response.decode().splitlines())) # 4 protocol.data_received(response) response = protocol.transport.read() with self.assertRaises(ValueError): protocol.transport.read(close_check=True) protocol.transport = utils.Socket() self.assertEqual(protocol.ENTRIES, response.decode().splitlines()[0]) self.assertEqual(3, len(response.decode().splitlines())) self.assertEqual(protocol.CLOSE, response.decode().splitlines()[-1]) # 5 protocol_b.data_received(response) with self.assertRaises(ValueError): protocol_b.transport.read(close_check=True) self.assertEqual(self.log.print_tree(), self.log_b.print_tree()) self.assertDictEqual(protocol.merkle, protocol_b.merkle) def test_sync_reverse(self): # Reverse communication protocol, protocol_b = self.init() path = os.path.join(os.sep, 'home-' + utils.random_ascii()) node = self.view.mkdir(path) protocol.update_merkle(node.entry) # 1 request = protocol.initial_request() # 2 protocol_b.data_received(request) response = protocol_b.transport.read() protocol_b.transport = utils.Socket() self.assertEqual(protocol.LS, response.decode().splitlines()[0]) self.assertEqual('*/ ' + self.log.root.hash, response.decode().splitlines()[1]) # 3 protocol.data_received(response) with self.assertRaises(ValueError): protocol.transport.read(close_check=True) response = protocol.transport.read() self.assertEqual(protocol.ENTRIES, response.decode().splitlines()[0]) self.assertEqual(3, len(response.decode().splitlines())) self.assertEqual(protocol.CLOSE, response.decode().splitlines()[-1]) # 4 protocol_b.data_received(response) with self.assertRaises(ValueError): protocol_b.transport.read(close_check=True) with self.assertRaises(ValueError): protocol.transport.read(close_check=True) self.assertEqual(self.log.print_tree(), self.log_b.print_tree()) self.assertDictEqual(protocol.merkle, protocol_b.merkle) # delete dnode = self.view.delete(path) protocol.update_merkle(dnode.entry) request = protocol.initial_request() # 2 protocol_b.data_received(request) response = protocol_b.transport.read() protocol_b.transport = utils.Socket() self.assertEqual(protocol.LS, response.decode().splitlines()[0]) self.assertEqual('*/ ' + self.log.root.hash, response.decode().splitlines()[1]) # 3 protocol.data_received(response) response = protocol.transport.read() protocol.transport = utils.Socket() self.assertEqual(protocol.LS, response.decode().splitlines()[0]) self.assertEqual( '*%s %s %s' % (path, node.entry.hash, dnode.entry.hash), response.decode().splitlines()[1]) self.assertEqual(2, len(response.decode().splitlines())) # 4 protocol_b.data_received(response) response = protocol_b.transport.read() protocol_b.transport = utils.Socket() self.assertEqual(protocol.ENTRY_REQ, response.decode().splitlines()[0]) self.assertEqual(dnode.entry.hash, response.decode().splitlines()[1]) # 5 protocol.data_received(response) response = protocol.transport.read() with self.assertRaises(ValueError): protocol.transport.read(close_check=True) protocol.transport = utils.Socket() self.assertEqual(protocol.ENTRIES, response.decode().splitlines()[0]) self.assertEqual(protocol.CLOSE, response.decode().splitlines()[-1]) self.assertEqual(3, len(response.decode().splitlines())) # BAD request before = self.log.print_tree() protocol.data_received(response) self.assertEqual(before, self.log.print_tree()) # 6 protocol_b.data_received(response) with self.assertRaises(ValueError): protocol_b.transport.read(close_check=True) self.assertDictEqual(protocol.merkle, protocol_b.merkle) self.assertEqual(self.log.print_tree(), self.log_b.print_tree())
class HandlerTests(unittest.TestCase): def init(self): __, self.logpath = tempfile.mkstemp() __, self.logpath_b = tempfile.mkstemp() self.addCleanup(os.remove, self.logpath) self.addCleanup(os.remove, self.logpath_b) self.log, self.root_key = utils.bootstrap(self.logpath) shutil.copy2(self.logpath, self.logpath_b) self.log_b = Log(self.logpath_b) self.log.load() self.log_b.load() self.view = View(self.log, self.root_key) self.view_b = View(self.log_b, self.root_key) self.view.build() self.view_b.build() protocol = BasefsSyncProtocol(self.log) protocol_b = BasefsSyncProtocol(self.log_b) protocol.transport = utils.Socket() protocol_b.transport = utils.Socket() return protocol, protocol_b def test_sync(self): protocol, protocol_b = self.init() self.assertEqual(protocol.initial_request(), protocol_b.initial_request()) path = os.path.join(os.sep, 'home-' + utils.random_ascii()) node = self.view.mkdir(path) protocol.update_merkle(node.entry) self.assertNotEqual(protocol.initial_request(), protocol_b.initial_request()) # 1 request = protocol_b.initial_request() # 2 protocol.data_received(request) response = protocol.transport.read() protocol.transport = utils.Socket() self.assertEqual(protocol.LS, response.decode().splitlines()[0]) self.assertEqual('*/ ' + self.log.root.hash, response.decode().splitlines()[1]) # 3 protocol_b.data_received(response) response = protocol_b.transport.read() protocol_b.transport = utils.Socket() self.assertEqual(protocol.PATH_REQ, response.decode().splitlines()[0]) self.assertEqual(path, response.decode().splitlines()[1]) self.assertEqual(2, len(response.decode().splitlines())) # 4 protocol.data_received(response) response = protocol.transport.read() with self.assertRaises(ValueError): protocol.transport.read(close_check=True) protocol.transport = utils.Socket() self.assertEqual(protocol.ENTRIES, response.decode().splitlines()[0]) self.assertEqual(3, len(response.decode().splitlines())) self.assertEqual(protocol.CLOSE, response.decode().splitlines()[-1]) # 5 protocol_b.data_received(response) with self.assertRaises(ValueError): protocol_b.transport.read(close_check=True) self.assertEqual(self.log.print_tree(), self.log_b.print_tree()) self.assertDictEqual(protocol.merkle, protocol_b.merkle) # delete dnode = self.view.delete(path) protocol.update_merkle(dnode.entry) request = protocol_b.initial_request() # 2 protocol.data_received(request) response = protocol.transport.read() protocol.transport = utils.Socket() self.assertEqual(protocol.LS, response.decode().splitlines()[0]) self.assertEqual('*/ ' + self.log.root.hash, response.decode().splitlines()[1]) # 3 protocol_b.data_received(response) response = protocol_b.transport.read() protocol_b.transport = utils.Socket() self.assertEqual(protocol.LS, response.decode().splitlines()[0]) self.assertEqual('*%s %s' % (path, node.entry.hash), response.decode().splitlines()[1]) self.assertEqual(2, len(response.decode().splitlines())) # 4 protocol.data_received(response) response = protocol.transport.read() with self.assertRaises(ValueError): protocol.transport.read(close_check=True) protocol.transport = utils.Socket() self.assertEqual(protocol.ENTRIES, response.decode().splitlines()[0]) self.assertEqual(3, len(response.decode().splitlines())) self.assertEqual(protocol.CLOSE, response.decode().splitlines()[-1]) # 5 protocol_b.data_received(response) with self.assertRaises(ValueError): protocol_b.transport.read(close_check=True) self.assertEqual(self.log.print_tree(), self.log_b.print_tree()) self.assertDictEqual(protocol.merkle, protocol_b.merkle) def test_sync_reverse(self): # Reverse communication protocol, protocol_b = self.init() path = os.path.join(os.sep, 'home-' + utils.random_ascii()) node = self.view.mkdir(path) protocol.update_merkle(node.entry) # 1 request = protocol.initial_request() # 2 protocol_b.data_received(request) response = protocol_b.transport.read() protocol_b.transport = utils.Socket() self.assertEqual(protocol.LS, response.decode().splitlines()[0]) self.assertEqual('*/ ' + self.log.root.hash, response.decode().splitlines()[1]) # 3 protocol.data_received(response) with self.assertRaises(ValueError): protocol.transport.read(close_check=True) response = protocol.transport.read() self.assertEqual(protocol.ENTRIES, response.decode().splitlines()[0]) self.assertEqual(3, len(response.decode().splitlines())) self.assertEqual(protocol.CLOSE, response.decode().splitlines()[-1]) # 4 protocol_b.data_received(response) with self.assertRaises(ValueError): protocol_b.transport.read(close_check=True) with self.assertRaises(ValueError): protocol.transport.read(close_check=True) self.assertEqual(self.log.print_tree(), self.log_b.print_tree()) self.assertDictEqual(protocol.merkle, protocol_b.merkle) # delete dnode = self.view.delete(path) protocol.update_merkle(dnode.entry) request = protocol.initial_request() # 2 protocol_b.data_received(request) response = protocol_b.transport.read() protocol_b.transport = utils.Socket() self.assertEqual(protocol.LS, response.decode().splitlines()[0]) self.assertEqual('*/ ' + self.log.root.hash, response.decode().splitlines()[1]) # 3 protocol.data_received(response) response = protocol.transport.read() protocol.transport = utils.Socket() self.assertEqual(protocol.LS, response.decode().splitlines()[0]) self.assertEqual('*%s %s %s' % (path, node.entry.hash, dnode.entry.hash), response.decode().splitlines()[1]) self.assertEqual(2, len(response.decode().splitlines())) # 4 protocol_b.data_received(response) response = protocol_b.transport.read() protocol_b.transport = utils.Socket() self.assertEqual(protocol.ENTRY_REQ, response.decode().splitlines()[0]) self.assertEqual(dnode.entry.hash, response.decode().splitlines()[1]) # 5 protocol.data_received(response) response = protocol.transport.read() with self.assertRaises(ValueError): protocol.transport.read(close_check=True) protocol.transport = utils.Socket() self.assertEqual(protocol.ENTRIES, response.decode().splitlines()[0]) self.assertEqual(protocol.CLOSE, response.decode().splitlines()[-1]) self.assertEqual(3, len(response.decode().splitlines())) # BAD request before = self.log.print_tree() protocol.data_received(response) self.assertEqual(before, self.log.print_tree()) # 6 protocol_b.data_received(response) with self.assertRaises(ValueError): protocol_b.transport.read(close_check=True) self.assertDictEqual(protocol.merkle, protocol_b.merkle) self.assertEqual(self.log.print_tree(), self.log_b.print_tree())
def command(mount=False, arg_parser=None): if arg_parser is None: # Invoking from code, not basefs bin set_parser(parser) arg_parser = parser args = arg_parser.parse_args() context = get_context(args.logpath, defaults) logpath = context.fs.logpath if context.mount_info: mountpoint = context.mount_info.mountpoint sys.stderr.write("Error: log %s already mounted in %s\n" % (logpath, mountpoint)) sys.exit(4) ip, *port = args.bind.split(':') if port: port = int(port[0]) else: port = context.fs.port if args.iface: iface_ip = utils.get_ip_address(args.iface) if ip != '0.0.0.0' and ip != iface_ip: sys.stderr.write("-bind and -iface ip addresses do not match %s != %s\n" % (ip, iface_ip)) sys.exit(9) ip = iface_ip # logpath = args.logpath config = get_or_create_config(defaults) hostname = args.hostname section = config[context.fs.name] if not hostname: if context.fs.name in config: hostname = section['hostname'] else: hostname = defaults.hostname rpc_port = port+1 sync_port = port+2 logpath = os.path.normpath(logpath) keypath = os.path.normpath(args.keypath) logging.basicConfig( level=logging.DEBUG if args.debug else logging.INFO, format='%(asctime)-15s [%(levelname)s] %(name)s: %(message)s', ) logpath = os.path.normpath(logpath) log = Log(logpath) log.load() if keypath == defaults.keypath and not os.path.exists(keypath): view = View(log) else: key = Key.load(keypath) view = View(log, key) view.build() serf = None serf_agent = None if args.serf: join = args.join.split(',') if args.join else [] serf, serf_agent = gossip.run(section, view, ip, port, hostname, join) if args.watcher: handler = handlers.Handler(args.watcher, view.log, state=serf.blockstate) else: if args.watcher: handler = handlers.Handler(args.watcher, view.log) if mount: init_function = lambda: None if args.serf: # Eventloop needs to run on a separated thread when using FUSE init_function = lambda: loop.run_thread(view, serf, port+2, config=section) mountpoint = args.mountpoint sys.stdout.write('Mounting %s into %s\n' % (logpath, mountpoint)) fs = FileSystem(view, serf=serf, serf_agent=serf_agent, init_function=init_function) fsname = '%s:%i' % (logpath, sync_port) foreground = args.foreground or args.debug FUSE(fs, mountpoint, fsname=fsname, nothreads=False, foreground=foreground) else: try: loop.run(view, serf, port+2, config=section) except KeyboardInterrupt: pass finally: serf_agent.stop()