Example #1
0
	def work(self):
		if self.running and not self.fs.single_node:
			self.restart = True

		@inlineCallbacks
		def do_work(db):
			inums = []
			try:
				all_done,n_nodes = yield db.DoFn("select min(event),count(*) from node where root=${root} and id != ${node}", root=self.fs.root_id, node=self.fs.node_id)
			except NoData:
				pass
			else:
				if not n_nodes:
					return
				trace('inodeclean',"%d nodes",n_nodes)
				try:
					last_syn, = yield db.DoFn("select id from event where node=${node} and typ = 's' and id <= ${evt} order by id desc limit 1", node=self.fs.node_id, evt=all_done)
				except NoData:
					pass
				else:
					# remove old inode entries
					trace('inodeclean',"upto %d",last_syn)

					yield db.DoSelect("select inode from event where node=${node} and typ = 'd' and id < ${id}", id=last_syn, node=self.fs.node_id, _callback=inums.append, _empty=True)
					yield db.Do("delete from event where node=${node} and id < ${id}", id=last_syn, node=self.fs.node_id, _empty=True)
			returnValue( inums )
		inums = yield self.fs.db(do_work, DB_RETRIES)

		# more deadlock prevention
		if inums:
			trace('inodeclean',"%d inodes",len(inums))
			yield self.fs.db(lambda db: db.Do("delete from inode where id in (%s)" % ",".join((str(x) for x in inums)), _empty=True), DB_RETRIES)
Example #2
0
	def _fclose(self):
		with self.lock:
			if not self.file:
				return
			trace('fs',"%d: close file", self.inum)
			self.file.close()
			self.file = None
Example #3
0
 def client_disconnected(self, client):
     try:
         self._clients.remove(client)
     except (ValueError, KeyError):
         pass
     else:
         trace("remote", "Disconnected client from %d", self.node_id)
Example #4
0
 def server_no_echo(self):
     self.echo_timer = None
     if "ping" in tracers:
         trace("ping", "%s: Echo timeout", str(self.node_id) if self.node_id else "?")
     else:
         self.disconnect_retry()
         trace("remote", "%s: Echo timeout", str(self.node_id) if self.node_id else "?")
Example #5
0
		def cb2(_):
			self.node.mtime = nowtuple()
			if self.node.size < end:
				trace('rw',"%d: end now %d", self.node.inum, end)
				self.node.size = end
			self.node.changes.add(offset,end)
			self.writes += 1
			return len(buf)
Example #6
0
 def server_disconnected(self, server):
     if self._server is server:
         self._server = None
         if self.echo_timer is not None:
             self.echo_timer.cancel()
             self.echo_timer = None
         trace("remote", "Disconnected server to %d", self.node_id)
         self.queue_retry()
Example #7
0
    def remote_exec(self, node, name, *a, **k):
        if node not in self.fs.topology:
            trace("remote", "NoLink remote %s %s %s %s %s", caller, node, name, repr(a), repr(k))
            raise NoLink(node, "remote exec")

            # TODO: cache calls to stuff like reading from a file
            # TODO: prevent cycles
        return self.fs.call_node(node, name, *a, **k)
Example #8
0
	def _save_done(self,db):
		if self._saving != db:
			trace('fs',"%s: save unlock error: %s %s",self,self._saving,db)
			raise RuntimeError("inode _save releasing")
		trace('fs',"%s: save unlock: %s",self,db)
		self._saving = None
		if self._saveq:
			self._saveq.pop(0).callback(None)
Example #9
0
				def chk(res=None):
					"""Test whether all data are here; if so, don't continue"""
					trace('cache',"%s: Callback %s: check ! %s - %s",self.inum, res, todo,self.available)
					if res is not None:
						if notfound[0] is None:
							notfound[0] = res
						else:
							notfound[0] &= res
					return not (todo - self.available)
Example #10
0
		def done(r):
			self.workerDefer = None
			if self.restart:
				if self.running:
					self.trigger(True)
				else:
					trace('background',"Stopped %s (shutdown)", self.__class__.__name__)
			else:
				trace('background',"Stopped %s", self.__class__.__name__)
Example #11
0
	def work(self):
		"""Sync root data"""
		d_inode = self.delta_inode; self.delta_inode = 0
		d_dir = self.delta_dir; self.delta_dir = 0
		d_block = self.delta_block; self.delta_block = 0
		if d_inode or d_block or d_dir:
			trace('rootup',"sync i%d b%d d%d", d_inode, d_block, d_dir)
			yield self.fs.db(lambda db: db.Do("update root set nfiles=nfiles+${inodes}, nblocks=nblocks+${blocks}, ndirs=ndirs+${dirs}", root=self.fs.root_id, inodes=d_inode, blocks=d_block, dirs=d_dir), DB_RETRIES)
		returnValue( None )
Example #12
0
	def startService(self):
		"""Startup. Part of IService."""
		trace('background',"StartService %s",self.__class__.__name__)
		super(BackgroundJob,self).startService()
		if self.restart and not self.workerCall:
			if self.restart > 1:
				self.run()
			else:
				self.trigger()
Example #13
0
	def work(self):
		nodes = set()
		if self.running:
			trace('nodecollect',"Start collecting")
			self.restart = True
		else:
			trace('nodecollect',"Not collecting")
			return # don't need this when shutting down

		@inlineCallbacks
		def do_work(db):
			yield db.DoSelect("select id from node where id != ${node} and root = ${root}",root=self.fs.root_id, node=self.fs.node_id, _empty=1, _callback=nodes.add)
			if not nodes:
				trace('nodecollect',"No other nodes: shutting down collector")
				self.fs.single_node = True
				return
			self.fs.single_node = False
			trace('nodecollect',"%d other nodes",len(nodes))

			# TODO: create a topology map
			## now create a topology map: how do I reach X from here?
			topo = yield self.get_paths(db)
			topo = next_hops(topo, self.fs.node_id)
			trace('nodecollect',"topology %s",repr(topo))
			returnValue( topo )
		topo = yield self.fs.db(do_work, DB_RETRIES)

		# drop obsolete nodes
		for k in self.fs.remote.keys():
			if k not in nodes:
				trace('nodecollect',"drop node %s",k)
				del self.fs.remote[k]

		#self.fs.topology = topo
		self.fs.topology,self.fs.neighbors = topo

		# add new nodes
		self.fs.missing_neighbors = set()
		for k in nodes:
			if k not in self.fs.remote:
				trace('nodecollect',"add node %s",k)
				d = self.fs.remote[k].connect_retry() # yes, this works, .remote auto-extends
				def pr(r):
					r.trap(NoLink)
					trace('error',"Node %s found, but not connected",k)
				d.addErrback(pr)
				def lerr(r):
					log.err(r,"Problem adding node")
				d.addErrback(lerr)

				yield d
			if k not in self.fs.topology:
				self.fs.missing_neighbors.add(k)

		trace('nodecollect',"Done.")
		self.fs.cleaner.trigger()
		self.fs.copier.trigger()
Example #14
0
	def now(self,node,event,db):
		"""Record that this inode's cache state needs to be written immediately."""
		trace('cacherecord',"update inode %d now (%d)",node.inum,event)
		if node.write_event is None or node.write_event < event:
			node.write_event = event
		db.call_rolledback(self.note,node)
		if node in self.caches:
			self.caches.remove(node)
		node._save(db,event=event)
Example #15
0
	def d_size(self,old,new):
		"""A file size changed."""
		old = (old+BLOCKSIZE-1)//BLOCKSIZE
		new = (new+BLOCKSIZE-1)//BLOCKSIZE
		delta = new-old
		if not delta: return
		trace('rootup',"#Size: %d", delta)
		self.delta_block += delta
		self.trigger()
Example #16
0
		def chk(res):
			lr = len(res)
			if lr < length:
				trace('error',"%d: read %d @ %d: only got %d", self.inum, length,offset,lr)
				self.available.delete(offset+lr,offset+length)
				self.known.delete(offset+lr,offset+length)
				# TODO: it's not necessarily gone, we just don't know where.
				# Thus, replace with "unknown" instead. Unfortunately we don't have that. Yet.
				self.fs.changer.note(self)
			return res
Example #17
0
	def note(self,node,event=None):
		"""Record that this inode's cache state needs to be written."""
		if event:
			trace('cacherecord',"update inode %d (%d)",node.inum,event)
			if node.write_event is None or node.write_event < event:
				node.write_event = event
		else:
			trace('cacherecord',"update inode %d",node.inum)
		self.caches.add(node)
		self.trigger()
Example #18
0
    def server_connected(self, server):
        if self._server is not None and self._server is not server:
            self._server.disconnect()
        trace("remote", "Connected server to %d", self.node_id)
        self._server = server
        self.retry_timeout = INITIAL_RETRY

        if self.echo_timer is not None:
            self.echo_timer.cancel()
        self.echo_timer = reactor.callLater(ECHO_TIMER, self.server_echo)
        self.fs.copier.trigger()
Example #19
0
	def remote_data(self, offset, data):
		"""\
			Data arrives.
			"""
		trace('cache',"%s: recv %d @ %d: known %s", self.inum,len(data),offset, self.known)
		if not self.inum:
			return
		if self.file_closer:
			self.file_closer.reset(5)
		yield self.write(offset,data)
		yield self.has(offset,offset+len(data))
		trace('cache',"%s: done; known %s",self.inum,self.known)
Example #20
0
		def do_work(db):
			if not d:
				s = yield db.Do("insert into `event`(`inode`,`node`,`typ`,`range`) values(${inode},${node},${event},${data})", inode=self.fs.root_inum,node=self.fs.node_id,data=None,event='s' )
				trace('eventrecord',"write SYN %d",s)
			else:
				for event,inode,data in d:
					if not inode.inum:
						continue
					s = yield db.Do("insert into `event`(`inode`,`node`,`typ`,`range`) values(${inode},${node},${event},${data})", inode=inode.inum,node=self.fs.node_id,data=data,event=event )
					inode['event'] = s
				trace('eventrecord',"wrote %d records",len(d))
				self.restart = True # unconditional. That's OK, we want a SYN afterwards.
Example #21
0
	def trim(self,end, do_file=True):
		r = Range()
		if end > 0:
			r.add(0,end)
		trace('fs' if do_file else 'cache',"%s: trim to %d",self.inum,end)
		self.known &= r
		self.available &= r
		self.fs.changer.note(self)
		if do_file:
			yield self.have_file("trim")
			try:
				yield deferToThread(self._trim,0)
			finally:
				self.timeout_file()
Example #22
0
	def _have_file(self, reason):
		with self.lock:
			if self.file:
				return False
			ipath=self._file_path()
			try:
				self.file = open(ipath,"r+")
				trace('fs',"%d: open file %s for %s", self.inum,ipath,reason)
			except EnvironmentError as e:
				if e.errno != errno.ENOENT:
					raise
				self.file = open(ipath,"w+")
				trace('fs',"%d: open file %s for %s (new)", self.inum,ipath,reason)
			return True
Example #23
0
		def do_work2(db):
			queue = DeferredQueue()
			defs = []
			nworkers = len(entries)//5+1
			if nworkers > self.nworkers:
				nworkers = self.nworkers

			for i in range(nworkers):
				d = self.fetch(db,queue)
				d.addErrback(log.err,"fetch()")
				defs.append(d)

			workers = set()
			for id,inum,typ in entries:
				if not self.running:
					break
				trace('copyrun',"%d: %s",inum,typ)

				self.last_entry = id

				if typ == 'd':
					def dt(inum):
						path = build_path(self.fs.store,inum, create=False)
						try:
							os.unlink(path)
						except EnvironmentError as e:
							if e.errno != errno.ENOENT:
								raise
					yield deferToThread(dt,inum)
				else:
					inode = SqlInode(self.fs,inum)
					yield inode._load(db)
					if typ == 'f':
						if inum in workers:
							trace('copyrun',"%d: in workers",inum,typ)
							continue
						workers.add(inum)
						queue.put((id,inode))
					elif typ == 't':
						if inode.cache:
							yield inode.cache.trim(inode.size)
					else:
						raise RuntimeError("Typ '%s' not found (inode %d)" % (typ,inum))
					continue

			for i in range(nworkers):
				queue.put(None)
			yield DeferredList(defs)
Example #24
0
    def call_node(self, dest, name, *a, **k):
        if dest in self.missing_neighbors:
            raise NoLink(dest, "missing")

        try:
            node = self.topology[dest]
            rem = self.remote[node]
        except KeyError:
            trace("error", "NoLink! %s %s %s %s", dest, name, repr(a), repr(k))
            self.missing_neighbors.add(dest)
            raise NoLink(dest, "missing 2")

        if dest == node:
            return getattr(rem, "do_" + name)(*a, **k)
        else:
            return rem.remote_exec(node, name, *a, **k)
Example #25
0
    def connect_timer(self):
        self.retry_timer = None

        if self._server or self._connector or self.node_id is None:
            if self.node_id:
                trace("remote", "connect_timer: inprogress to node %d", self.node_id)
            return
        d = self.connect()

        def grab_nolink(r):
            r.trap(NoLink)
            trace("remote", "connect to node %d failed: %s", self.node_id, r.getErrorMessage())
            self.queue_retry()

        d.addErrback(grab_nolink)
        d.addErrback(lambda r: log.err(r, "Connection timer"))
Example #26
0
    def connect(self):
        """\
			Try to connect to this node's remote side.
			"""
        assert self.node_id is not None
        if self._server:
            return  # already done
        if self._connector:  # in progress: wait for it
            trace("remote", "Chain connect to node %d", self.node_id)
            yield triggeredDefer(self._connector)
            return
        if self.node_id not in self.fs.topology:
            trace("remote", "Not connecting to node %d: no topo", self.node_id)
            return
        if self.fs.topology[self.node_id] != self.node_id:
            trace("remote", "Not connecting to node %d: via %s", self.node_id, self.fs.topology[self.node_id])
            return
        trace("remote", "Connecting to node %d", self.node_id)
        try:
            with self.fs.db() as db:
                try:
                    m, = yield db.DoFn(
                        "select method from updater where src=${src} and dest=${dest}",
                        src=self.fs.node_id,
                        dest=self.node_id,
                    )
                except NoData:
                    raise NoLink(self.node_id, "No Data")
                m = __import__("sqlfuse.connect." + m, fromlist=("NodeClient",))
            m = m.NodeClient(self)
            self._connector = m.connect()
            # Do this to avoid having a single Deferred both in the inline
            # callback chain and as a possible cancellation point
            yield triggeredDefer(self._connector)
            if self._server is None:
                raise NoLink(self.node_id, "No _server")
        except NoLink:
            raise
        except Exception as e:  # no connection
            if isinstance(e, (err.ConnectionRefusedError, NoConnection)):
                trace("remote", "No link to %d, retrying", self.node_id)
            else:
                f = failure.Failure()
                log.err(f, "Connecting remote")
            self.queue_retry()
        finally:
            self._connector = None
Example #27
0
	def _save(self,db):
		if self.inum is None:
			return

		event = self.write_event
		self.write_event = None
		db.call_rolledback(setattr,self,'write_event',event)

		if self.cache_id is None:
			trace('cacherecord',"new for inode %d: ev=%s range=%s",self.inum, event or "-", str(self.known))
			ev1=",event" if event else ""
			ev2=",${event}" if event else ""
			self.cache_id = yield db.Do("insert into cache(cached,inode,node"+ev1+") values (${data},${inode},${node}"+ev2+")", inode=self.inum, node=self.fs.node_id, data=self.known.encode(), event=event)
		else:
			trace('cacherecord',"old for inode %d: ev=%s range=%s",self.inum, event or "-", str(self.known))
			ev=",event=${event}" if event else ""
			yield db.Do("update cache set cached=${data}"+ev+" where id=${cache}", cache=self.cache_id, data=self.known.encode(), event=event, _empty=True)
Example #28
0
		def do_work(db):
			inums = []
			try:
				all_done,n_nodes = yield db.DoFn("select min(event),count(*) from node where root=${root} and id != ${node}", root=self.fs.root_id, node=self.fs.node_id)
			except NoData:
				pass
			else:
				if not n_nodes:
					return
				trace('inodeclean',"%d nodes",n_nodes)
				try:
					last_syn, = yield db.DoFn("select id from event where node=${node} and typ = 's' and id <= ${evt} order by id desc limit 1", node=self.fs.node_id, evt=all_done)
				except NoData:
					pass
				else:
					# remove old inode entries
					trace('inodeclean',"upto %d",last_syn)

					yield db.DoSelect("select inode from event where node=${node} and typ = 'd' and id < ${id}", id=last_syn, node=self.fs.node_id, _callback=inums.append, _empty=True)
					yield db.Do("delete from event where node=${node} and id < ${id}", id=last_syn, node=self.fs.node_id, _empty=True)
			returnValue( inums )
Example #29
0
    def server_echo(self):
        self.echo_timer = reactor.callLater(ECHO_TIMEOUT, self.server_no_echo)

        def get_echo(r):
            if self.echo_timer:
                self.echo_timer.cancel()
            if "ping" in tracers:
                trace("ping", "%s: got echo", str(self.node_id) if self.node_id else "?")
            self.echo_timer = reactor.callLater(ECHO_TIMER, self.server_echo)

        def get_echo_error(r):
            if self.echo_timer:
                self.echo_timer.cancel()
                self.echo_timer = None
            log.err(r, "Echo " + (str(self.node_id) if self.node_id else "?"))
            self.disconnect_retry()

        if "ping" in tracers:
            trace("ping", "%s: send echo", str(self.node_id) if self.node_id else "?")
        d = self.do_echo("ping")
        d.addCallbacks(get_echo, get_echo_error)
Example #30
0
	def run(self, dummy=None):
		"""Background loop. Started via timer from trigger()."""

		self.workerCall = None
		if self.workerDefer:
			if not self.restart:
				self.restart = True
			return
		self.restart = False
		trace('background',"Starting %s", self.__class__.__name__)
		self.workerDefer = maybeDeferred(self.work)
		def done(r):
			self.workerDefer = None
			if self.restart:
				if self.running:
					self.trigger(True)
				else:
					trace('background',"Stopped %s (shutdown)", self.__class__.__name__)
			else:
				trace('background',"Stopped %s", self.__class__.__name__)
		self.workerDefer.addErrback(lambda e: log.err(e,"Running "+self.__class__.__name__))
		self.workerDefer.addBoth(done)