def connectToServer(sockfile=None, host=None, port=None, tmout=3): '''Makes connection to server.''' if host: if not port: raise IKException(ErrorCode.unknownDataFormat, "Port number is not provided.") try: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) except socket.error as msg: raise IKException( 'Failed to create socket. Error code: {0}, Error message: {1}'. format(msg[0], msg[1])) else: if not os.path.exists(sockfile): raise IKException(ErrorCode.objectNotExists, sockfile, "Server's socket file doesn't exist") sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) sock.setsockopt(socket.SOL_SOCKET, socket.SO_PASSCRED, 1) sock.settimeout(tmout) try: if host: sock.connect((host, int(port))) else: sock.connect(sockfile) except OSError as er: raise IKException( ErrorCode.cannotConnectToServer, "Socket error {0}: {1}\nsockfile: {2}; host: {3}; port: {4}". format(er.errno, er.strerror, sockfile, host, port)) return sock
def _getAddOptions(self, cmd): par = cmd["params"] if not par: raise IKException(ErrorCode.unrecognizedSyntax, moreInfo="No items specified.") dest = None addMode = df.noOverwrite if df.DEST in cmd: if not cmd[df.DEST] or not self._isPathValid(path=cmd[df.DEST][0], cmd=cmd): raise IKException( ErrorCode.unrecognizedParameter, moreInfo="Parameter '{0}' must contain a valid section name." .format(df.DEST)) dest = cmd[df.DEST][0] if df.PERS in cmd: if dest: raise IKException( ErrorCode.unrecognizedSyntax, moreInfo= "{0} and {1} cannot be both specified for one command.". format(df.DEST, df.PERS)) dest = PERSPATH if df.FORCE in cmd: addMode = df.overwrite if df.SUM in cmd: addMode = df.sumUp if df.ATTRS in cmd: attrs = util.pairsListToMap(cmd[df.ATTRS]) else: attrs = None return dest, addMode, attrs
def _addItem(self, path: list = None, val: SVal = None, attrs=None, addMode=defs.noOverwrite): if not path: raise IKException(ErrorCode.unknownDataFormat, val, "Token path is not specified") if len(path) == 1 and not self.isWritable(): print("Directory permission: {0}".format(oct(self.st.st_mode))) raise IKException(ErrorCode.permissionDenied, self.pathName(), "Directory is not writeable") curname = path[0] if curname not in self.keys() and (not val or len(path) > 1): self[curname] = getstor(rootStor=self.rootStor, mode=self.st.st_mode) if len(path) == 1: if val: return self.insertNamVal(curname, val, addMode, attrs) return self[curname] return self[curname]._addItem(path=path[1:], val=val, attrs=attrs, addMode=addMode)
def __setitem__(self, key, val): if not hasattr(self, "name"): return dict.__setitem__(self, key, val) if self.name and not key: raise IKException(ErrorCode.unrecognizedParameter, "section name is empty") if key: if "/" in key: raise IKException(ErrorCode.unrecognizedParameter, "section name contains '/'") if key == '.': raise IKException(ErrorCode.unrecognizedParameter, key, "not valid section name") if type(val) == type(self): if self.path: val.setPath(self.path + "/" + self.name) else: if self.name: val.setPath("/" + self.name) else: val.setPath('') else: if not isinstance(val, SVal): val = SVal(self.rootStor, val=val, uid=self.st.st_uid, gid=self.st.st_gid) val.stor = self val.setName(key) if val.storageRef is None: val.setStorageRef(self.storageRef) return super(Stor, self).__setitem__(key, val)
def recvPack( sock, pack ): data = bytearray() packlen = 0 datalen = -1 while packlen != datalen: try: newdata = sock.recv( 4096 ) except OSError as e: raise IKException( ErrorCode.clientConnectionError, moreInfo = str(e) ) data.extend( newdata ) datalen = len( data ) if not packlen and datalen >= 10: # First chunk try: packlen = int( data[:10].decode( 'utf-8' ).strip() ) + 10 except: log.debug( "wrong format: %s" % ( data.decode( 'utf-8' ) ) ) raise IKException( ErrorCode.unknownDataFormat ) if not packlen: raise IKException( ErrorCode.clientConnectionError, moreInfo = "No data received" ) pack.extend( data )
def createPacket_old( cpars : 'in map', bpack : 'out bytearray' ): '''Create a command packet for sending to a regd server.''' # Format: <packlen> <cmd|opt> <numparams> [<paramlen> <param>]... if not cpars.get( "cmd", None ): raise IKException( ErrorCode.unknownDataFormat, "Command parameters must have 'cmd' field." ) bpack.extend( ( cpars["cmd"] + ' ' ).encode( 'utf-8' ) ) if cpars.get( "params" ): bpack.extend( ( str( len( cpars["params"] ) ) + ' ' ).encode( 'utf-8' ) ) for par in cpars["params"]: b = bytearray( par, encoding = 'utf-8' ) bpack.extend( ( str( len( b ) ) + ' ' ).encode( 'utf-8' ) ) bpack.extend( b ) else: bpack.extend( b'0' ) for k, v in cpars.items(): if not k or k in ["cmd", "params"]: continue bpack.extend( ( ' ' + k + ' ' ).encode( 'utf-8' ) ) if v and not( v is True ): bpack.extend( ( str( len( v ) ) + ' ' ).encode( 'utf-8' ) ) for par in v: if type( par ) is str: b = bytearray( par, encoding = 'utf-8' ) elif type( par ) is bytearray or type( par ) is bytes: b = par else: raise IKException( ErrorCode.unknownDataFormat, par ) bpack.extend( ( str( len( b ) ) + ' ' ).encode( 'utf-8' ) ) bpack.extend( b ) else: bpack.extend( b'0' )
def joinPath(p1, p2): if not (p1 or p2): raise IKException( ErrorCode.unsupportedParameterValue, (p1, p2), "Path component is empty.") if p2[0] == '/': raise IKException( ErrorCode.unsupportedParameterValue, p2, "Middle path component begins with /") if p1[-1] != '/': p1 += '/' return p1 + p2
def parsePacket_old( data : 'in bytes', cmdOptions : 'out list', cmdData : 'out list' ) -> str: # Server commands and regd command line commands and options are two different sets: # the latter is the CL user interface, and the former is the internal communication # protocol. # Regd client receives commands from the command line, converts it to server command # syntax, and sends it in the form of command packet to the server. # Each packet contains exactly one command and related options, if any. # Format of command packets: # <COMMAND> <NUMPARAMS> [<PARAMLENGTH> <PARAM>] ... [OPTION NUMPARAMS PARAMLENGTH PARAM]... cmd = None while data: params = [] word, _, data = data.partition( b' ' ) if not data: raise IKException( ErrorCode.unknownDataFormat, word + ' ' + data ) numparams, _, data = data.partition( b' ' ) try: numparams = int( numparams.decode( 'utf-8' ) ) word = word.decode( 'utf-8' ) for _ in range( 0, numparams ): paramlen, _, data = data.partition( b' ' ) paramlen = int( paramlen.decode( 'utf-8' ) ) if not ( paramlen <= len( data ) ): raise if not paramlen: raise param = data[:paramlen] data = data[paramlen + 1:] if word == "binary": params.append( bytes( param ) ) else: params.append( param.decode( 'utf-8' ) ) if word in defs.all_cmds: # One command per packet if cmd: raise cmd = word if len( params ) == 0: cmdData = None else: cmdData.extend( params ) elif word in defs.cmd_opts: cmdOptions.append( ( word, params ) ) else: raise except: raise IKException( ErrorCode.unknownDataFormat, word ) if not cmd: raise IKException( ErrorCode.unknownDataFormat, "Command is not recognized." ) return cmd
def getSerializationFile(self, fhd, read, fpath=None): logsr.debug("getSerFile() is called...") fhcur = fhd.get("cur", None) if not fpath: if self.storageRef == None: raise IKException(ErrorCode.valueNotExists, self.pathName(), "No storage for") fname = self.storageRef.getAttr(SItem.Attrs.persPath.name) if fhcur: fpath = self.getPersStoragePath(os.path.dirname(fhcur.name)) else: fpath = fname if not fpath: raise IKException(ErrorCode.valueNotExists, self.name, "Storage not specified for") sfx = "" if read else ".tmp" if fpath + sfx in fhd: fh = fhd[fpath + sfx] if fh == fhcur: logsr.debug("getSerFile() returns file: {0}".format(fh.name)) return fh else: if not os.path.isabs(fpath): if not fhcur: raise IKException(ErrorCode.valueNotExists, fpath, "Cannot resolve relative path") fpath = os.path.normpath( os.path.join(os.path.dirname(fhcur.name), fpath)) logsr.debug("getSerFile() opens file {0}".format(fpath)) if read: fh = open(fpath, "rb") else: # if os.path.isfile(fpath): # shutil.move(fpath, fpath + ".backup~") fh = open(fpath + sfx, "wb") fhd[fpath + sfx] = fh if not read and fhcur and isinstance(self, Stor): dinc = "\n" + dirInclude + " " + fname + " " + self.name + "\n\n" logsr.debug("getSerFile() writes to fhcur {0}: {1}".format( fhcur, dinc)) fhcur.write(dinc.encode()) if not fh: raise IKException(ErrorCode.programError, self.name, "No storage file") logsr.debug("getSerFile() returns file: {0}".format(fh.name)) return fh
def chAddToken(self, cmd): '''Add item''' dest, addMode, attrs = self._getAddOptions(cmd) cnt = 0 #with self.lock_seriz: for tok in cmd["params"]: if (dest and dest.startswith( BINPATH ) ) or \ tok.startswith( BINPATH ): self.handleBinToken(dest, tok, cmd) continue # Without --dest or --pers options tokens with relative path # are always added to session tokens if not dest and tok[0] != '/': dest = SESPATH binaryVal = None if df.BINARY in cmd: if not cmd[df.BINARY] or len(cmd[df.BINARY]) < cnt + 1: raise IKException(ErrorCode.unknownDataFormat, tok) binaryVal = cmd[df.BINARY][cnt] if not attrs: attrs = {} attrs[stor.SItem.persPathAttrName] = df.BINARY cnt += 1 if dest: tok = joinPath(dest, tok) if binaryVal: tk = None pt = tok else: tk = tok pt = None if not self._isPathValid(tok=tk, path=pt, cmd=cmd): raise IKException(ErrorCode.unsupportedParameterValue, tok[:50], "Path is not valid.") sec = self.fs.addItem(tok=tok, addMode=addMode, binaryVal=binaryVal, attrs=attrs) if sec: sec.markChanged() return composeResponse()
def sendMsgToStorage(cmd): '''Forwarder of messages to storage''' log.debug("In sendMsgToStorage - cmd: {0}".format(cmd)) try: if not util.connLock.acquire(True, 5): raise IKException( ErrorCode.operationFailed, "Failed to acquire the lock for connecting to storage.") log.debug("Sending...") connHere.send(cmd) log.debug("Sent. Receiving...") if connHere.poll(15): ret = connHere.recv() log.debug("In sendMsgToStorage - ret: {0}".format(ret)) else: ret = composeResponse( "0", "Socket timed out or no data to receive.") log.debug("Nothing to receive") util.connLock.release() except Exception as e: ret = composeResponse("0", str(e)) log.error( "In sendMsgToStorage exception received: {0}".format(ret)) return ret
def serialize(self, fhd, read=True, addMode=defs.noOverwrite, relPath=None, fpath=None, indent=0): if self.storageRef and self.storageRef != self: raise IKException( ErrorCode.programError, self.pathName(), "serialize() called on non-root of backing storage.") if read == True: if addMode == defs.clean: self.clear() oldRoot = self.rootStor self.rootStor = self self._serialize(fhd=fhd, read=read, addMode=addMode, relPath=relPath, fpath=fpath) self.rootStor = oldRoot else: self._serialize(fhd=fhd, read=read, addMode=addMode, relPath=relPath, fpath=fpath)
def chReport(self, cmd): '''Report server variables.''' par = cmd["params"][0] retCode = '1' if par == df.ACCESS: resp = "{0}".format(self.srv.acc) elif par == df.REP_STORAGE: path = None if len(cmd["params"]) > 1: if len(cmd["params"] == 2): path = cmd["params"][1] else: raise IKException(ErrorCode.unrecognizedSyntax, " ".join(cmd["params"])) resp = self.fs.stat(path) elif par == df.REP_SERVER: # TODO: bs = getShared("bytesSent") br = getShared("bytesReceived") resp = "Bytes sent: {0}\n".format(bs) resp += "Bytes received: {0}\n".format(br) elif par == df.REP_COMMANDS: resp = util.printMap(self.info["cmd"], 0) pass elif par == df.DATAFILE: resp = self.fs.datafile else: retCode = '0' resp = "Unrecognized command parameter: " + par return composeResponse(retCode, resp)
def numItems(self, itemType=EnumMode.both): cnt = None if itemType == EnumMode.both: cnt = len(self) elif itemType in (EnumMode.tokens, EnumMode.tokensAll): cnt = 0 for k in self.keys(): if type(self[k]) != type(self): cnt += 1 elif itemType == EnumMode.tokensAll: cnt += self[k].numItems(EnumMode.tokensAll) elif itemType in (EnumMode.sections, EnumMode.sectionsAll): cnt = 0 for k in self.keys(): if type(self[k]) == type(self): cnt += 1 if itemType == EnumMode.sectionsAll: cnt += self[k].numItems(EnumMode.sectionsAll) elif itemType == EnumMode.all: cnt = self.numItems(EnumMode.tokensAll) + self.numItems( EnumMode.sectionsAll) else: raise IKException() return cnt
def createPacket( cpars : "in map" ): '''Create a command packet for sending to a regd server.''' if not cpars.get( "cmd", None ): raise IKException( ErrorCode.unknownDataFormat, "Command parameters must have 'cmd' field." ) #return pickle.dumps( cpars, -1) fh = io.StringIO() js = dtl.Jsonator() json.dump( cpars, fh, default = js.tojson ) return fh.getvalue().encode()
def flush(): nonlocal curTok, curPath if curTok: mtok = reTok.match(curTok) if not mtok or mtok.lastindex != 2: raise IKException(ErrorCode.unknownDataFormat, l, "Cannot parse token.") stok.addToken(curPath + curTok, addMode) curTok = None
def getItem(self, path): '''Return a token or section with the given path.''' if type(path) is str: lpath, _ = parse_token(path, bVal=defs.auto) return self.getItem(lpath) else: if path: if path[0] not in self.keys(): raise IKException(ErrorCode.objectNotExists, path[0], "Item doesn't exist") if len(path) > 1: return self[path[0]].getItem(path[1:]) else: return self[path[0]] raise IKException(ErrorCode.nameError, "getItem: path is empty") return self
def getShared(nam): tok = util.joinPath("/_sys", nam) cmd = {"cmd": df.GET_ITEM, "params": [tok], "internal": True} ret = CmdSwitcher.switchCmd(cmd) ret = util.parsePacket(ret) if cmd['res'] == 0: raise IKException(ErrorCode.programError, ret[1]) return ret[1]
def unpackObject( _data, seqout ): data = _data[0] hdr, _, data = data.partition( b' ' ) ot = hdr[0:1].decode( 'utf-8' ) # object type _data[0] = data if ot == 'N': seqout.append( None ) return sz = int( hdr[1:] ) # object size / number of items if ot == 'I': arr = data[:sz] data = data[sz:] seqout.append( int( arr.decode( 'utf-8' ) ) ) _data[0] = data elif ot == 'F': arr = data[:sz] data = data[sz:] seqout.append( float( arr.decode( 'utf-8' ) ) ) _data[0] = data elif ot == 'B': arr = data[:sz] data = data[sz:] seqout.append( arr ) _data[0] = data elif ot == 'S': arr = data[:sz] data = data[sz:] seqout.append( arr.decode( 'utf-8' ) ) _data[0] = data elif ot == 'L': l = [] # seqout.append( l ) for _ in range( 0, sz ): unpackObject( _data, l ) if len( l ) == 1: seqout.append( l[0] ) elif len( l ) > 1: seqout.append( l ) elif ot == 'D': m = {} seqout.append( m ) for _ in range( 0, int( sz / 2 ) ): l = [] unpackObject( _data, l ) if len( l ) > 1: raise IKException( ErrorCode.unknownDataFormat, resp[:20], moreInfo = "Dictionary key is a compound object" ) k = l[0] l = [] unpackObject( _data, l ) if len( l ) == 1: v = l[0] else: v = l m[k] = v
def __getitem1__(self, key): item = super(Stor, self).__getitem__(key) if isinstance(item, baseSectType): return item elif isinstance(item, baseValType): return item.val else: raise IKException( ErrorCode.programError, "Storage item type is: {0}".format(str(type(item))))
def startStorage(acc, datafile, binsectfile): '''Starts storage in new process''' connHere, connThere = Pipe(True) sigHere, sigThere = socketpair() sigHere.setblocking(False) sigThere.setblocking(False) os.set_inheritable(sigThere.fileno(), True) util.connLock = Lock() def sendMsgToStorage(cmd): '''Forwarder of messages to storage''' log.debug("In sendMsgToStorage - cmd: {0}".format(cmd)) try: if not util.connLock.acquire(True, 5): raise IKException( ErrorCode.operationFailed, "Failed to acquire the lock for connecting to storage.") log.debug("Sending...") connHere.send(cmd) log.debug("Sent. Receiving...") if connHere.poll(15): ret = connHere.recv() log.debug("In sendMsgToStorage - ret: {0}".format(ret)) else: ret = composeResponse( "0", "Socket timed out or no data to receive.") log.debug("Nothing to receive") util.connLock.release() except Exception as e: ret = composeResponse("0", str(e)) log.error( "In sendMsgToStorage exception received: {0}".format(ret)) return ret FS.registerGroupHandlers(sendMsgToStorage) log.info("Starting storage process...") # Timeout for sending from fs back to connectionHandler #connThere.settimeout( 5 ) p = Process(target=FS.start_loop, args=(connThere, sigThere, acc, datafile, binsectfile), name="Regd Storage") p.start() time.sleep(1) if p.is_alive(): log.info("Storage started OK.") else: log.info("Failed to start storage.") raise IKException(ErrorCode.operationFailed, "Failed to start storage.") return connHere, sigHere
def setPath(self, path): self.path = path for v in self.values(): if self.path: v.setPath(self.path + "/" + self.name) else: if self.name: v.setPath("/" + self.name) else: raise IKException(ErrorCode.programError, "Storage item has no name.")
def removeItem(self, tok): path, _ = parse_token(tok, bVal=defs.auto) if len(path) == 1: if not (self.isWritable() and self[path[0]].isWritable()): raise IKException(ErrorCode.permissionDenied, tok, "Item is read-only") del self[path[0]] return self sec = self.getItem(path[:-1]) return sec.removeItem(path[-1])
def chShowLog(self, cmd): '''Show server log''' par = cmd["params"] n = 20 if par: try: n = int(par[0]) except ValueError: raise IKException( ErrorCode.unrecognizedParameter, par[0], moreInfo="Number of lines only must contain digits.") return composeResponse('1', util.getLog(n))
def handleDataDirective(direc, val): '''Handles data directives.''' logsr.debug("handleDataDirective() is called...") nonlocal fhd, read, addMode if direc == defs.dirInclude: largs = app.splitArgs(val) if len(largs) == 2: file, sect = largs else: file = largs[0] sect = None fname = file.strip(' \n') if not os.path.isabs(fname): if not sect: sect = fname fpath = os.path.normpath( os.path.join(os.path.dirname(fh.name), fname)) else: fpath = fname if not os.path.isfile(fpath): raise IKException(ErrorCode.objectNotExists, fpath, "Included file not found:") if fpath in fhd: # File is read more than once. # fh = fhd[fpath] raise IKException( ErrorCode.valueAlreadyExists, fpath, "Data file is referenced more than once") logsr.debug("haDaDir() opens file {0}".format(fpath)) fhd[fpath] = open(fpath, "rb") logsr.debug("haDaDir() creates section {0}".format( self.rootStor.pathName() + "/" + sect)) sec = self.rootStor.addItem(tok=sect) logsr.debug( "haDaDir() sets section's persPath to {0}".format(fpath)) sec.setAttr(SItem.Attrs.persPath.name, fname) logsr.debug("haDaDir() calls section's serialize()") sec._serialize(fhd=fhd, read=read, addMode=addMode)
def read_tokens_from_file(filename, tok, addMode=defs.noOverwrite): '''Reads tokens from a text file. If the file contains several sections, their names should be specified in the file in square brackets in the beginning of each section. If tokens are loaded into a single section, its name can be specified either in the file or if the section name is not specified, tokens are loaded into the root section. ''' if not os.path.exists(filename): raise IKException(ErrorCode.objectNotExists, filename, "File with tokens is not found.") with open(filename) as f: lines = f.readlines() read_tokens_from_lines(lines, tok, addMode)
def parse_server_name( server_string ): atuser = None servername = None if server_string: if server_string.find( '@' ) != -1: atuser = server_string[0:server_string.index( '@' )] servername = server_string[( len( atuser ) + 1 ):] else: servername = server_string if not servername: servername = None else: if len( servername ) > 32: raise IKException( ErrorCode.unknownDataFormat, server_string, "The server name must not exceed 32 characters." ) for c in "\\/:@": if c in servername: raise IKException( ErrorCode.unknownDataFormat, server_string, "Server name contains not permitted character: '{0}'".format( c ) ) return ( atuser, servername )
def setShared(nam, val, mode=None): '''Assign value to internal shared data.''' "Returns json packet" tok = "{0}={1}".format(util.joinPath("/_sys", nam), val) cmd = {"cmd": df.ADD_TOKEN, "params": [tok], "internal": True} if mode == df.FORCE: cmd[df.FORCE] = True elif mode == df.SUM: cmd[df.SUM] = True ret = CmdSwitcher.switchCmd(cmd) if cmd['res'] == 0: ret = util.parsePacket(ret) #log.error( "Set shared data '%s' failed: %s" % ( nam, l[1] ) ) raise IKException(ErrorCode.programError, ret[1])
def chCreateSection(self, cmd): '''Create section''' feeder = self._getItemFeeder(cmd) for i in feeder: if not self._isPathValid(path=i, cmd=cmd): raise IKException(ErrorCode.unsupportedParameterValue, i, "Path is not valid") sec = self.fs.addItem(i) if df.ATTRS in cmd: sec.setAttrs(cmd[df.ATTRS]) sec.readFromFile(updateFromStorage=True) sec.markChanged() return composeResponse()
def addItem(self, tok: str = None, path: list = None, val=None, binaryVal=None, attrs=None, addMode=defs.noOverwrite): '''Add a token or section to stor''' if tok: if not binaryVal: path, val = parse_token(tok, bVal=defs.auto) else: path, _ = parse_token(tok, bVal=defs.no) val = binaryVal else: if val and binaryVal: raise IKException(ErrorCode.unsupportedParameterValue, val, "Both value and binary value are specified") if binaryVal: val = binaryVal if not path: raise IKException(ErrorCode.unknownDataFormat, val, "Token name is not specified") if val: # Storage contains items of two types: SVal and Stor: if not isinstance(val, SVal) and not isinstance(val, dict): nam = path[-1] val = SVal(rootStor=self.rootStor, name=nam, attrs=attrs, val=val) return self._addItem(path=path, val=val, attrs=attrs, addMode=addMode)