def connectToServer(sockfile=None, host=None, port=None, tmout=3):
'''Makes connection to server.'''
if host:
if not port:
raise IKException(ErrorCode.unknownDataFormat,
"Port number is not provided.")
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
except socket.error as msg:
raise IKException(
'Failed to create socket. Error code: {0}, Error message: {1}'.
format(msg[0], msg[1]))
else:
if not os.path.exists(sockfile):
raise IKException(ErrorCode.objectNotExists, sockfile,
"Server's socket file doesn't exist")
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_PASSCRED, 1)
sock.settimeout(tmout)
try:
if host:
sock.connect((host, int(port)))
else:
sock.connect(sockfile)
except OSError as er:
raise IKException(
ErrorCode.cannotConnectToServer,
"Socket error {0}: {1}\nsockfile: {2}; host: {3}; port: {4}".
format(er.errno, er.strerror, sockfile, host, port))
return sock
def _getAddOptions(self, cmd):
par = cmd["params"]
if not par:
raise IKException(ErrorCode.unrecognizedSyntax,
moreInfo="No items specified.")
dest = None
addMode = df.noOverwrite
if df.DEST in cmd:
if not cmd[df.DEST] or not self._isPathValid(path=cmd[df.DEST][0],
cmd=cmd):
raise IKException(
ErrorCode.unrecognizedParameter,
moreInfo="Parameter '{0}' must contain a valid section name."
.format(df.DEST))
dest = cmd[df.DEST][0]
if df.PERS in cmd:
if dest:
raise IKException(
ErrorCode.unrecognizedSyntax,
moreInfo=
"{0} and {1} cannot be both specified for one command.".
format(df.DEST, df.PERS))
dest = PERSPATH
if df.FORCE in cmd:
addMode = df.overwrite
if df.SUM in cmd:
addMode = df.sumUp
if df.ATTRS in cmd:
attrs = util.pairsListToMap(cmd[df.ATTRS])
else:
attrs = None
return dest, addMode, attrs
def _addItem(self,
path: list = None,
val: SVal = None,
attrs=None,
addMode=defs.noOverwrite):
if not path:
raise IKException(ErrorCode.unknownDataFormat, val,
"Token path is not specified")
if len(path) == 1 and not self.isWritable():
print("Directory permission: {0}".format(oct(self.st.st_mode)))
raise IKException(ErrorCode.permissionDenied, self.pathName(),
"Directory is not writeable")
curname = path[0]
if curname not in self.keys() and (not val or len(path) > 1):
self[curname] = getstor(rootStor=self.rootStor,
mode=self.st.st_mode)
if len(path) == 1:
if val:
return self.insertNamVal(curname, val, addMode, attrs)
return self[curname]
return self[curname]._addItem(path=path[1:],
val=val,
attrs=attrs,
addMode=addMode)
def __setitem__(self, key, val):
if not hasattr(self, "name"):
return dict.__setitem__(self, key, val)
if self.name and not key:
raise IKException(ErrorCode.unrecognizedParameter,
"section name is empty")
if key:
if "/" in key:
raise IKException(ErrorCode.unrecognizedParameter,
"section name contains '/'")
if key == '.':
raise IKException(ErrorCode.unrecognizedParameter, key,
"not valid section name")
if type(val) == type(self):
if self.path:
val.setPath(self.path + "/" + self.name)
else:
if self.name:
val.setPath("/" + self.name)
else:
val.setPath('')
else:
if not isinstance(val, SVal):
val = SVal(self.rootStor,
val=val,
uid=self.st.st_uid,
gid=self.st.st_gid)
val.stor = self
val.setName(key)
if val.storageRef is None:
val.setStorageRef(self.storageRef)
return super(Stor, self).__setitem__(key, val)
def recvPack( sock, pack ): data = bytearray() packlen = 0 datalen = -1 while packlen != datalen: try: newdata = sock.recv( 4096 ) except OSError as e: raise IKException( ErrorCode.clientConnectionError, moreInfo = str(e) ) data.extend( newdata ) datalen = len( data ) if not packlen and datalen >= 10: # First chunk try: packlen = int( data[:10].decode( 'utf-8' ).strip() ) + 10 except: log.debug( "wrong format: %s" % ( data.decode( 'utf-8' ) ) ) raise IKException( ErrorCode.unknownDataFormat ) if not packlen: raise IKException( ErrorCode.clientConnectionError, moreInfo = "No data received" ) pack.extend( data )
def createPacket_old( cpars : 'in map', bpack : 'out bytearray' ): '''Create a command packet for sending to a regd server.''' # Format: <packlen> <cmd|opt> <numparams> [<paramlen> <param>]... if not cpars.get( "cmd", None ): raise IKException( ErrorCode.unknownDataFormat, "Command parameters must have 'cmd' field." ) bpack.extend( ( cpars["cmd"] + ' ' ).encode( 'utf-8' ) ) if cpars.get( "params" ): bpack.extend( ( str( len( cpars["params"] ) ) + ' ' ).encode( 'utf-8' ) ) for par in cpars["params"]: b = bytearray( par, encoding = 'utf-8' ) bpack.extend( ( str( len( b ) ) + ' ' ).encode( 'utf-8' ) ) bpack.extend( b ) else: bpack.extend( b'0' ) for k, v in cpars.items(): if not k or k in ["cmd", "params"]: continue bpack.extend( ( ' ' + k + ' ' ).encode( 'utf-8' ) ) if v and not( v is True ): bpack.extend( ( str( len( v ) ) + ' ' ).encode( 'utf-8' ) ) for par in v: if type( par ) is str: b = bytearray( par, encoding = 'utf-8' ) elif type( par ) is bytearray or type( par ) is bytes: b = par else: raise IKException( ErrorCode.unknownDataFormat, par ) bpack.extend( ( str( len( b ) ) + ' ' ).encode( 'utf-8' ) ) bpack.extend( b ) else: bpack.extend( b'0' )
def joinPath(p1, p2): if not (p1 or p2): raise IKException( ErrorCode.unsupportedParameterValue, (p1, p2), "Path component is empty.") if p2[0] == '/': raise IKException( ErrorCode.unsupportedParameterValue, p2, "Middle path component begins with /") if p1[-1] != '/': p1 += '/' return p1 + p2
def parsePacket_old( data : 'in bytes', cmdOptions : 'out list', cmdData : 'out list' ) -> str: # Server commands and regd command line commands and options are two different sets: # the latter is the CL user interface, and the former is the internal communication # protocol. # Regd client receives commands from the command line, converts it to server command # syntax, and sends it in the form of command packet to the server. # Each packet contains exactly one command and related options, if any. # Format of command packets: # <COMMAND> <NUMPARAMS> [<PARAMLENGTH> <PARAM>] ... [OPTION NUMPARAMS PARAMLENGTH PARAM]... cmd = None while data: params = [] word, _, data = data.partition( b' ' ) if not data: raise IKException( ErrorCode.unknownDataFormat, word + ' ' + data ) numparams, _, data = data.partition( b' ' ) try: numparams = int( numparams.decode( 'utf-8' ) ) word = word.decode( 'utf-8' ) for _ in range( 0, numparams ): paramlen, _, data = data.partition( b' ' ) paramlen = int( paramlen.decode( 'utf-8' ) ) if not ( paramlen <= len( data ) ): raise if not paramlen: raise param = data[:paramlen] data = data[paramlen + 1:] if word == "binary": params.append( bytes( param ) ) else: params.append( param.decode( 'utf-8' ) ) if word in defs.all_cmds: # One command per packet if cmd: raise cmd = word if len( params ) == 0: cmdData = None else: cmdData.extend( params ) elif word in defs.cmd_opts: cmdOptions.append( ( word, params ) ) else: raise except: raise IKException( ErrorCode.unknownDataFormat, word ) if not cmd: raise IKException( ErrorCode.unknownDataFormat, "Command is not recognized." ) return cmd
def getSerializationFile(self, fhd, read, fpath=None):
logsr.debug("getSerFile() is called...")
fhcur = fhd.get("cur", None)
if not fpath:
if self.storageRef == None:
raise IKException(ErrorCode.valueNotExists, self.pathName(),
"No storage for")
fname = self.storageRef.getAttr(SItem.Attrs.persPath.name)
if fhcur:
fpath = self.getPersStoragePath(os.path.dirname(fhcur.name))
else:
fpath = fname
if not fpath:
raise IKException(ErrorCode.valueNotExists, self.name,
"Storage not specified for")
sfx = "" if read else ".tmp"
if fpath + sfx in fhd:
fh = fhd[fpath + sfx]
if fh == fhcur:
logsr.debug("getSerFile() returns file: {0}".format(fh.name))
return fh
else:
if not os.path.isabs(fpath):
if not fhcur:
raise IKException(ErrorCode.valueNotExists, fpath,
"Cannot resolve relative path")
fpath = os.path.normpath(
os.path.join(os.path.dirname(fhcur.name), fpath))
logsr.debug("getSerFile() opens file {0}".format(fpath))
if read:
fh = open(fpath, "rb")
else:
# if os.path.isfile(fpath):
# shutil.move(fpath, fpath + ".backup~")
fh = open(fpath + sfx, "wb")
fhd[fpath + sfx] = fh
if not read and fhcur and isinstance(self, Stor):
dinc = "\n" + dirInclude + " " + fname + " " + self.name + "\n\n"
logsr.debug("getSerFile() writes to fhcur {0}: {1}".format(
fhcur, dinc))
fhcur.write(dinc.encode())
if not fh:
raise IKException(ErrorCode.programError, self.name,
"No storage file")
logsr.debug("getSerFile() returns file: {0}".format(fh.name))
return fh
def chAddToken(self, cmd):
'''Add item'''
dest, addMode, attrs = self._getAddOptions(cmd)
cnt = 0
#with self.lock_seriz:
for tok in cmd["params"]:
if (dest and dest.startswith( BINPATH ) ) or \
tok.startswith( BINPATH ):
self.handleBinToken(dest, tok, cmd)
continue
# Without --dest or --pers options tokens with relative path
# are always added to session tokens
if not dest and tok[0] != '/':
dest = SESPATH
binaryVal = None
if df.BINARY in cmd:
if not cmd[df.BINARY] or len(cmd[df.BINARY]) < cnt + 1:
raise IKException(ErrorCode.unknownDataFormat, tok)
binaryVal = cmd[df.BINARY][cnt]
if not attrs:
attrs = {}
attrs[stor.SItem.persPathAttrName] = df.BINARY
cnt += 1
if dest:
tok = joinPath(dest, tok)
if binaryVal:
tk = None
pt = tok
else:
tk = tok
pt = None
if not self._isPathValid(tok=tk, path=pt, cmd=cmd):
raise IKException(ErrorCode.unsupportedParameterValue,
tok[:50], "Path is not valid.")
sec = self.fs.addItem(tok=tok,
addMode=addMode,
binaryVal=binaryVal,
attrs=attrs)
if sec: sec.markChanged()
return composeResponse()
def sendMsgToStorage(cmd):
'''Forwarder of messages to storage'''
log.debug("In sendMsgToStorage - cmd: {0}".format(cmd))
try:
if not util.connLock.acquire(True, 5):
raise IKException(
ErrorCode.operationFailed,
"Failed to acquire the lock for connecting to storage.")
log.debug("Sending...")
connHere.send(cmd)
log.debug("Sent. Receiving...")
if connHere.poll(15):
ret = connHere.recv()
log.debug("In sendMsgToStorage - ret: {0}".format(ret))
else:
ret = composeResponse(
"0", "Socket timed out or no data to receive.")
log.debug("Nothing to receive")
util.connLock.release()
except Exception as e:
ret = composeResponse("0", str(e))
log.error(
"In sendMsgToStorage exception received: {0}".format(ret))
return ret
def serialize(self,
fhd,
read=True,
addMode=defs.noOverwrite,
relPath=None,
fpath=None,
indent=0):
if self.storageRef and self.storageRef != self:
raise IKException(
ErrorCode.programError, self.pathName(),
"serialize() called on non-root of backing storage.")
if read == True:
if addMode == defs.clean:
self.clear()
oldRoot = self.rootStor
self.rootStor = self
self._serialize(fhd=fhd,
read=read,
addMode=addMode,
relPath=relPath,
fpath=fpath)
self.rootStor = oldRoot
else:
self._serialize(fhd=fhd,
read=read,
addMode=addMode,
relPath=relPath,
fpath=fpath)
def chReport(self, cmd):
'''Report server variables.'''
par = cmd["params"][0]
retCode = '1'
if par == df.ACCESS:
resp = "{0}".format(self.srv.acc)
elif par == df.REP_STORAGE:
path = None
if len(cmd["params"]) > 1:
if len(cmd["params"] == 2):
path = cmd["params"][1]
else:
raise IKException(ErrorCode.unrecognizedSyntax,
" ".join(cmd["params"]))
resp = self.fs.stat(path)
elif par == df.REP_SERVER:
# TODO:
bs = getShared("bytesSent")
br = getShared("bytesReceived")
resp = "Bytes sent: {0}\n".format(bs)
resp += "Bytes received: {0}\n".format(br)
elif par == df.REP_COMMANDS:
resp = util.printMap(self.info["cmd"], 0)
pass
elif par == df.DATAFILE:
resp = self.fs.datafile
else:
retCode = '0'
resp = "Unrecognized command parameter: " + par
return composeResponse(retCode, resp)
def numItems(self, itemType=EnumMode.both):
cnt = None
if itemType == EnumMode.both:
cnt = len(self)
elif itemType in (EnumMode.tokens, EnumMode.tokensAll):
cnt = 0
for k in self.keys():
if type(self[k]) != type(self):
cnt += 1
elif itemType == EnumMode.tokensAll:
cnt += self[k].numItems(EnumMode.tokensAll)
elif itemType in (EnumMode.sections, EnumMode.sectionsAll):
cnt = 0
for k in self.keys():
if type(self[k]) == type(self):
cnt += 1
if itemType == EnumMode.sectionsAll:
cnt += self[k].numItems(EnumMode.sectionsAll)
elif itemType == EnumMode.all:
cnt = self.numItems(EnumMode.tokensAll) + self.numItems(
EnumMode.sectionsAll)
else:
raise IKException()
return cnt
def createPacket( cpars : "in map" ): '''Create a command packet for sending to a regd server.''' if not cpars.get( "cmd", None ): raise IKException( ErrorCode.unknownDataFormat, "Command parameters must have 'cmd' field." ) #return pickle.dumps( cpars, -1) fh = io.StringIO() js = dtl.Jsonator() json.dump( cpars, fh, default = js.tojson ) return fh.getvalue().encode()
def flush():
nonlocal curTok, curPath
if curTok:
mtok = reTok.match(curTok)
if not mtok or mtok.lastindex != 2:
raise IKException(ErrorCode.unknownDataFormat, l,
"Cannot parse token.")
stok.addToken(curPath + curTok, addMode)
curTok = None
def getItem(self, path):
'''Return a token or section with the given path.'''
if type(path) is str:
lpath, _ = parse_token(path, bVal=defs.auto)
return self.getItem(lpath)
else:
if path:
if path[0] not in self.keys():
raise IKException(ErrorCode.objectNotExists, path[0],
"Item doesn't exist")
if len(path) > 1:
return self[path[0]].getItem(path[1:])
else:
return self[path[0]]
raise IKException(ErrorCode.nameError, "getItem: path is empty")
return self
def getShared(nam):
tok = util.joinPath("/_sys", nam)
cmd = {"cmd": df.GET_ITEM, "params": [tok], "internal": True}
ret = CmdSwitcher.switchCmd(cmd)
ret = util.parsePacket(ret)
if cmd['res'] == 0:
raise IKException(ErrorCode.programError, ret[1])
return ret[1]
def unpackObject( _data, seqout ):
data = _data[0]
hdr, _, data = data.partition( b' ' )
ot = hdr[0:1].decode( 'utf-8' ) # object type
_data[0] = data
if ot == 'N':
seqout.append( None )
return
sz = int( hdr[1:] ) # object size / number of items
if ot == 'I':
arr = data[:sz]
data = data[sz:]
seqout.append( int( arr.decode( 'utf-8' ) ) )
_data[0] = data
elif ot == 'F':
arr = data[:sz]
data = data[sz:]
seqout.append( float( arr.decode( 'utf-8' ) ) )
_data[0] = data
elif ot == 'B':
arr = data[:sz]
data = data[sz:]
seqout.append( arr )
_data[0] = data
elif ot == 'S':
arr = data[:sz]
data = data[sz:]
seqout.append( arr.decode( 'utf-8' ) )
_data[0] = data
elif ot == 'L':
l = []
# seqout.append( l )
for _ in range( 0, sz ):
unpackObject( _data, l )
if len( l ) == 1:
seqout.append( l[0] )
elif len( l ) > 1:
seqout.append( l )
elif ot == 'D':
m = {}
seqout.append( m )
for _ in range( 0, int( sz / 2 ) ):
l = []
unpackObject( _data, l )
if len( l ) > 1:
raise IKException( ErrorCode.unknownDataFormat, resp[:20],
moreInfo = "Dictionary key is a compound object" )
k = l[0]
l = []
unpackObject( _data, l )
if len( l ) == 1:
v = l[0]
else:
v = l
m[k] = v
def __getitem1__(self, key):
item = super(Stor, self).__getitem__(key)
if isinstance(item, baseSectType):
return item
elif isinstance(item, baseValType):
return item.val
else:
raise IKException(
ErrorCode.programError,
"Storage item type is: {0}".format(str(type(item))))
def startStorage(acc, datafile, binsectfile):
'''Starts storage in new process'''
connHere, connThere = Pipe(True)
sigHere, sigThere = socketpair()
sigHere.setblocking(False)
sigThere.setblocking(False)
os.set_inheritable(sigThere.fileno(), True)
util.connLock = Lock()
def sendMsgToStorage(cmd):
'''Forwarder of messages to storage'''
log.debug("In sendMsgToStorage - cmd: {0}".format(cmd))
try:
if not util.connLock.acquire(True, 5):
raise IKException(
ErrorCode.operationFailed,
"Failed to acquire the lock for connecting to storage.")
log.debug("Sending...")
connHere.send(cmd)
log.debug("Sent. Receiving...")
if connHere.poll(15):
ret = connHere.recv()
log.debug("In sendMsgToStorage - ret: {0}".format(ret))
else:
ret = composeResponse(
"0", "Socket timed out or no data to receive.")
log.debug("Nothing to receive")
util.connLock.release()
except Exception as e:
ret = composeResponse("0", str(e))
log.error(
"In sendMsgToStorage exception received: {0}".format(ret))
return ret
FS.registerGroupHandlers(sendMsgToStorage)
log.info("Starting storage process...")
# Timeout for sending from fs back to connectionHandler
#connThere.settimeout( 5 )
p = Process(target=FS.start_loop,
args=(connThere, sigThere, acc, datafile, binsectfile),
name="Regd Storage")
p.start()
time.sleep(1)
if p.is_alive():
log.info("Storage started OK.")
else:
log.info("Failed to start storage.")
raise IKException(ErrorCode.operationFailed,
"Failed to start storage.")
return connHere, sigHere
def setPath(self, path):
self.path = path
for v in self.values():
if self.path:
v.setPath(self.path + "/" + self.name)
else:
if self.name:
v.setPath("/" + self.name)
else:
raise IKException(ErrorCode.programError,
"Storage item has no name.")
def removeItem(self, tok):
path, _ = parse_token(tok, bVal=defs.auto)
if len(path) == 1:
if not (self.isWritable() and self[path[0]].isWritable()):
raise IKException(ErrorCode.permissionDenied, tok,
"Item is read-only")
del self[path[0]]
return self
sec = self.getItem(path[:-1])
return sec.removeItem(path[-1])
def chShowLog(self, cmd):
'''Show server log'''
par = cmd["params"]
n = 20
if par:
try:
n = int(par[0])
except ValueError:
raise IKException(
ErrorCode.unrecognizedParameter,
par[0],
moreInfo="Number of lines only must contain digits.")
return composeResponse('1', util.getLog(n))
def handleDataDirective(direc, val):
'''Handles data directives.'''
logsr.debug("handleDataDirective() is called...")
nonlocal fhd, read, addMode
if direc == defs.dirInclude:
largs = app.splitArgs(val)
if len(largs) == 2:
file, sect = largs
else:
file = largs[0]
sect = None
fname = file.strip(' \n')
if not os.path.isabs(fname):
if not sect: sect = fname
fpath = os.path.normpath(
os.path.join(os.path.dirname(fh.name), fname))
else:
fpath = fname
if not os.path.isfile(fpath):
raise IKException(ErrorCode.objectNotExists, fpath,
"Included file not found:")
if fpath in fhd:
# File is read more than once.
# fh = fhd[fpath]
raise IKException(
ErrorCode.valueAlreadyExists, fpath,
"Data file is referenced more than once")
logsr.debug("haDaDir() opens file {0}".format(fpath))
fhd[fpath] = open(fpath, "rb")
logsr.debug("haDaDir() creates section {0}".format(
self.rootStor.pathName() + "/" + sect))
sec = self.rootStor.addItem(tok=sect)
logsr.debug(
"haDaDir() sets section's persPath to {0}".format(fpath))
sec.setAttr(SItem.Attrs.persPath.name, fname)
logsr.debug("haDaDir() calls section's serialize()")
sec._serialize(fhd=fhd, read=read, addMode=addMode)
def read_tokens_from_file(filename, tok, addMode=defs.noOverwrite):
'''Reads tokens from a text file. If the file contains several sections, their names
should be specified in the file in square brackets in the beginning of each section.
If tokens are loaded into a single section, its name can be specified either in the
file or if the section name is not specified, tokens are loaded into the root section.
'''
if not os.path.exists(filename):
raise IKException(ErrorCode.objectNotExists, filename,
"File with tokens is not found.")
with open(filename) as f:
lines = f.readlines()
read_tokens_from_lines(lines, tok, addMode)
def parse_server_name( server_string ):
atuser = None
servername = None
if server_string:
if server_string.find( '@' ) != -1:
atuser = server_string[0:server_string.index( '@' )]
servername = server_string[( len( atuser ) + 1 ):]
else:
servername = server_string
if not servername:
servername = None
else:
if len( servername ) > 32:
raise IKException( ErrorCode.unknownDataFormat, server_string,
"The server name must not exceed 32 characters." )
for c in "\\/:@":
if c in servername:
raise IKException( ErrorCode.unknownDataFormat, server_string,
"Server name contains not permitted character: '{0}'".format( c ) )
return ( atuser, servername )
def setShared(nam, val, mode=None):
'''Assign value to internal shared data.'''
"Returns json packet"
tok = "{0}={1}".format(util.joinPath("/_sys", nam), val)
cmd = {"cmd": df.ADD_TOKEN, "params": [tok], "internal": True}
if mode == df.FORCE:
cmd[df.FORCE] = True
elif mode == df.SUM:
cmd[df.SUM] = True
ret = CmdSwitcher.switchCmd(cmd)
if cmd['res'] == 0:
ret = util.parsePacket(ret)
#log.error( "Set shared data '%s' failed: %s" % ( nam, l[1] ) )
raise IKException(ErrorCode.programError, ret[1])
def chCreateSection(self, cmd):
'''Create section'''
feeder = self._getItemFeeder(cmd)
for i in feeder:
if not self._isPathValid(path=i, cmd=cmd):
raise IKException(ErrorCode.unsupportedParameterValue, i,
"Path is not valid")
sec = self.fs.addItem(i)
if df.ATTRS in cmd:
sec.setAttrs(cmd[df.ATTRS])
sec.readFromFile(updateFromStorage=True)
sec.markChanged()
return composeResponse()
def addItem(self,
tok: str = None,
path: list = None,
val=None,
binaryVal=None,
attrs=None,
addMode=defs.noOverwrite):
'''Add a token or section to stor'''
if tok:
if not binaryVal:
path, val = parse_token(tok, bVal=defs.auto)
else:
path, _ = parse_token(tok, bVal=defs.no)
val = binaryVal
else:
if val and binaryVal:
raise IKException(ErrorCode.unsupportedParameterValue, val,
"Both value and binary value are specified")
if binaryVal:
val = binaryVal
if not path:
raise IKException(ErrorCode.unknownDataFormat, val,
"Token name is not specified")
if val:
# Storage contains items of two types: SVal and Stor:
if not isinstance(val, SVal) and not isinstance(val, dict):
nam = path[-1]
val = SVal(rootStor=self.rootStor,
name=nam,
attrs=attrs,
val=val)
return self._addItem(path=path, val=val, attrs=attrs, addMode=addMode)