def create_tuplespace(self, req, msgid, message, data):
# return a new tuplespace id
ts = "S%s" % (guid.generate(), )
local_ts.newTupleSpace(ts)
local_ts.addReference(ts, utils.getProcessIdFromThreadId(str(data[0])))
req.send(msgid, ("RESULT_STRING", ts))
def __init__(self, transientdb, persistentdb):
self.service = None
self.tdb = transientdb
self.pdb = persistentdb
socket = gevent.socket.socket()
guid = Guid.generate()
self.stub = rpc.RpcStub(guid, socket, ChunkServer.ChunkServer)
def __init__(self, transientdb, persistentdb): self.service=None self.tdb=transientdb self.pdb=persistentdb socket=gevent.socket.socket() guid=Guid.generate() self.stub=rpc.RpcStub(guid, socket, ChunkServer.ChunkServer)
def create_tuplespace(self, req, msgid, message, data):
# return a new tuplespace id
ts = "S%s" % (guid.generate(), )
local_ts.newTupleSpace(ts)
local_ts.addReference(ts, utils.getProcessIdFromThreadId(str(data[0])))
req.send(msgid, ("RESULT_STRING", ts))
def process(self, file):
"""
The overridden process function, moves the file and renames it.
Assumes the file is rewound
"""
if not file.has_key('fname'):
return file
log.debug('Moving %s', file['fname'])
to = os.path.join(self.to, guid.generate())
if not os.path.isabs(file['fname']):
frm = os.path.join(self.frm, file['fname'])
else:
frm = file['fname']
if not os.path.exists(frm):
log.info("Given filename does not exist, bailing")
file['msg'] = "File does not exist"
file['na'] = na.FAILURE
self.cleanup(file)
return False
move(frm, to)
file["fname"] = to
return file
def saveFile(self, strokes, labels, originalFile, outFile):
''' Save the labels of the stroke objects and the stroke objects themselves
in an XML format that can be visualized by the labeler.
Need to input the original file from which the strokes were read
so that we can retrieve a lot of data that we don't store here'''
sketch = xml.dom.minidom.parse(originalFile)
# copy most of the data, including all points, substrokes, strokes
# then just add the shapes onto the end
impl = xml.dom.minidom.getDOMImplementation()
newdoc = impl.createDocument(sketch.namespaceURI, "sketch",
sketch.doctype)
top_element = newdoc.documentElement
# Add the attibutes from the sketch document
for attrib in sketch.documentElement.attributes.keys():
top_element.setAttribute(
attrib, sketch.documentElement.getAttribute(attrib))
# Now add all the children from sketch as long as they are points, strokes
# or substrokes
sketchElem = sketch.getElementsByTagName("sketch")[0]
for child in sketchElem.childNodes:
if child.nodeType == xml.dom.Node.ELEMENT_NODE:
if child.tagName == "point":
top_element.appendChild(child)
elif child.tagName == "shape":
if child.getAttribute("type") == "substroke" or \
child.getAttribute("type") == "stroke":
top_element.appendChild(child)
# Finally, add the new elements for the labels
for i in range(len(strokes)):
# make a new element
newElem = newdoc.createElement("shape")
# Required attributes are type, name, id and time
newElem.setAttribute("type", labels[i])
newElem.setAttribute("name", "shape")
newElem.setAttribute("id", guid.generate())
newElem.setAttribute("time", str(
strokes[i].points[-1][2])) # time is finish time
# Now add the children
for ss in strokes[i].substrokeIds:
ssElem = newdoc.createElement("arg")
ssElem.setAttribute("type", "substroke")
ssElem.appendChild(newdoc.createTextNode(ss))
newElem.appendChild(ssElem)
top_element.appendChild(newElem)
# Write to the file
filehandle = open(outFile, "w")
newdoc.writexml(filehandle)
filehandle.close()
# unlink the docs
newdoc.unlink()
sketch.unlink()
def saveFile( self, strokes, labels, originalFile, outFile ):
''' Save the labels of the stroke objects and the stroke objects themselves
in an XML format that can be visualized by the labeler.
Need to input the original file from which the strokes were read
so that we can retrieve a lot of data that we don't store here'''
#print labels, strokes
sketch = xml.dom.minidom.parse(originalFile)
# copy most of the data, including all points, substrokes, strokes
# then just add the shapes onto the end
impl = xml.dom.minidom.getDOMImplementation()
newdoc = impl.createDocument(sketch.namespaceURI, "sketch", sketch.doctype)
top_element = newdoc.documentElement
# Add the attibutes from the sketch document
for attrib in sketch.documentElement.attributes.keys():
top_element.setAttribute(attrib, sketch.documentElement.getAttribute(attrib))
# Now add all the children from sketch as long as they are points, strokes
# or substrokes
sketchElem = sketch.getElementsByTagName("sketch")[0]
for child in sketchElem.childNodes:
if child.nodeType == xml.dom.Node.ELEMENT_NODE:
if child.tagName == "point":
top_element.appendChild(child)
elif child.tagName == "shape":
if child.getAttribute("type") == "substroke" or \
child.getAttribute("type") == "stroke":
top_element.appendChild(child)
# Finally, add the new elements for the labels
for i in range(len(strokes)):
# make a new element
newElem = newdoc.createElement("shape")
# Required attributes are type, name, id and time
newElem.setAttribute("type", labels[i])
newElem.setAttribute("name", "shape")
newElem.setAttribute("id", guid.generate() )
newElem.setAttribute("time", str(strokes[i].points[-1][2])) # time is finish time
# Now add the children
for ss in strokes[i].substrokeIds:
ssElem = newdoc.createElement("arg")
ssElem.setAttribute("type", "substroke")
ssElem.appendChild(newdoc.createTextNode(ss))
newElem.appendChild(ssElem)
top_element.appendChild(newElem)
# Write to the file
filehandle = open(outFile, "w")
newdoc.writexml(filehandle)
filehandle.close()
# unlink the docs
newdoc.unlink()
sketch.unlink()
def register_process(self, req, msgid, message, data):
# When a new process connects they need to acquire new process id
p_id = "P%s" % (guid.generate(), )
processes[p_id] = req
req.type = "CLIENT"
req.name = p_id
pthreads[p_id] = []
pthread_count[p_id] = utils.Counter()
req.send(msgid, ("RESULT_STRING", p_id))
def register_process(self, req, msgid, message, data):
# When a new process connects they need to acquire new process id
p_id = "P%s" % (guid.generate(), )
processes[p_id] = req
req.type = "CLIENT"
req.name = p_id
pthreads[p_id] = []
pthread_count[p_id] = utils.Counter()
req.send(msgid, ("RESULT_STRING", p_id))
def process(self, file):
assert file and len(file)>0
if not file.has_key('filetype'):
return file
if file['filetype'] == 'mp3':
return file
elif file['filetype'] == 'mp4':
#This will not work on windows
if not self.enabled:
log.info('mp4 file uploaded, but no transcoders available')
file['msg'] = 'No transcoder available'
file['na'] = na.FAILURE
self.cleanup(file)
return False
new_fname = \
path.join(config['upload_dir'], file['fbid'], guid.generate())
res = os.system(
'nice -n +17 faad -q -o - %s 2>/dev/null | nice -n +17 lame --quiet -b 128 - %s' %
(file['fname'], new_fname))
if res != 0:
log.info('Error transcoding mp4 file')
file['msg'] = 'Transcoding error'
file['na'] = na.FAILURE
if os.path.exists(new_fname):
os.remove(new_fname)
self.cleanup(file)
return False
os.remove(file['fname'])
file['fname'] = new_fname
new_sha = sha1(open(new_fname, 'rb').read()).hexdigest()
new_fname = path.join(config['upload_dir'], file['fbid'], new_sha)
try:
os.rename(file['fname'], new_fname)
except OSError:
self.cleanup(file)
return False
file['fname'] = new_fname
file['usersha'] = new_sha
return file
else:
log.info('Unknown filetype encountered')
file['msg'] = 'Unknown file encoding'
file['na'] = na.FAILURE
self.cleanup(file)
return False
def registerType(type, pid):
assert type.isType()
cache_lock.acquire()
try:
id = "T"+generate()
if isNodeId(pid):
__cache[id] = [type, None, __getTypeReferences(type)]
else:
__cache[id] = [type, pid, __getTypeReferences(type)]
return id
finally:
cache_lock.release()
def registerType(type, pid):
assert type.isType()
cache_lock.acquire()
try:
id = "T" + generate()
if isNodeId(pid):
__cache[id] = [type, None, __getTypeReferences(type)]
else:
__cache[id] = [type, pid, __getTypeReferences(type)]
return id
finally:
cache_lock.release()
def NewChunk(self, req):
self.lvm.reload()
ret = msg.NewChunk_Response()
size = str(req.size) + "M"
for i in range(req.count):
a_guid = Guid.generate()
lv_name = self.prefix_vol + Guid.toStr(a_guid)
lv_size = size
output = self.lvm.lv_create(self.vgname, lv_name, lv_size)
if output != None:
ret.error = str(i) + ":" + output + " "
break
t = ret.guids.add()
Guid.assign(t, a_guid)
return ret
def NewChunk(self, req): self.lvm.reload() ret = msg.NewChunk_Response() size = str(req.size)+'M' for i in range(req.count): a_guid = Guid.generate() lv_name = self.prefix_vol+Guid.toStr(a_guid) lv_size = size output = self.lvm.lv_create(self.vgname, lv_name, lv_size) if output!=None: ret.error = str(i) + ':' + output + ' ' break t=ret.guids.add() Guid.assign(t, a_guid) return ret
def heartBeat(self, confobj):
guid=Guid.generate()
stub=rpc.RpcStub(guid, None, mds.MDS)
while True:
try:
socket=gevent.socket.socket()
socket.connect((confobj.mdsaddress, int(confobj.mdsport)))
mdsEndpoint=(confobj.mdsaddress, int(confobj.mdsport))
socket.connect(mdsEndpoint)
stub.socket=socket
self.doHeartBeat(int(confobj.port), stub)
except KeyboardInterrupt:
raise
except:
logging.debug('An error occured during heart beat, preparing to retry', exc_info=1)
gevent.sleep(2)
def heartBeat(self, confobj):
guid = Guid.generate()
stub = rpc.RpcStub(guid, None, mds.MDS)
while True:
try:
socket = gevent.socket.socket()
socket.connect((confobj.mdsaddress, int(confobj.mdsport)))
mdsEndpoint = (confobj.mdsaddress, int(confobj.mdsport))
socket.connect(mdsEndpoint)
stub.socket = socket
self.doHeartBeat(int(confobj.port), stub)
except KeyboardInterrupt:
raise
except:
logging.debug("An error occured during heart beat, preparing to retry", exc_info=1)
gevent.sleep(2)
def __init__(self):
self.registry = {}
self.service = None
self.guid = Guid.generate()
self.stub = rpc.RpcStub(self.guid, None, ChkSvr_mock.ChunkServer)
def main():
"""\internal
\brief Parse command line options and start the server.
"""
global server, domain_server, node_id, neighbours, local_ts, options
options = getOptions()
if not options.daemon:
import subprocess
subprocess.Popen("linda_monitor", shell=True)
#pid = os.fork()
#if pid != 0:
# from monitor import monitor
# return monitor.Thread(options.port).run()
_linda_server.init()
if options.peer:
options.peer.append("127.0.0.1") # always allow local connections.
def lookupname(addr):
try:
addr,r = addr.split("/")
except ValueError:
r = "32"
addr = socket.gethostbyname(addr)
if addr.count(".") != 3:
print "%s is not in n.n.n.n[/r] format" % (addr+"/"+r)
sys.exit(0)
return addr+"/"+r
node_id = "N"+guid.generate()
_linda_server.setnodeid(node_id)
if options.mdns and not options.disable_mdns:
import mdns
s = mdns.connect()
if s:
options.connect = s[0]
options.connectport = s[1]
if options.connect is not None and len(options.connect) > 0:
if options.connectport:
con = zip(options.connect, options.connectport) + [(x, 2102) for x in options.connect[len(options.connectport):]]
else:
con = zip(options.connect, [2102 for _ in options.connect])
assert len(con) == len(options.connect)
i = 0
while True:
svr, port = con[i]
s = _linda_server.connect("%s:%s" % (svr, port));
if s is None:
if i < len(con)-1:
i += 1
continue
else:
print "Unable to connect to server %s:%i." % (svr, port)
sys.exit(-1)
s = Connection(s)
s.type = None
s.send(None, (get_node_id, ))
node = s.recv()[2]
assert utils.isNodeId(node), repr(node)
s.send(None, (my_name_is, node_id))
s.recv()
s.type = "SERVER"
s.name = node
neighbours[node] = s
connections.sockets.append(s)
break
local_ts.newTupleSpace("UTS")
while True:
try:
socket_watcher()
except KeyboardInterrupt:
cleanShutdown()
raise
break
def __init__(self, mdsip, mdsport): self.guid = Guid.generate() self.chkpool = Pool(ChunkServerClient, ChunkServerClient.__del__) self.mds = MDSClient(self.guid, mdsip, mdsport) self.dmclient = DMClient() self.CheckVolume()
def handler(req, args):
if args.has_key("error_msg"):
error_msg = args["error_msg"]
else:
error_msg = ""
if not preprocess.input_matches(req, args, expected_args):
return postprocess.fill_page(template_path, "", error_msg, utils.build_dict(expected_args, []))
elif args.has_key("error_msg"):
return postprocess.fill_page(template_path, "", error_msg, args)
login = args["login"].strip()
passwd = args["passwd"]
if login == "" or passwd == "":
return postprocess.fill_page(template_path, "", "Username or password not specified", args)
con = MySQLdb.connect( host = settings.database_settings["host"],
user = settings.database_settings["login"],
passwd = settings.database_settings["password"],
db = settings.database_settings["database"])
cur = con.cursor()
try:
cur.execute(""" SELECT logins.id, login, passwd, name, surname
FROM logins JOIN emails
ON logins.emails_ref = emails.id
WHERE login=%s and passwd=%s""", (login, md5.new(passwd).digest()) )
result = cur.fetchone()
if result is None:
return postprocess.fill_page(template_path, "", "Bad username or password", args)
name = result[3]
surname = result[4]
#generate session id
session_id = guid.generate("") # ANALYSIS FIX
expire_time = utils.get_session_expire_time()
cur.execute("""DELETE FROM sessions WHERE expire_time < now()""")
cur.execute("""INSERT INTO sessions (session_id, expire_time, user_id) VALUES (%s, %s, %s) """, (session_id, expire_time, result[0]))
#set cookie
req.headers_out.add( "Set-Cookie", utils.forge_cookie("session", session_id, "/"))
#process statistics
UserAgent = ""
if req.headers_in.has_key("User-Agent"):
UserAgent = req.headers_in["User-Agent"]
if UserAgent.find("/") > 0:
UserAgent = UserAgent[0:UserAgent.find("/")]
if len(UserAgent) > 0:
cur.execute("SELECT id FROM stat_browser WHERE browser = '%s'" % (UserAgent,))
result = cur.fetchone()
if result is None:
cur.execute("INSERT INTO stat_browser (browser, counter) VALUES (%s, %s)", (UserAgent, 1))
else:
cur.execute("UPDATE stat_browser SET counter = counter + 1 WHERE id = %s", (result[0], ))
finally:
con.commit()
cur.close()
con.close()
logging.info('Login of %s from %s' % (login, req.connection.remote_ip))
return {"Location":"news.html", "notice_msg":"Hello, %s %s!" % (name, surname)}
def main():
"""\internal
\brief Parse command line options and start the server.
"""
global server, domain_server, node_id, neighbours, local_ts, options
options = getOptions()
if not options.daemon:
import subprocess
subprocess.Popen("linda_monitor", shell=True)
#pid = os.fork()
#if pid != 0:
# from monitor import monitor
# return monitor.Thread(options.port).run()
_linda_server.init()
if options.peer:
options.peer.append("127.0.0.1") # always allow local connections.
def lookupname(addr):
try:
addr, r = addr.split("/")
except ValueError:
r = "32"
addr = socket.gethostbyname(addr)
if addr.count(".") != 3:
print "%s is not in n.n.n.n[/r] format" % (addr + "/" + r)
sys.exit(0)
return addr + "/" + r
node_id = "N" + guid.generate()
_linda_server.setnodeid(node_id)
if options.mdns and not options.disable_mdns:
import mdns
s = mdns.connect()
if s:
options.connect = s[0]
options.connectport = s[1]
if options.connect is not None and len(options.connect) > 0:
if options.connectport:
con = zip(options.connect, options.connectport) + [
(x, 2102) for x in options.connect[len(options.connectport):]
]
else:
con = zip(options.connect, [2102 for _ in options.connect])
assert len(con) == len(options.connect)
i = 0
while True:
svr, port = con[i]
s = _linda_server.connect("%s:%s" % (svr, port))
if s is None:
if i < len(con) - 1:
i += 1
continue
else:
print "Unable to connect to server %s:%i." % (svr, port)
sys.exit(-1)
s = Connection(s)
s.type = None
s.send(None, (get_node_id, ))
node = s.recv()[2]
assert utils.isNodeId(node), repr(node)
s.send(None, (my_name_is, node_id))
s.recv()
s.type = "SERVER"
s.name = node
neighbours[node] = s
connections.sockets.append(s)
break
local_ts.newTupleSpace("UTS")
while True:
try:
socket_watcher()
except KeyboardInterrupt:
cleanShutdown()
raise
break
def __init__(self):
self.registry = {}
self.service = None
self.guid = Guid.generate()
self.stub = rpc.RpcStub(self.guid, None, ChkSvr_mock.ChunkServer)
def CreateVolume(self, volname, volsize, voltype, chksizes, volnames, params):
vollist = []
if len(volnames) > 0:
volsize = 0
for name in volnames:
try:
vol = self.mds.ReadVolumeInfo(name)
except IOError as ex:
if str(ex).find('No such file or directory') > -1:
logging.error('Volume '+name+' does not exist')
return False
else:
raise ex
if vol.parameters[2] == 'subvolume':
logging.error('volume '+name+' is a subvolume of another volume')
return False
if vol.parameters[3] == 'inactive':
logging.warn('Volume '+name+' is inactive')
logging.warn('Mounting volume '+name)
if not self.MountVolume(name):
logging.error('Mount volume '+name+' failed')
return False
volsize += vol.size
vollist.append(vol)
if len(chksizes) == 0 and len(volnames) == 0:
totsize = volsize
while totsize > CHUNKSIZE:
chksizes.append(CHUNKSIZE)
totsize -= CHUNKSIZE
chksizes.append(totsize)
if voltype == 'gfs':
temlist = []
for chksize in chksizes:
temlist.append(chksize)
temlist.append(chksize)
temlist.append(chksize)
chksizes = temlist
if len(chksizes) > 0:
vollist = self.NewChunkList(chksizes)
if vollist == None:
logging.error('New Chunk(s) failed')
return False
volume = Object()
volume.size = volsize
volume.assembler = voltype
volume.subvolumes = vollist
volume.guid = Guid.toStr(Guid.generate())
volume.parameters = [volname, '/dev/mapper/'+volname, 'volume', 'active']
volume.parameters.extend(params)
if self.dmclient.MapVolume(volume) == False:
logging.error('device mapper: create volume '+volname+' failed')
if volume.subvolumes[0].assembler == 'chunk':
for subvol in volume.subvolumes:
self.DeleteVolumeTree(subvol)
return False
self.mds.WriteVolumeInfo(volume)
for vol in vollist:
if vol.assembler != 'chunk':
vol.parameters[2] = 'subvolume'
vol.parameters[3] = 'active'
self.mds.WriteVolumeInfo(vol)
return True
