def create_tuplespace(self, req, msgid, message, data): # return a new tuplespace id ts = "S%s" % (guid.generate(), ) local_ts.newTupleSpace(ts) local_ts.addReference(ts, utils.getProcessIdFromThreadId(str(data[0]))) req.send(msgid, ("RESULT_STRING", ts))
def __init__(self, transientdb, persistentdb): self.service = None self.tdb = transientdb self.pdb = persistentdb socket = gevent.socket.socket() guid = Guid.generate() self.stub = rpc.RpcStub(guid, socket, ChunkServer.ChunkServer)
def __init__(self, transientdb, persistentdb): self.service=None self.tdb=transientdb self.pdb=persistentdb socket=gevent.socket.socket() guid=Guid.generate() self.stub=rpc.RpcStub(guid, socket, ChunkServer.ChunkServer)
def process(self, file): """ The overridden process function, moves the file and renames it. Assumes the file is rewound """ if not file.has_key('fname'): return file log.debug('Moving %s', file['fname']) to = os.path.join(self.to, guid.generate()) if not os.path.isabs(file['fname']): frm = os.path.join(self.frm, file['fname']) else: frm = file['fname'] if not os.path.exists(frm): log.info("Given filename does not exist, bailing") file['msg'] = "File does not exist" file['na'] = na.FAILURE self.cleanup(file) return False move(frm, to) file["fname"] = to return file
def saveFile(self, strokes, labels, originalFile, outFile): ''' Save the labels of the stroke objects and the stroke objects themselves in an XML format that can be visualized by the labeler. Need to input the original file from which the strokes were read so that we can retrieve a lot of data that we don't store here''' sketch = xml.dom.minidom.parse(originalFile) # copy most of the data, including all points, substrokes, strokes # then just add the shapes onto the end impl = xml.dom.minidom.getDOMImplementation() newdoc = impl.createDocument(sketch.namespaceURI, "sketch", sketch.doctype) top_element = newdoc.documentElement # Add the attibutes from the sketch document for attrib in sketch.documentElement.attributes.keys(): top_element.setAttribute( attrib, sketch.documentElement.getAttribute(attrib)) # Now add all the children from sketch as long as they are points, strokes # or substrokes sketchElem = sketch.getElementsByTagName("sketch")[0] for child in sketchElem.childNodes: if child.nodeType == xml.dom.Node.ELEMENT_NODE: if child.tagName == "point": top_element.appendChild(child) elif child.tagName == "shape": if child.getAttribute("type") == "substroke" or \ child.getAttribute("type") == "stroke": top_element.appendChild(child) # Finally, add the new elements for the labels for i in range(len(strokes)): # make a new element newElem = newdoc.createElement("shape") # Required attributes are type, name, id and time newElem.setAttribute("type", labels[i]) newElem.setAttribute("name", "shape") newElem.setAttribute("id", guid.generate()) newElem.setAttribute("time", str( strokes[i].points[-1][2])) # time is finish time # Now add the children for ss in strokes[i].substrokeIds: ssElem = newdoc.createElement("arg") ssElem.setAttribute("type", "substroke") ssElem.appendChild(newdoc.createTextNode(ss)) newElem.appendChild(ssElem) top_element.appendChild(newElem) # Write to the file filehandle = open(outFile, "w") newdoc.writexml(filehandle) filehandle.close() # unlink the docs newdoc.unlink() sketch.unlink()
def saveFile( self, strokes, labels, originalFile, outFile ): ''' Save the labels of the stroke objects and the stroke objects themselves in an XML format that can be visualized by the labeler. Need to input the original file from which the strokes were read so that we can retrieve a lot of data that we don't store here''' #print labels, strokes sketch = xml.dom.minidom.parse(originalFile) # copy most of the data, including all points, substrokes, strokes # then just add the shapes onto the end impl = xml.dom.minidom.getDOMImplementation() newdoc = impl.createDocument(sketch.namespaceURI, "sketch", sketch.doctype) top_element = newdoc.documentElement # Add the attibutes from the sketch document for attrib in sketch.documentElement.attributes.keys(): top_element.setAttribute(attrib, sketch.documentElement.getAttribute(attrib)) # Now add all the children from sketch as long as they are points, strokes # or substrokes sketchElem = sketch.getElementsByTagName("sketch")[0] for child in sketchElem.childNodes: if child.nodeType == xml.dom.Node.ELEMENT_NODE: if child.tagName == "point": top_element.appendChild(child) elif child.tagName == "shape": if child.getAttribute("type") == "substroke" or \ child.getAttribute("type") == "stroke": top_element.appendChild(child) # Finally, add the new elements for the labels for i in range(len(strokes)): # make a new element newElem = newdoc.createElement("shape") # Required attributes are type, name, id and time newElem.setAttribute("type", labels[i]) newElem.setAttribute("name", "shape") newElem.setAttribute("id", guid.generate() ) newElem.setAttribute("time", str(strokes[i].points[-1][2])) # time is finish time # Now add the children for ss in strokes[i].substrokeIds: ssElem = newdoc.createElement("arg") ssElem.setAttribute("type", "substroke") ssElem.appendChild(newdoc.createTextNode(ss)) newElem.appendChild(ssElem) top_element.appendChild(newElem) # Write to the file filehandle = open(outFile, "w") newdoc.writexml(filehandle) filehandle.close() # unlink the docs newdoc.unlink() sketch.unlink()
def register_process(self, req, msgid, message, data): # When a new process connects they need to acquire new process id p_id = "P%s" % (guid.generate(), ) processes[p_id] = req req.type = "CLIENT" req.name = p_id pthreads[p_id] = [] pthread_count[p_id] = utils.Counter() req.send(msgid, ("RESULT_STRING", p_id))
def process(self, file): assert file and len(file)>0 if not file.has_key('filetype'): return file if file['filetype'] == 'mp3': return file elif file['filetype'] == 'mp4': #This will not work on windows if not self.enabled: log.info('mp4 file uploaded, but no transcoders available') file['msg'] = 'No transcoder available' file['na'] = na.FAILURE self.cleanup(file) return False new_fname = \ path.join(config['upload_dir'], file['fbid'], guid.generate()) res = os.system( 'nice -n +17 faad -q -o - %s 2>/dev/null | nice -n +17 lame --quiet -b 128 - %s' % (file['fname'], new_fname)) if res != 0: log.info('Error transcoding mp4 file') file['msg'] = 'Transcoding error' file['na'] = na.FAILURE if os.path.exists(new_fname): os.remove(new_fname) self.cleanup(file) return False os.remove(file['fname']) file['fname'] = new_fname new_sha = sha1(open(new_fname, 'rb').read()).hexdigest() new_fname = path.join(config['upload_dir'], file['fbid'], new_sha) try: os.rename(file['fname'], new_fname) except OSError: self.cleanup(file) return False file['fname'] = new_fname file['usersha'] = new_sha return file else: log.info('Unknown filetype encountered') file['msg'] = 'Unknown file encoding' file['na'] = na.FAILURE self.cleanup(file) return False
def registerType(type, pid): assert type.isType() cache_lock.acquire() try: id = "T"+generate() if isNodeId(pid): __cache[id] = [type, None, __getTypeReferences(type)] else: __cache[id] = [type, pid, __getTypeReferences(type)] return id finally: cache_lock.release()
def registerType(type, pid): assert type.isType() cache_lock.acquire() try: id = "T" + generate() if isNodeId(pid): __cache[id] = [type, None, __getTypeReferences(type)] else: __cache[id] = [type, pid, __getTypeReferences(type)] return id finally: cache_lock.release()
def NewChunk(self, req): self.lvm.reload() ret = msg.NewChunk_Response() size = str(req.size) + "M" for i in range(req.count): a_guid = Guid.generate() lv_name = self.prefix_vol + Guid.toStr(a_guid) lv_size = size output = self.lvm.lv_create(self.vgname, lv_name, lv_size) if output != None: ret.error = str(i) + ":" + output + " " break t = ret.guids.add() Guid.assign(t, a_guid) return ret
def NewChunk(self, req): self.lvm.reload() ret = msg.NewChunk_Response() size = str(req.size)+'M' for i in range(req.count): a_guid = Guid.generate() lv_name = self.prefix_vol+Guid.toStr(a_guid) lv_size = size output = self.lvm.lv_create(self.vgname, lv_name, lv_size) if output!=None: ret.error = str(i) + ':' + output + ' ' break t=ret.guids.add() Guid.assign(t, a_guid) return ret
def heartBeat(self, confobj): guid=Guid.generate() stub=rpc.RpcStub(guid, None, mds.MDS) while True: try: socket=gevent.socket.socket() socket.connect((confobj.mdsaddress, int(confobj.mdsport))) mdsEndpoint=(confobj.mdsaddress, int(confobj.mdsport)) socket.connect(mdsEndpoint) stub.socket=socket self.doHeartBeat(int(confobj.port), stub) except KeyboardInterrupt: raise except: logging.debug('An error occured during heart beat, preparing to retry', exc_info=1) gevent.sleep(2)
def heartBeat(self, confobj): guid = Guid.generate() stub = rpc.RpcStub(guid, None, mds.MDS) while True: try: socket = gevent.socket.socket() socket.connect((confobj.mdsaddress, int(confobj.mdsport))) mdsEndpoint = (confobj.mdsaddress, int(confobj.mdsport)) socket.connect(mdsEndpoint) stub.socket = socket self.doHeartBeat(int(confobj.port), stub) except KeyboardInterrupt: raise except: logging.debug("An error occured during heart beat, preparing to retry", exc_info=1) gevent.sleep(2)
def __init__(self): self.registry = {} self.service = None self.guid = Guid.generate() self.stub = rpc.RpcStub(self.guid, None, ChkSvr_mock.ChunkServer)
def main(): """\internal \brief Parse command line options and start the server. """ global server, domain_server, node_id, neighbours, local_ts, options options = getOptions() if not options.daemon: import subprocess subprocess.Popen("linda_monitor", shell=True) #pid = os.fork() #if pid != 0: # from monitor import monitor # return monitor.Thread(options.port).run() _linda_server.init() if options.peer: options.peer.append("127.0.0.1") # always allow local connections. def lookupname(addr): try: addr,r = addr.split("/") except ValueError: r = "32" addr = socket.gethostbyname(addr) if addr.count(".") != 3: print "%s is not in n.n.n.n[/r] format" % (addr+"/"+r) sys.exit(0) return addr+"/"+r node_id = "N"+guid.generate() _linda_server.setnodeid(node_id) if options.mdns and not options.disable_mdns: import mdns s = mdns.connect() if s: options.connect = s[0] options.connectport = s[1] if options.connect is not None and len(options.connect) > 0: if options.connectport: con = zip(options.connect, options.connectport) + [(x, 2102) for x in options.connect[len(options.connectport):]] else: con = zip(options.connect, [2102 for _ in options.connect]) assert len(con) == len(options.connect) i = 0 while True: svr, port = con[i] s = _linda_server.connect("%s:%s" % (svr, port)); if s is None: if i < len(con)-1: i += 1 continue else: print "Unable to connect to server %s:%i." % (svr, port) sys.exit(-1) s = Connection(s) s.type = None s.send(None, (get_node_id, )) node = s.recv()[2] assert utils.isNodeId(node), repr(node) s.send(None, (my_name_is, node_id)) s.recv() s.type = "SERVER" s.name = node neighbours[node] = s connections.sockets.append(s) break local_ts.newTupleSpace("UTS") while True: try: socket_watcher() except KeyboardInterrupt: cleanShutdown() raise break
def __init__(self, mdsip, mdsport): self.guid = Guid.generate() self.chkpool = Pool(ChunkServerClient, ChunkServerClient.__del__) self.mds = MDSClient(self.guid, mdsip, mdsport) self.dmclient = DMClient() self.CheckVolume()
def handler(req, args): if args.has_key("error_msg"): error_msg = args["error_msg"] else: error_msg = "" if not preprocess.input_matches(req, args, expected_args): return postprocess.fill_page(template_path, "", error_msg, utils.build_dict(expected_args, [])) elif args.has_key("error_msg"): return postprocess.fill_page(template_path, "", error_msg, args) login = args["login"].strip() passwd = args["passwd"] if login == "" or passwd == "": return postprocess.fill_page(template_path, "", "Username or password not specified", args) con = MySQLdb.connect( host = settings.database_settings["host"], user = settings.database_settings["login"], passwd = settings.database_settings["password"], db = settings.database_settings["database"]) cur = con.cursor() try: cur.execute(""" SELECT logins.id, login, passwd, name, surname FROM logins JOIN emails ON logins.emails_ref = emails.id WHERE login=%s and passwd=%s""", (login, md5.new(passwd).digest()) ) result = cur.fetchone() if result is None: return postprocess.fill_page(template_path, "", "Bad username or password", args) name = result[3] surname = result[4] #generate session id session_id = guid.generate("") # ANALYSIS FIX expire_time = utils.get_session_expire_time() cur.execute("""DELETE FROM sessions WHERE expire_time < now()""") cur.execute("""INSERT INTO sessions (session_id, expire_time, user_id) VALUES (%s, %s, %s) """, (session_id, expire_time, result[0])) #set cookie req.headers_out.add( "Set-Cookie", utils.forge_cookie("session", session_id, "/")) #process statistics UserAgent = "" if req.headers_in.has_key("User-Agent"): UserAgent = req.headers_in["User-Agent"] if UserAgent.find("/") > 0: UserAgent = UserAgent[0:UserAgent.find("/")] if len(UserAgent) > 0: cur.execute("SELECT id FROM stat_browser WHERE browser = '%s'" % (UserAgent,)) result = cur.fetchone() if result is None: cur.execute("INSERT INTO stat_browser (browser, counter) VALUES (%s, %s)", (UserAgent, 1)) else: cur.execute("UPDATE stat_browser SET counter = counter + 1 WHERE id = %s", (result[0], )) finally: con.commit() cur.close() con.close() logging.info('Login of %s from %s' % (login, req.connection.remote_ip)) return {"Location":"news.html", "notice_msg":"Hello, %s %s!" % (name, surname)}
def main(): """\internal \brief Parse command line options and start the server. """ global server, domain_server, node_id, neighbours, local_ts, options options = getOptions() if not options.daemon: import subprocess subprocess.Popen("linda_monitor", shell=True) #pid = os.fork() #if pid != 0: # from monitor import monitor # return monitor.Thread(options.port).run() _linda_server.init() if options.peer: options.peer.append("127.0.0.1") # always allow local connections. def lookupname(addr): try: addr, r = addr.split("/") except ValueError: r = "32" addr = socket.gethostbyname(addr) if addr.count(".") != 3: print "%s is not in n.n.n.n[/r] format" % (addr + "/" + r) sys.exit(0) return addr + "/" + r node_id = "N" + guid.generate() _linda_server.setnodeid(node_id) if options.mdns and not options.disable_mdns: import mdns s = mdns.connect() if s: options.connect = s[0] options.connectport = s[1] if options.connect is not None and len(options.connect) > 0: if options.connectport: con = zip(options.connect, options.connectport) + [ (x, 2102) for x in options.connect[len(options.connectport):] ] else: con = zip(options.connect, [2102 for _ in options.connect]) assert len(con) == len(options.connect) i = 0 while True: svr, port = con[i] s = _linda_server.connect("%s:%s" % (svr, port)) if s is None: if i < len(con) - 1: i += 1 continue else: print "Unable to connect to server %s:%i." % (svr, port) sys.exit(-1) s = Connection(s) s.type = None s.send(None, (get_node_id, )) node = s.recv()[2] assert utils.isNodeId(node), repr(node) s.send(None, (my_name_is, node_id)) s.recv() s.type = "SERVER" s.name = node neighbours[node] = s connections.sockets.append(s) break local_ts.newTupleSpace("UTS") while True: try: socket_watcher() except KeyboardInterrupt: cleanShutdown() raise break
def CreateVolume(self, volname, volsize, voltype, chksizes, volnames, params): vollist = [] if len(volnames) > 0: volsize = 0 for name in volnames: try: vol = self.mds.ReadVolumeInfo(name) except IOError as ex: if str(ex).find('No such file or directory') > -1: logging.error('Volume '+name+' does not exist') return False else: raise ex if vol.parameters[2] == 'subvolume': logging.error('volume '+name+' is a subvolume of another volume') return False if vol.parameters[3] == 'inactive': logging.warn('Volume '+name+' is inactive') logging.warn('Mounting volume '+name) if not self.MountVolume(name): logging.error('Mount volume '+name+' failed') return False volsize += vol.size vollist.append(vol) if len(chksizes) == 0 and len(volnames) == 0: totsize = volsize while totsize > CHUNKSIZE: chksizes.append(CHUNKSIZE) totsize -= CHUNKSIZE chksizes.append(totsize) if voltype == 'gfs': temlist = [] for chksize in chksizes: temlist.append(chksize) temlist.append(chksize) temlist.append(chksize) chksizes = temlist if len(chksizes) > 0: vollist = self.NewChunkList(chksizes) if vollist == None: logging.error('New Chunk(s) failed') return False volume = Object() volume.size = volsize volume.assembler = voltype volume.subvolumes = vollist volume.guid = Guid.toStr(Guid.generate()) volume.parameters = [volname, '/dev/mapper/'+volname, 'volume', 'active'] volume.parameters.extend(params) if self.dmclient.MapVolume(volume) == False: logging.error('device mapper: create volume '+volname+' failed') if volume.subvolumes[0].assembler == 'chunk': for subvol in volume.subvolumes: self.DeleteVolumeTree(subvol) return False self.mds.WriteVolumeInfo(volume) for vol in vollist: if vol.assembler != 'chunk': vol.parameters[2] = 'subvolume' vol.parameters[3] = 'active' self.mds.WriteVolumeInfo(vol) return True