def stage(): filename = ".\\tests\\shakespeare.txt" key = makeKeyFile(filename,smartChunk) cfs = getCFSsingleton() cfs.writeFile(key) time.sleep(2) # generate the id for the keyfile from the filename hashid =hash_util.hash_str(filename) keyfile_raw = cfs.getChunk(hashid).contents keyfile = KeyFile.parse(keyfile_raw) atoms = [] for key in keyfile.chunklist: atoms.append(Data_Atom(key,key)) return atoms
def map_func(atom): atom = getCFSsingleton().getChunk(atom.hashkeyID) freqs = {} text = atom.contents text = text.split() for word in text: word = word.lower() word = word.strip(" !?.,;:\"\'*[]()/<>-*~%") if word is u"": continue if word in freqs: freqs[word] = freqs[word] + 1 else: freqs[word] = 1 # print freqs atom = Data_Atom(freqs,atom.hashkeyID) return atom
def map_func(atom): # get the atom off the disk #print "LOOK AT ME!!!!!!!!", type(atom.hashkeyID) atom = getCFSsingleton().getChunk(atom.hashkeyID) freqs = {} text = atom.contents #temporary for testing # we'll have to pull the contents of the file for the given key text = text.split() for word in text: word = word.lower() word = word.strip(" !?.,;:\"\'*[]()/<>-*~%") if word is u"": continue if word in freqs: freqs[word] = freqs[word] + 1 else: freqs[word] = 1 # print freqs atom = Data_Atom(freqs,atom.hashkeyID) return atom
def setup_Node(addr="localhost", port=None): # Setup the info for the node node.IPAddr = addr node.ctrlPort = port node.thisNode = node.Node_Info(node.IPAddr, node.ctrlPort) # Setup and attach the network service # Unlike the others, this one is not added to services node.net_server = simple_network.NETWORK_SERVICE("", node.ctrlPort) #node.net_server = dummy_network.start(node.thisNode, node.handle_message) #### setup services here add_service(service.Internal_Service()) add_service(service.ECHO_service()) add_service(Topology_Service.Topology()) add_service(map_reduce.Map_Reduce_Service()) add_service(cfs.getCFSsingleton()) #add_service(httpservice.WEBSERVICE(database)) attach_services()
def console(): cmd = "-" loaded_script = Queue.Queue() try: if loaded_script.empty(): cmd = raw_input() else: cmd = loaded_script.get() loaded_script.task_done() except EOFError: #the user does not have a terminal pass while not ( cmd == "q" or cmd == "Q"): command, args = None, None splitted = cmd.split(' ',1) if len(splitted) >= 1: command = splitted[0] if len(splitted) == 2: args = splitted[1] if command == "test": CFS = cfs.getCFSsingleton() a = cfs.Data_Atom("HELLOO WORLD") CFS.putChunk(a) time.sleep(1) print CFS.getChunk(hash_util.Key(str(a.hashkeyID))) if command.lower() == "help" or command == "?":#USER NEEDS HELP if not args or args == "list": print "Help is availible on the following topics:" for h in help_texts: print "\t"+h print "use: help <topic> \n to get more help" else: if args in help_texts: print help_texts[args] else: print "I have no help on "+args elif command in commands.keys(): mytarget = lambda: commands[command].handle_command(command, args) t = threading.Thread(target=mytarget) t.daemon = True t.start() elif command == "run": file2open = file(args,"r") for l in file2open: loaded_script.put(l) file2open.close() elif command == "stat": input_size = node.todo.qsize(); print "backlog: "+str(input_size) if input_size > 0: print threading.activeCount(), "Active threads. Cheating, spawning new worker." t = threading.Thread(target=node.message_handler_worker) t.setDaemon(True) t.start() elif command == "threads": for t in threading.enumerate(): print t elif command == "num_threads": print threading.activeCount() else: print "successor ", node.successor print "predecessor", node.predecessor try: if loaded_script.empty(): cmd = raw_input() else: cmd = loaded_script.get() loaded_script.task_done() except EOFError: # the user does not have a terminal #print "I do not see a terminal!" time.sleep(1) pass node.net_server.stop() os.exit()