def stage():
filename = ".\\tests\\shakespeare.txt"
key = makeKeyFile(filename,smartChunk)
cfs = getCFSsingleton()
cfs.writeFile(key)
time.sleep(2)
# generate the id for the keyfile from the filename
hashid =hash_util.hash_str(filename)
keyfile_raw = cfs.getChunk(hashid).contents
keyfile = KeyFile.parse(keyfile_raw)
atoms = []
for key in keyfile.chunklist:
atoms.append(Data_Atom(key,key))
return atoms
def map_func(atom):
atom = getCFSsingleton().getChunk(atom.hashkeyID)
freqs = {}
text = atom.contents
text = text.split()
for word in text:
word = word.lower()
word = word.strip(" !?.,;:\"\'*[]()/<>-*~%")
if word is u"":
continue
if word in freqs:
freqs[word] = freqs[word] + 1
else:
freqs[word] = 1
# print freqs
atom = Data_Atom(freqs,atom.hashkeyID)
return atom
def map_func(atom):
# get the atom off the disk
#print "LOOK AT ME!!!!!!!!", type(atom.hashkeyID)
atom = getCFSsingleton().getChunk(atom.hashkeyID)
freqs = {}
text = atom.contents #temporary for testing
# we'll have to pull the contents of the file for the given key
text = text.split()
for word in text:
word = word.lower()
word = word.strip(" !?.,;:\"\'*[]()/<>-*~%")
if word is u"":
continue
if word in freqs:
freqs[word] = freqs[word] + 1
else:
freqs[word] = 1
# print freqs
atom = Data_Atom(freqs,atom.hashkeyID)
return atom
def setup_Node(addr="localhost", port=None):
# Setup the info for the node
node.IPAddr = addr
node.ctrlPort = port
node.thisNode = node.Node_Info(node.IPAddr, node.ctrlPort)
# Setup and attach the network service
# Unlike the others, this one is not added to services
node.net_server = simple_network.NETWORK_SERVICE("", node.ctrlPort)
#node.net_server = dummy_network.start(node.thisNode, node.handle_message)
#### setup services here
add_service(service.Internal_Service())
add_service(service.ECHO_service())
add_service(Topology_Service.Topology())
add_service(map_reduce.Map_Reduce_Service())
add_service(cfs.getCFSsingleton())
#add_service(httpservice.WEBSERVICE(database))
attach_services()
def console():
cmd = "-"
loaded_script = Queue.Queue()
try:
if loaded_script.empty():
cmd = raw_input()
else:
cmd = loaded_script.get()
loaded_script.task_done()
except EOFError: #the user does not have a terminal
pass
while not ( cmd == "q" or cmd == "Q"):
command, args = None, None
splitted = cmd.split(' ',1)
if len(splitted) >= 1:
command = splitted[0]
if len(splitted) == 2:
args = splitted[1]
if command == "test":
CFS = cfs.getCFSsingleton()
a = cfs.Data_Atom("HELLOO WORLD")
CFS.putChunk(a)
time.sleep(1)
print CFS.getChunk(hash_util.Key(str(a.hashkeyID)))
if command.lower() == "help" or command == "?":#USER NEEDS HELP
if not args or args == "list":
print "Help is availible on the following topics:"
for h in help_texts:
print "\t"+h
print "use: help <topic> \n to get more help"
else:
if args in help_texts:
print help_texts[args]
else:
print "I have no help on "+args
elif command in commands.keys():
mytarget = lambda: commands[command].handle_command(command, args)
t = threading.Thread(target=mytarget)
t.daemon = True
t.start()
elif command == "run":
file2open = file(args,"r")
for l in file2open:
loaded_script.put(l)
file2open.close()
elif command == "stat":
input_size = node.todo.qsize();
print "backlog: "+str(input_size)
if input_size > 0:
print threading.activeCount(), "Active threads. Cheating, spawning new worker."
t = threading.Thread(target=node.message_handler_worker)
t.setDaemon(True)
t.start()
elif command == "threads":
for t in threading.enumerate():
print t
elif command == "num_threads":
print threading.activeCount()
else:
print "successor ", node.successor
print "predecessor", node.predecessor
try:
if loaded_script.empty():
cmd = raw_input()
else:
cmd = loaded_script.get()
loaded_script.task_done()
except EOFError: # the user does not have a terminal
#print "I do not see a terminal!"
time.sleep(1)
pass
node.net_server.stop()
os.exit()