def main():
global WALLPOST
global USER_POST
global dispatcher
print "START______________________",time.asctime()
# reset dispatcher and wait for reply
print "resetting dispatcher"
dispatcher = Pyro4.core.Proxy("PYRONAME:example.distributed.dispatcher")
async2=Pyro4.async(dispatcher)
async2_result= async2.reset()
print "killing the worker on my machine"
os.system("kill -9 `ps -eaf | grep -i worke[r] | awk '{print $2}'`")
print "start download for self",time.asctime()
u = urllib.urlretrieve('http://enl.usc.edu/~cs694/casebook/config70.txt',"file.txt")
print "downloaded file",time.asctime()
#sending trigger to photo server1
photo_client=Pyro4.Proxy("PYRONAME:photo.async")
photo_async=Pyro4.async(photo_client)
photo_result1=photo_async.process_photo("http://enl.usc.edu/~cs694/casebook/config20.txt")
#sending trigger to photo server2
photo_client2=Pyro4.Proxy("PYRONAME:photo.async2")
photo_async2=Pyro4.async(photo_client2)
photo_result2=photo_async2.process_photo("http://enl.usc.edu/~cs694/casebook/config20.txt")
# trigger the downloaders asynchronously and do not wait for replies
print "sending to downloader1"
download_daemon=Pyro4.Proxy("PYRONAME:download_daemon")
#download_daemon.download('http://enl.usc.edu/~cs694/casebook/config70.txt',"file.txt")
async1=Pyro4.async(download_daemon)
async1_result= async1.download('http://enl.usc.edu/~cs694/casebook/config20.txt')
download_daemon1=Pyro4.Proxy("PYRONAME:download_daemon2")
print "sending to downloader2"
#download_daemon.download('http://enl.usc.edu/~cs694/casebook/config70.txt',"file.txt")
async3=Pyro4.async(download_daemon1)
async3_result= async3.download('http://enl.usc.edu/~cs694/casebook/config20.txt')
# starting worker on my own machine
print"starting worker on my node"
os.system("python worker.py &")
#os.system("python worker.py &")
post(1)
print "calling the aggregate function",time.asctime()
async5=Pyro4.async(dispatcher)
async5_result= async5.aggregate()
photo_merge_done=0
post_index_done=0
while(1):
if photo_merge_done==0:
if photo_result1.ready and photo_result2.ready:
photo_merge_done=1
merged={}
result1 = photo_result1.value
result2 = photo_result2.value
print "photo Results received... Merging them"
A = set(result1.keys())
B = set(result2.keys())
common = A.intersection(B)
onlyA = set(A.difference(common))
onlyB = set(B.difference(common))
for keys in common:
merged[keys] = result1[keys].union(result2[keys])
for keys in onlyA:
merged[keys] = result1[keys]
for keys in onlyB:
merged[keys] = result2[keys]
photo_index = {}
for keys in merged:
photo_index[keys] = {}
photo_index[keys]['photoid']=','.join(merged[keys])
CASS.save_photo(photo_index)
print"inserted photo in cass"
if post_index_done==0:
if async5_result.ready:
post_index_done=1
user_frnd(1)
print "inserting USER_POST in cassandra",time.asctime()
CASS.save_WALL(USER_POST)
print"inserting WALLPOST into cassandra ",time.asctime()
length=len(WALLPOST.keys())
PPP=0
temp={}
for keys in WALLPOST:
PPP=PPP+1
temp[keys]=WALLPOST[keys]
if PPP==500:
CASS.save_post(temp)
temp.clear()
PPP=0
CASS.save_post(temp)
if photo_merge_done==1 and post_index_done==1:
break
#post(1)
print "DONE_____________________",time.asctime()
return
def main():
global WALLPOST
global USER_POST
global dispatcher
print "START______________________",time.asctime()
print "killing the worker on my machine"
os.system("kill -9 `ps -eaf | grep -i worke[r] | awk '{print $2}'`")
print "start download for self",time.asctime()
u = urllib.urlretrieve('http://enl.usc.edu/~cs694/casebook/config.txt',"/home/ubuntu/casebook/LOAD/file.txt")
print "downloaded file",time.asctime()
#Parsing and distributing photo's to processor's
photo_urls = {}
photo_urls = parse()
total_photos = len(photo_urls.keys())
#chunk_size = total_photos/3
#split photo's in to 3 disparate sets.
photo_url_1={}
photo_url_2={}
photo_url_3={}
shuffle=1
for key in photo_urls:
if shuffle == 1:
photo_url_1[key] = []
photo_url_1[key].extend(photo_urls[key])
shuffle=2
continue
if shuffle == 2:
photo_url_2[key] = []
photo_url_2[key].extend(photo_urls[key])
shuffle=3
continue
if shuffle == 3:
photo_url_3[key] = []
photo_url_3[key].extend(photo_urls[key])
shuffle=1
continue
#sending trigger to photo server1
photo_client=Pyro4.Proxy("PYRONAME:photo.async")
photo_async=Pyro4.async(photo_client)
photo_result1=photo_async.process_photo(photo_url_1)
#sending trigger to photo server2
photo_client2=Pyro4.Proxy("PYRONAME:photo.async2")
photo_async2=Pyro4.async(photo_client2)
photo_result2=photo_async2.process_photo(photo_url_2)
#sending trigger to photo server3
photo_client3=Pyro4.Proxy("PYRONAME:photo.async3")
photo_async3=Pyro4.async(photo_client3)
photo_result3=photo_async3.process_photo(photo_url_3)
# trigger the downloaders asynchronously and do not wait for replies
"""
print "sending to downloader1"
download_daemon=Pyro4.Proxy("PYRONAME:download_daemon")
#download_daemon.download('http://enl.usc.edu/~cs694/casebook/config5.txt',"file.txt")
async1=Pyro4.async(download_daemon)
async1_result= async1.download()
"""
print "done all photo work- ",time.asctime()
# reset dispatcher and wait for reply
print "resetting dispatcher"
dispatcher = Pyro4.core.Proxy("PYRONAME:example.distributed.dispatcher")
async2=Pyro4.async(dispatcher)
async2_result= async2.reset()
download_daemon1=Pyro4.Proxy("PYRONAME:download_daemon2")
print "sending to downloader2"
#download_daemon.download('http://enl.usc.edu/~cs694/casebook/config70.txt',"file.txt")
async3=Pyro4.async(download_daemon1)
async3_result= async3.download()
# starting worker on my own machine
print"starting worker on my node"
os.system("python /home/ubuntu/casebook/LOAD/worker.py &")
#os.system("python worker.py &")
#Parsing user post and submiting to dispatcher
post(1)
print "calling the aggregate function",time.asctime()
async5=Pyro4.async(dispatcher)
async5_result= async5.aggregate()
#temp=async5_result.value
#print "-------------------------------------",temp
os.system("rm -rf /home/ubuntu/casebook/media/photos/*")
print "inserting USER_POST and usr_frnd in cassandra",time.asctime()
user_frnd(1)
CASS.save_WALL(USER_POST)
print "inserting DONE in USER_POST and usr_frnd in cassandra",time.asctime()
photo_merge_done=0
post_index_done=0
while(1):
if photo_merge_done==0:
if photo_result1.ready and photo_result2.ready and photo_result3.ready:
photo_merge_done=1
#merged={}
result1 = photo_result1.value
result2 = photo_result2.value
result3 = photo_result3.value
print "photo Results received... Merging them", time.asctime()
m1 = combine(result1, result2)
merged = combine(result3, m1)
photo_index = {}
for keys in merged:
photo_index[keys] = {}
photo_index[keys]['photoid']=','.join(merged[keys])
CASS.save_photo_index(photo_index)
print"inserted photo in cass"
if post_index_done==0:
if async5_result.ready:
post_index_done=1
#user_frnd(1)
#print "inserting USER_POST in cassandra",time.asctime()
#CASS.save_WALL(USER_POST)
"""
print"inserting WALLPOST into cassandra ",time.asctime()
length=len(WALLPOST.keys())
PPP=0
temp={}
for keys in WALLPOST:
PPP=PPP+1
temp[keys]=WALLPOST[keys]
if PPP==5000:
CASS.save_post(temp)
temp.clear()
PPP=0
CASS.save_post(temp)
print "completed WALLPOST in cassandra",time.asctime()
"""
"""
print "start copiing photos", time.asctime()
os.system("scp -i /home/ubuntu/abhi.pem -q -r [email protected]:./distributed/photos /home/ubuntu/casebook/media/")
os.system("scp -i /home/ubuntu/abhi.pem -q -r [email protected]:./distributed/photos /home/ubuntu/casebook/media/")
os.system("scp -i /home/ubuntu/abhi.pem -q -r [email protected]:./distributed/photos /home/ubuntu/casebook/media/")
print "completed copiing photos", time.asctime()
dir_name = "/home/ubuntu/casebook/media/photos/"
photo_data_dict={}
num_inserted = 0
for file_name in os.listdir(dir_name):
dirfile = os.path.join(dir_name, file_name)
f = open(dirfile, 'r')
data = f.read()
num_inserted = num_inserted + 1
photo_data_dict[str(file_name)]={}
photo_data_dict[str(file_name)]['data'] = data
if num_inserted == 10:
CASS.save_photo(photo_data_dict)
photo_data_dict.clear()
num_inserted = 0
CASS.save_photo(photo_data_dict)
print "Inserted photo's in cassandra", time.asctime()
"""
if photo_merge_done==1 and post_index_done==1:
break
#post(1)
print "DONE_____________________",time.asctime()
return
