def decode_from_files(outfile, files, verbose=False):
filefec.decode_from_files(outfile, files, verbose)
#Remove comment to test zfec decoder with testfile
# files=[]
# for x in xrange(0,10):
# if (x != 2 and x != 3):
# files.append(open('test/100KB_0.0'+str(x)+'_10.fec'))
# with open('test/100KB_0_restore','a') as f:
# decode(f, files, False)
def main():
if '-V' in sys.argv or '--version' in sys.argv:
print("zfec library version: ", libversion)
print("zunfec command-line tool version: ", __version__)
return 0
parser = argparse.ArgumentParser(description="Decode data from share files.")
parser.add_argument('-o', '--outputfile', required=True, help='file to write the resulting data to, or "-" for stdout', type=str, metavar='OUTF')
parser.add_argument('sharefiles', nargs='*', help='shares file to read the encoded data from', type=str, metavar='SHAREFILE')
parser.add_argument('-v', '--verbose', help='print out messages about progress', action='store_true')
parser.add_argument('-f', '--force', help='overwrite any file which already in place of the output file', action='store_true')
parser.add_argument('-V', '--version', help='print out version number and exit', action='store_true')
args = parser.parse_args()
if len(args.sharefiles) < 2:
print("At least two sharefiles are required.")
return 1
if args.force:
outf = open(args.outputfile, 'wb')
else:
try:
flags = os.O_WRONLY|os.O_CREAT|os.O_EXCL | (hasattr(os, 'O_BINARY') and os.O_BINARY)
outfd = os.open(args.outputfile, flags)
except OSError:
print("There is already a file named %r -- aborting. Use --force to overwrite." % (args.outputfile,))
return 2
outf = os.fdopen(outfd, "wb")
sharefs = []
# This sort() actually matters for performance (shares with numbers < k
# are much faster to use than the others), as well as being important for
# reproducibility.
args.sharefiles.sort()
for fn in args.sharefiles:
sharefs.append(open(fn, 'rb'))
try:
filefec.decode_from_files(outf, sharefs, args.verbose)
except filefec.InsufficientShareFilesError as e:
print(str(e))
return 3
finally:
outf.close()
for f in sharefs:
f.close()
return 0
def _fec_decode(ns):
logging.debug('UNFEC pass started')
tmpd = tempfile.mkdtemp(dir=os.getcwd())
logging.debug('created tempdir at %s' % tmpd)
# walk first input dir, decode as we go along
for root, dirs, files in os.walk(ns.inputs[0]):
unrooted = os.path.relpath(root, ns.inputs[0])
logging.debug('unrooted path: %s' % unrooted)
for dname in dirs:
osubdir = os.path.join(tmpd, dname)
os.mkdir(osubdir)
logging.debug('created: %s' % osubdir)
for f in files:
# get real name
rname = re.split('\.[0-9]*_[0-9]*\.fec$', f, re.IGNORECASE)[0]
logging.debug('processing chunks for file: %s' % rname)
# get all the file chunks into a list
fecs = []
for indir in ns.inputs:
gpath = common.fec_glob(os.path.join(indir, unrooted, rname))
fecs.extend(glob.glob(gpath))
logging.debug('FEC chunks found for %s: %s' % (rname, fecs))
fec_fds = [open(fec, 'rb') for fec in fecs]
try:
outpath = os.path.join(tmpd, unrooted, rname)
outfd = open(outpath, 'wb')
filefec.decode_from_files(outfd, fec_fds, False)
logging.debug('decoded successfully to %s' % outpath)
except filefec.InsufficientShareFilesError as e:
logging.debug('failed to write %s' % outpath)
sys.stderr.write(repr(e))
common.cleanup(tmpd)
sys.exit(ERR['INSUF_SHARES'])
# all done, rename to output dir
if os.path.exists(ns.output) and ns.force:
shutil.rmtree(ns.output)
logging.debug('removed existing output dir at %s' % ns.output)
shutil.move(tmpd, ns.output)
logging.debug('renamed temp dir %s to output dir %s' % (tmpd, ns.output))
logging.info('UNFEC pass completed')
def main():
if '-V' in sys.argv or '--version' in sys.argv:
print("zfec library version: ", libversion)
print("zunfec command-line tool version: ", __version__)
return 0
parser = argparse.ArgumentParser(
description="Decode data from share files.")
parser.add_argument(
'-o',
'--outputfile',
required=True,
help='file to write the resulting data to, or "-" for stdout',
type=str,
metavar='OUTF')
parser.add_argument('sharefiles',
nargs='*',
help='shares file to read the encoded data from',
type=str,
metavar='SHAREFILE')
parser.add_argument('-v',
'--verbose',
help='print out messages about progress',
action='store_true')
parser.add_argument(
'-f',
'--force',
help='overwrite any file which already in place of the output file',
action='store_true')
parser.add_argument('-V',
'--version',
help='print out version number and exit',
action='store_true')
args = parser.parse_args()
if len(args.sharefiles) < 2:
print("At least two sharefiles are required.")
return 1
if args.force:
outf = open(args.outputfile, 'wb')
else:
try:
flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | (hasattr(
os, 'O_BINARY') and os.O_BINARY)
outfd = os.open(args.outputfile, flags)
except OSError:
print(
"There is already a file named %r -- aborting. Use --force to overwrite."
% (args.outputfile, ))
return 2
outf = os.fdopen(outfd, "wb")
sharefs = []
# This sort() actually matters for performance (shares with numbers < k
# are much faster to use than the others), as well as being important for
# reproducibility.
args.sharefiles.sort()
for fn in args.sharefiles:
sharefs.append(open(fn, 'rb'))
try:
filefec.decode_from_files(outf, sharefs, args.verbose)
except filefec.InsufficientShareFilesError as e:
print(str(e))
return 3
finally:
outf.close()
for f in sharefs:
f.close()
return 0
def download(self, video_name, start_frame):
print '[user.py] P2Puser starts downloading'
connected_caches = []
self.not_connected_caches = not_connected_caches = []
# Connect to the caches
cache_ip_addr = retrieve_caches_address_from_tracker(self.tracker_address, 100, self.user_name)
#cache_ip_addr[0][0] = '[' + cache_ip_addr[0][0] + ']'
self.cache_ip_addr = cache_ip_addr
self.num_of_caches = min(self.num_of_caches, len(cache_ip_addr))
choke_state = 0 # 0 : usual state, 1 : overhead state
choke_ct = 0
for i in range(self.num_of_caches):
each_client = ThreadClient(self, cache_ip_addr[i], self.packet_size, i)
each_client.put_instruction('ID %s' % self.user_name)
self.clients.append(each_client)
connected_caches.append(each_client)
print '[user.py] ', i, 'th connection is CONNECTED : ' , cache_ip_addr[i]
if DEBUG_RYAN:
pdb.set_trace()
for i in range(self.num_of_caches, len(cache_ip_addr)): #Is it not entering this statement here?
if DEBUG_RYAN:
pdb.set_trace()
each_client = ThreadClient(self, cache_ip_addr[i], self.packet_size, i)
each_client.put_instruction('ID %s' % self.user_name)
not_connected_caches.append(each_client)
print '[user.py] ', i, 'th connection is RESERVED: ' , cache_ip_addr[i]
available_chunks = set([])
print '[user.py] putting VLEN', video_name
self.clients[0].put_instruction('VLEN file-%s' % (video_name))
print '[user.py] retrieving VLEN'
vlen_str = self.clients[0].get_response().split('\n')[0]
vlen_items = vlen_str.split('&')
print "VLEN: ", vlen_items
num_frames, code_param_n, code_param_k = int(vlen_items[0]), int(vlen_items[4]), int(vlen_items[5])
base_file_name = video_name + '.mkv'
#turning it into an .mvk also works. Probably should store what kind of file it is server side
#or just make everything .mkv. .MKV is a container file for video, audio, and other stuff.
#Read here for a nice description:
#http://lifehacker.com/5893250/whats-the-difference-between-all-these-video-formats-and-which-one-should-i-use
#base_file_name = video_name + '.mkv'
try:
os.mkdir('video-' + video_name)
except:
pass
# Set internal chunk_size through putting an internal instruction into
# the queue.
base_file = open('video-' + video_name + '/' + base_file_name, 'ab')
base_file_full_path = os.path.abspath('video-' + video_name + '/' + base_file_name)
self.info_thread = infoThread(video_name, code_param_n, code_param_k, self)
self.info_thread.flag = True
self.info_thread.start()
for frame_number in xrange(start_frame, num_frames + 1):
sys.stdout.flush()
effective_rates = [0]*len(self.clients)
assigned_chunks = [0]*len(self.clients)
if frame_number < num_frames: # Usual frames
inst_INTL = 'INTL ' + 'CNKN ' + vlen_items[2] # chunk size of typical frame (not last one)
for client in self.clients:
client.put_instruction(inst_INTL)
self.server_client.put_instruction(inst_INTL)
else: # Last frame
inst_INTL = 'INTL ' + 'CNKN ' + vlen_items[3] # chunk size of last frame
for client in self.clients:
client.put_instruction(inst_INTL)
self.server_client.put_instruction(inst_INTL)
print '[user.py] frame_number : ', frame_number
filename = 'file-' + video_name + '.' + str(frame_number)
# directory for this frame
folder_name = 'video-' + video_name + '/' + video_name + '.' + str(frame_number) + '.dir/'
# get available chunks lists from cache A and B.
inst_CNKS = 'CNKS ' + filename
inst_RETR = 'RETR ' + filename
inst_UPDG = 'UPDG '
inst_NOOP = 'NOOP'
inst_CACHEDATA = 'CACHEDATA '
###### DECIDING WHICH CHUNKS TO DOWNLOAD FROM CACHES: TIME 0 ######
if DEBUG_RYAN:
pdb.set_trace()
available_chunks = [0]*len(self.clients) # available_chunks[i] = cache i's availble chunks
rates = [0]*len(self.clients) # rates[i] = cache i's offered rate
union_chunks = [] # union of all available indices
for i in range(len(self.clients)):
client = self.clients[i]
client.put_instruction(inst_CNKS)
return_str = client.get_response().split('&')
if return_str[0] == '':
available_chunks[i] = []
else:
available_chunks[i] = map(str, return_str[0].split('%'))
for j in range(len(available_chunks[i])):
available_chunks[i][j] = available_chunks[i][j].zfill(2)
rates[i] = int(return_str[1])
union_chunks = list( set(union_chunks) | set(available_chunks[i]) )
## index assignment here
# Assign chunks to cache using cache_chunks_to_request.
print '[user.py] Rates ', rates
print '[user.py] Available chunks', available_chunks
assigned_chunks = cache_chunks_to_request(available_chunks, rates, code_param_n, code_param_k)
effective_rates = [0]*len(rates)
for i in range(len(rates)):
effective_rates[i] = len(assigned_chunks[i])
chosen_chunks = [j for i in assigned_chunks for j in i]
flag_deficit = int(sum(effective_rates) < code_param_k) # True if user needs more rate from caches
list_of_cache_requests = []
# request assigned chunks
for i in range(len(self.clients)):
client = self.clients[i]
client_ip_address = client.address[0] + ':' + str(client.address[1])
print '[user.py] Server_request 2 = "' , assigned_chunks[i] , '"'
client_request_string = '%'.join(assigned_chunks[i]) + '&1'
print "[user.py] [Client " + str(i) + "] flag_deficit: ", flag_deficit, \
", Assigned chunks: ", assigned_chunks[i], \
", Request string: ", client_request_string
if DEBUG_RYAN:
pdb.set_trace()
cachedata_request_string = client_request_string.replace('&1','&' + client_ip_address)
list_of_cache_requests.append(filename+ '.' + cachedata_request_string)
client.put_instruction(inst_UPDG + str(flag_deficit))
client.put_instruction(inst_RETR + '.' + client_request_string)
if False: # WHY IS THIS NOT WORKING?
if not assigned_chunks[i]:
pass
#client.put_instruction(inst_NOOP)
else:
client.put_instruction(inst_RETR + '.' + client_request_string)
###### DECIDING CHUNKS THAT HAVE TO BE DOWNLOADED FROM CACHE: TIME 0 ######
# Before CACHE_DOWNLOAD_DURATION, also start requesting chunks from server.
server_request = []
server_request_2 = []
cdrs = '_' .join(list_of_cache_requests) #cache data request string. Used for parsing inside of server.py's ftp_CACHEDATA
if frame_number < num_frames:
size_of_chunks = vlen_items[2]
else:
size_of_chunks = vlen_items[3]
cdrs = cdrs + '?' + str(size_of_chunks)
chosen_chunks = list(chosen_chunks)
num_chunks_rx_predicted = len(chosen_chunks)
server_request = chunks_to_request(chosen_chunks, range(0, code_param_n), code_param_k - num_chunks_rx_predicted)
num_of_chks_from_server = len(server_request)
if num_of_chks_from_server == 0:
self.server_client.put_instruction(inst_NOOP)
print '[user.py] Caches handling code_param_k chunks, so no request to server. Sending a NOOP'
else:
print '[user.py] Server_request = "' , server_request , '"'
server_request_string = '%'.join(server_request) + '&1'
if DEBUG_RYAN:
pdb.set_trace()
self.server_client.put_instruction(inst_CACHEDATA + cdrs)
self.server_client.put_instruction(inst_RETR + '.' + server_request_string)
if(DEBUGGING_MSG):
print "[user.py] Requesting from server: ", server_request, ", Request string: ", server_request_string
print "[user.py] Sending to server's ftp_CACHEDATA: ", inst_CACHEDATA + cdrs
#update_server_load(tracker_address, video_name, num_of_chks_from_server)
sleep(CACHE_DOWNLOAD_DURATION)
###### STOPPING CACHE DOWNLOADS: TIME 8 (CACHE_DOWNLOAD_DURATION) ######
# immediately stop cache downloads.
for client in self.clients:
try:
client.client.abort()
except:
print "[user.py] Cache connections suddenly aborted. Stopping all download."
return
print "[user.py] Cache connections aborted for frame %d" % (frame_number)
###### REQUEST ADDITIONAL CHUNKS FROM SERVER: TIME 8 (CACHE_DOWNLOAD_DURATION) ######
# Request from server remaining chunks missing
# Look up the download directory and count the downloaded chunks
chunk_nums_rx = chunk_nums_in_frame_dir(folder_name)
if (DEBUGGING_MSG):
print "%d chunks received so far for frame %d: " % (len(chunk_nums_rx), frame_number)
print chunk_nums_rx
# Add the chunks that have already been requested from server
chunk_nums_rx = list (set(chunk_nums_in_frame_dir(folder_name)) | set(server_request))
print "[user.py] chunk_nums_rx", chunk_nums_rx
addtl_server_request = []
num_chunks_rx = len(chunk_nums_rx)
if (num_chunks_rx >= code_param_k):
print "[user.py] No additional chunks to download from the server. Sending a NOOP"
self.server_client.put_instruction(inst_NOOP)
else:
addtl_server_request = chunks_to_request(chunk_nums_rx, range(0, code_param_n), code_param_k - num_chunks_rx)
print "[user.py] addtl_server_requests", addtl_server_request
if addtl_server_request:
addtl_server_request_string = '%'.join(addtl_server_request) + '&1' # The last digit '1' means 'I am user'
# server should always be set with flag_deficit = 0 (has all chunks)
self.server_client.put_instruction(inst_RETR + '.' + addtl_server_request_string)
if(DEBUGGING_MSG):
print "[user.py] Requesting from server: ", addtl_server_request
elif (DEBUGGING_MSG):
print "No unique chunks from server requested."
###### WAIT FOR CHUNKS FROM SERVER TO FINISH DOWNLOADING: TIME 10 ######
sleep(SERVER_DOWNLOAD_DURATION)
if (DEBUGGING_MSG):
print "[user.py] Waiting to receive all elements from server."
if frame_number > start_frame and (server_request or addtl_server_request) and VLC_PLAYER_USE:
# Need to pause it!
self.VLC_pause_video()
if server_request:
resp_RETR = self.server_client.get_response()
parsed_form = parse_chunks(resp_RETR)
fname, framenum, chunks, user_or_cache = parsed_form
print "[user.py] Downloaded chunks from server: ", chunks
if addtl_server_request:
resp_RETR = self.server_client.get_response()
parsed_form = parse_chunks(resp_RETR)
fname, framenum, chunks, user_or_cache = parsed_form
print "[user.py] Downloaded chunks from server: ", chunks
# Now play it
if frame_number > start_frame and (server_request or addtl_server_request) and VLC_PLAYER_USE:
self.VLC_pause_video()
chunk_nums = chunk_nums_in_frame_dir(folder_name)
num_chunks_rx = len(chunk_nums)
if num_chunks_rx >= code_param_k and DEBUGGING_MSG:
print "[user.py] Received", code_param_k, "packets"
else:
print "[user.py] Did not receive", code_param_k, "packets for this frame."
# abort the connection to the server
self.server_client.client.abort()
# put together chunks into single frame; then concatenate onto original file.
print 'about to decode...'
chunksList = chunk_files_in_frame_dir(folder_name)
if frame_number != start_frame:
print 'size of base file:', os.path.getsize('video-' + video_name + '/' + base_file_name)
print 'trying to decode'
filefec.decode_from_files(base_file, chunksList)
print 'decoded. Size of base file =', os.path.getsize('video-' + video_name + '/' + base_file_name)
if frame_number == 1 and VLC_PLAYER_USE:
self.VLC_empty_list()
self.VLC_start_video(base_file_full_path)
if USER_TOPOLOGY_UPDATE:
if choke_state == 0: # Normal state
print '[user.py] Normal state : ', choke_ct
choke_ct += 1
if choke_ct == T_choke:
choke_ct = 0
if len(not_connected_caches) == 0:
pass
else: # Add a new cache temporarily
new_cache_index = random.sample(range(len(not_connected_caches)), 1)
if new_cache_index >= 0:
new_cache = not_connected_caches[new_cache_index[0]]
self.clients.append(new_cache)
connected_caches.append(new_cache)
not_connected_caches.remove(new_cache)
print '[user.py] Topology Update : Temporarily added ', new_cache.address
choke_state = 1 # Now, move to transitional state
choke_ct = 0
print '[user.py] Topology Update : Now the state is changed to overhead staet'
#print '[user.py]', connected_caches, not_connected_caches, self.clients
print '[user.py] conneced caches', self.clients
elif choke_state == 1: # Overhead state
print '[user.py] Overhead state : ', choke_ct
choke_ct += 1
if choke_ct == T_choke2: # Temporary period to spend with temporarily added node
rate_vector = [0] * len(self.clients)
p_vector = [0] * len(self.clients)
for i in range(len(self.clients)):
rate_vector[i] = len(assigned_chunks[i])
p_vector[i] = math.exp( -eps_choke * rate_vector[i])
p_sum = sum(p_vector)
for i in range(len(self.clients)):
p_vector[i] /= p_sum
cdf = [(0,0)] * len(self.clients)
cdf[0] = (0, 0)
for i in range(1, len(self.clients)):
cdf[i] = (i, cdf[i-1][1] + p_vector[i-1])
print '[user.py] cdf :', cdf
client_index = max(i for r in [random.random()] for i,c in cdf if c <= r) # http://stackoverflow.com/questions/4265988/generate-random-numbers-with-a-given-numerical-distribution
removed_cache = self.clients[client_index]
#removed_cache.put_instruction('QUIT')
self.clients.remove(removed_cache)
connected_caches.remove(removed_cache)
not_connected_caches.append(removed_cache)
print '[user.py] Topology Update : ', removed_cache.address, 'is chocked.'
choke_state = 0 # Now, move to normal state
choke_ct = 0
