def get(key, nodeInfo):
if key == "":
print("Key field empty")
return
else:
keyHash = helper.getHash(key)
node = nodeInfo.findSuccessor(keyHash)
val = helper.getKeyFromNode(node, str(keyHash))
if val == "":
print("Key Not found in node, maybe the node got deleted.")
print("Checking in Successors for Replica")
val = helper.getKeyFromNodeReplica(node, str(keyHash))
if val == "":
print("Key does not exist!")
else:
print("Found ", key, " : ", val)
else:
print("Found ", key, " : ", val)
def updateSuccessorList(self):
if self.successor[1] != self.id:
suc_list = helper.getSuccessorListFromNode(self.successor[0][0],
self.successor[0][1])
if len(suc_list) != R + 1:
return
self.successorList[1] = self.successor
for i in range(2, R + 1):
self.successorList[i][0][0] = suc_list[i - 1][0]
self.successorList[i][0][1] = suc_list[i - 1][1]
self.successorList[i][1] = helper.getHash(suc_list[i - 1][0] +
":" +
str(suc_list[i -
1][1]))
else:
suc_list = self.successorList
self.successorList[1] = self.successor
for i in range(2, R + 1):
self.successorList[i][0][0] = suc_list[i - 1][0][0]
self.successorList[i][0][1] = suc_list[i - 1][0][1]
self.successorList[i][1] = suc_list[i - 1][1]
def join(myNode, ip, port):
'''ip should be socket.gethostname() [in my case oblivious]'''
# if(helper.isNodeAlive(ip,atoi(port.c_str())) == False:
# print("Sorry but no node is active on this ip or port")
# return
myId = myNode.getId()
msg = str(myId) + 'SendSuccessorForThisKeyJoin'
ipAndPort = helper.socket_send_recv(ip, port, msg, "No")
if ipAndPort == "No":
print('Error : ip address not found')
else:
while myId == helper.getHash(ipAndPort):
print("have same HASH", str(myId), " so going to update mine")
myNode.updateSocket()
myId = myNode.getId()
msg = str(myId) + 'SendSuccessorForThisKeyJoin'
ipAndPort = helper.socket_send_recv(ip, port, msg, "No")
address = ipAndPort.split(":")
ip, port, hashVal = address[0], int(
address[1]), helper.getHash(ipAndPort)
myNode.setSuccessor(ip, port, hashVal)
myNode.setSuccessorList(ip, port, hashVal)
myNode.setPredecessor("", -1, -1)
myNode.setFingerTable(ip, port, hashVal)
myNode.setInRing()
helper.getKeysFromSuccessor(myNode, ip, port)
helper.getRepsFromSuccessor(myNode, ip, port)
print("Successfully joined the ring")
# launch threads,one thread will listen to request from other nodes,one will do stabilization
second = threading.Thread(target=listenTo, args=(myNode, ))
second.daemon = True
second.start()
fifth = threading.Thread(target=doStabilize, args=(myNode, ))
fifth.daemon = True
fifth.start()
def callNotify(nodeInfo, ipAndPort):
ipAndPort = ipAndPort[:-len("GetPredecessorNotify")]
#.pop_back()
#ipAndPort.pop_back()
ipAndPortPair = helper.getIpAndPort(ipAndPort)
ip = ipAndPortPair[0]
port = ipAndPortPair[1]
hash_code = helper.getHash(ipAndPort)
node = [[ip, port], hash_code]
nodeInfo.notify(node)
def put(key, value, nodeInfo):
if key == "" or value == "":
print("Key or value field empty")
return
else:
keyHash = helper.getHash(key)
print("Key is ", key, " and hash : ", keyHash)
node = nodeInfo.findSuccessor(keyHash)
helper.sendKeyToNode(node, keyHash, value)
print("key entered successfully")
def findSuccessor(self, nodeId):
this_node = [[self.getIpAddress(), self.getPortNumber()], self.id]
if nodeId > self.id and nodeId <= self.successor[1]:
return self.successor
elif self.id == self.successor[1] or nodeId == self.id:
return this_node
elif self.successor[1] == self.predecessor[1]:
if self.successor[1] >= self.id:
if nodeId > self.successor[1] or nodeId < self.id:
return this_node
else:
if (nodeId > self.id and nodeId > self.successor[1]) or (
nodeId < self.id and nodeId < self.successor[1]):
return self.successor
else:
return this_node
else:
node = self.closestPrecedingNode(nodeId)
if node[1] == self.id:
return self.successor
else:
#/* if this node couldn't find closest preciding node for given node id then now ask it's successor to do so */
if node[1] == -1:
node = self.successor
ip = node[0][0]
port = node[0][1]
msg = str(nodeId) + 'SendSuccessorForThisKey'
ipAndPort = helper.socket_send_recv(ip, port, msg, '')
if len(ipAndPort) <= 0:
node = [['', -1], -1]
return node
# /* set ip,port and hash for this node and return it */
key = ipAndPort
hash_code = helper.getHash(ipAndPort)
ipAndPortPair = helper.getIpAndPort(key)
return [ipAndPortPair, hash_code]
def merge_video(folder='./Videos', overwrite='n', derivatives=False):
directory, subdirectory, files = next(os.walk(folder))
print(directory, subdirectory, files)
# Before checking all files, recursively call on levels below
if len(subdirectory) > 0:
for s in subdirectory:
if s == 'derivatives':
continue
else:
merge_video(f'{directory}/{s}',
overwrite=overwrite,
derivatives=derivatives)
# Read in merge history
mergehistory = []
if os.path.isfile(f'{directory}/.mergehistory'):
with open(f'{directory}/.mergehistory',
'r') as previous_merge_file:
mergehistory = []
for line in previous_merge_file.readlines():
path = line.split(' ')[1].replace('\'',
'').replace('\n', '')
mergehistory.append(path)
# Check all sub-files and merge
manifest = []
for s in subdirectory:
if s == 'derivatives':
continue
_, _, files = next(os.walk(f'{directory}/{s}'))
if len(files) > 0:
for f in files:
if 'cropped' in f or '.mergehistory' in f:
continue
else:
path = f'{directory}/{s}/{f}'
time_now = datetime.now()
folder_end_time = parseTimeFromFolder(path)
if time_now >= folder_end_time:
manifest.append({
'createtime':
os.path.getmtime(path),
'path':
path
})
if len(manifest) > 1:
# handle merge history
if overwrite == 'n' and len(mergehistory) > 0:
paths_to_exclude = set(mergehistory)
manifest = list(
filter(lambda x: x['path'] not in paths_to_exclude,
manifest))
manifest.sort(key=lambda x: x['path'])
manifest.insert(0, {
'createtime': 0.0,
'path': f'{folder}/merged.mp4'
})
else:
manifest.sort(key=lambda x: x['path'])
if len(manifest) > 1:
# create manifest for rendering the video
tmp = getHash()
with open(tmp, 'w') as manifest_file:
print(tmp)
for m in manifest:
print(f'Adding file {m["path"]} to the manifest')
print(f"file \'{m['path']}\'", file=manifest_file)
# record the history of the manifest
merge_file_format = [f"file \'{m}\'" for m in mergehistory]
path_format = [f"file \'{m['path']}\'" for m in manifest]
if overwrite == 'n' and len(merge_file_format) > 0:
new_files = merge_file_format + path_format[1:]
else:
new_files = path_format
with open(f'{folder}/.mergehistory', 'w') as mergehistoryfile:
for n in new_files:
print(n.replace('\n', ''), file=mergehistoryfile)
try:
# Check if the bucket isn't full, if it isn't full we default to overwrite to account for new data
time_now = datetime.now()
folder_end_time = parseTimeFromFolder(folder)
if time_now < folder_end_time:
overwrite = 'y'
# Create merged file
out = getHash()
os.system(
f'ffmpeg -{overwrite} -f concat -safe 0 -i {tmp} -c copy {folder}/{out}.mp4'
)
os.replace(f'{folder}/{out}.mp4', f'{folder}/merged.mp4')
# Create cropped & sped up video
if derivatives:
if os.path.isdir(f'{folder}/derivatives') == False:
os.mkdir(f'{folder}/derivatives')
os.system(
f'ffmpeg -{overwrite} -i {folder}/merged.mp4 -c:v libx264 -b:v 10M -maxrate 10M -bufsize 10M -filter:v "setpts=(1/4)*PTS" {folder}/derivatives/merged-4x.mp4'
)
os.system(
f'ffmpeg -{overwrite} -i {folder}/merged.mp4 -c:v libx264 -b:v 10M -maxrate 10M -bufsize 10M -filter:v "setpts=(1/8)*PTS" {folder}/derivatives/merged-8x.mp4'
)
os.system(
f'ffmpeg -{overwrite} -i {folder}/merged.mp4 -c:v libx264 -b:v 10M -maxrate 10M -bufsize 10M -filter:v "setpts=(1/16)*PTS" {folder}/derivatives/merged-16x.mp4'
)
# Plants were too big for crop after end of March
#if folder_end_time < datetime(year = 2021, month = 3, day = 1):
# os.system(f'ffmpeg -{overwrite} -i {folder}/merged.mp4 -c:v libx264 -b:v 10M -maxrate 10M -bufsize 10M -vf crop=1200:800:450:500 {folder}/derivatives/merged-cropped.mp4')
# os.system(f'ffmpeg -{overwrite} -i {folder}/derivatives/merged-4x.mp4 -c:v libx264 -b:v 10M -maxrate 10M -bufsize 10M -vf crop=1200:800:450:500 {folder}/derivatives/merged-cropped-4x.mp4')
# os.system(f'ffmpeg -{overwrite} -i {folder}/derivatives/merged-8x.mp4 -c:v libx264 -b:v 10M -maxrate 10M -bufsize 10M -vf crop=1200:800:450:500 {folder}/derivatives/merged-cropped-8x.mp4')
# os.system(f'ffmpeg -{overwrite} -i {folder}/derivatives/merged-16x.mp4 -c:v libx264 -b:v 10M -maxrate 10M -bufsize 10M -vf crop=1200:800:450:500 {folder}/derivatives/merged-cropped-16x.mp4')
finally:
os.remove(tmp)
def intializeId(self):
ip, port = self.getIpAddress(), self.getPortNumber()
key = ip + ":" + str(port)
hashVal = helper.getHash(key)
self.setId(hashVal)
def getHash(self):
return getHash(self.state)