def publish_view(topic):
# Redirect if not primary
if primary_replica != my_address:
return redirect(primary_replica + '/publish/' + topic, code=307)
# CHECK IF TOPIC EXISTS
if not topic in metadata_manager.get_topics():
return 'Topic not found', 404
publish_lock[topic].acquire()
data = json.loads(request.data)
publish(topic, data)
for replica in replicas:
tries = 0
while tries < MAXTRIES:
try:
r = requests.post(replica + '/publish_repl/' + topic,
data=request.data,
timeout=1)
if r.status_code == 200:
print('Publish successful to replica : ' + replica)
break
except:
print("Timeout replica: %s try number: %s" % (replica, tries))
tries += 1
if tries == MAXTRIES:
print('Replica at ' + replica + ' is down!!')
remove_replica(replica)
publish_lock[topic].release()
return 'Success'
def read_view(topic, offset):
"""
Get message from topic with an offset
Parameters:
topic (str): Name of topic
offset (int): Offset of message starting from 1
Returns:
str: message
"""
if 'sub_name' not in request.args:
return 'Invalid request', 404
try:
offset = int(offset)
except:
return 'Invalid request', 404
subscriber_name = request.args['sub_name']
# CHECK IF TOPIC EXISTS
topic_list = metadata_manager.get_topics()
if topic not in topic_list:
return 'Topic does not exist', 404
# CHECK IF SUBSCRIBER SUBSCRIBED
if not metadata_manager.check_subscription(subscriber_name, topic):
return 'Subscriber not subscribed to topic', 404
# READ
data = read(topic, offset)
if data:
return data, 200
else:
return 'Offset invalid', 404
def publish_view_replica(topic):
if not topic in metadata_manager.get_topics():
return 'Topic not found', 404
data = json.loads(request.data)
publish_lock[topic].acquire()
publish(topic, data)
publish_lock[topic].release()
return 'Success'
def createtopic_view():
# Redirect if not primary
if primary_replica != my_address:
return redirect(primary_replica + '/createtopic', code=307)
try:
data = json.loads(request.data)
publisher_name = data['pub_name']
topic_name = data['topic_name']
except:
return 'Invalid POST data', 404
topic_list = metadata_manager.get_topics()
if topic_name in topic_list:
return 'Topic already exists', 404
# CREATE IF DOES NOT EXIST
metadata_manager.add_topic(topic_name)
# CREATE DATA DIRECTORY AND INDEX
try:
os.makedirs(os.path.join(storage_dir, topic_name))
with open(os.path.join(storage_dir, topic_name, 'index.pickle'),
'wb') as f:
pickle.dump([0], f)
except:
raise
return 'Topic creation failed', 404
# TODO remove topic
for replica in replicas:
tries = 0
while tries < MAXTRIES:
try:
r = requests.post(replica + '/createtopic_repl',
data=request.data,
timeout=1)
if r.status_code == 200:
print('Topic added successfully to replica : ' + replica)
break
except:
print("Timeout replica: %s try number: %s" % (replica, tries))
tries += 1
if tries == MAXTRIES:
print('Replica at ' + replica + ' is down!!')
remove_replica(replica)
publish_lock[topic_name] = Lock()
return 'Topic added successfully: ' + topic_name
def subscribe_view_replica():
# Check POST data
try:
data = json.loads(request.data)
subscriber_name = data['sub_name']
topic_name = data['topic_name']
except:
return 'Invalid POST data', 404
# CHECK IF TOPIC EXISTS
topic_list = metadata_manager.get_topics()
if topic_name not in topic_list:
return 'Topic does not exist', 404
# SUBSCRIBE
if metadata_manager.check_subscription(subscriber_name, topic_name):
return 'Already subscribed', 200
else:
subs = metadata_manager.add_subscription(subscriber_name, topic_name)
return 'Successfully subscribed', 200
def subscribe_view():
# Redirect to primary
if primary_replica != my_address:
return redirect(primary_replica + '/subscribe', code=307)
# Check POST data
try:
data = json.loads(request.data)
subscriber_name = data['sub_name']
topic_name = data['topic_name']
except:
return 'Invalid POST data', 404
# CHECK IF TOPIC EXISTS
topic_list = metadata_manager.get_topics()
if topic_name not in topic_list:
return 'Topic does not exist', 404
# SUBSCRIBE
if metadata_manager.check_subscription(subscriber_name, topic_name):
return 'Already subscribed', 200
else:
subs = metadata_manager.add_subscription(subscriber_name, topic_name)
for replica in replicas:
tries = 0
while tries < MAXTRIES:
try:
r = requests.post(replica + '/subscribe_repl',
data=request.data,
timeout=1)
if r.status_code == 200:
print('Subscription added successfully to replica : ' +
replica)
break
except:
print("Timeout replica: %s try number: %s" % (replica, tries))
tries += 1
if tries == MAXTRIES:
print('Replica at ' + replica + ' is down!!')
remove_replica(replica)
return 'Successfully subscribed', 200
def createtopic_view_replica():
try:
data = json.loads(request.data)
publisher_name = data['pub_name']
topic_name = data['topic_name']
except:
return 'Invalid POST data', 404
topic_list = metadata_manager.get_topics()
if topic_name in topic_list:
return 'Topic already exists', 404
# CREATE IF DOES NOT EXIST
metadata_manager.add_topic(topic_name)
# CREATE DATA DIRECTORY AND INDEX
try:
os.makedirs(os.path.join(storage_dir, topic_name))
with open(os.path.join(storage_dir, topic_name, 'index.pickle'),
'wb') as f:
pickle.dump([0], f)
except:
raise
return 'Topic creation failed', 404
# TODO remove topic
publish_lock[topic_name] = Lock()
return 'Topic added successfully: ' + topic_name
REPLICAS: %s
PRIMARY: %s
=================================""" %
(my_address, str(replicas), primary_replica))
storage_dir = './storage'
app = Flask(__name__)
MAXTRIES = 3
publish_lock = {}
replicas_lock = Lock()
# Load locks
topic_list = metadata_manager.get_topics()
for topic in topic_list:
publish_lock[topic] = Lock()
# Utility functions
def publish(topic, data):
# publish_lock[topic].acquire()
# Open index and data files
index_file = open(os.path.join(storage_dir, topic, 'index.pickle'), 'rb+')
old_index = pickle.load(index_file)
data_file = open(os.path.join(storage_dir, topic, 'data.dat'), 'ab')
for data_item in data:
data_item = data_item.encode()
old_index.append(old_index[-1] + len(data_item))
def test_get_topic(self):
metadata_manager.add_topic(topic1)
t = metadata_manager.add_topic(topic2)
t2 = metadata_manager.get_topics()
self.assertListEqual(t, t2)