def index():
db_handler = DBHandler()
all_providers = db_handler.get_all_providers()
providers_selected = np.random.choice(all_providers,
size=CINEMA_TO_SHOW,
replace=False)
is_auth = current_user.is_authenticated
negative_item = None
similar_user = None
if is_auth:
user_id = current_user.id
negative_item = find_negative_item(user_id)
similar_user = find_similar_user(user_id)
feedback = LikeForm()
if feedback.validate_on_submit():
db_handler.add_interaction(user_id=user_id, item_id=negative_item.id)
flash(flash_messages["got_feedback"])
redirect(url_for("index"))
return render_template(
"index.html",
providers=providers_selected,
negative_item=negative_item,
similar_user=similar_user,
is_auth=is_auth,
form=feedback,
)
def cinema():
db_handler = DBHandler()
id = request.args.get("id")
provider = db_handler.get_provider_by_id(id=id)
movies_in = db_handler.get_items_in_provider(provider_id=provider.id)
movies_id = [movie.id for movie in movies_in]
movies_obj = db_handler.get_items_by_ids(movies_id)
auth = current_user.is_authenticated
""" We are going to sort movies for user """
if auth:
model = ALSRecommender()
model.load()
scores = model.score(id, movies_id)
movies_obj = list(np.array(movies_obj)[np.argsort(scores)[::-1]])
movies_obj = movies_obj[:MOVIES_TO_SHOW]
return render_template(
"cinema.html",
provider=provider,
movies=movies_obj,
auth=auth,
len_movies=len(movies_obj),
)
def run(self):
while True:
time.sleep(1)
print(app.engineThread.waterLevel)
db_handler = DBHandler()
db_handler.save_to_db(app.engineThread.waterLevel,
app.engineThread.aim)
def alloc_port(src_ip, src_port, dst_ip, dst_port, switch, switch_port):
"""
Allocate a temporary EIP:port for this flow
"""
handler = DBHandler()
if src_port == 0:
return
nat_rule = handler.get(int_ip=src_ip)
if nat_rule:
eip = nat_rule[0].get('ext_ip')
for port in range(1, 65536):
if port not in used_ports.get(eip).keys():
#get unused flow ID
for flowid in range(0, 999):
if flowid not in used_flowid:
used_flowid.append(flowid)
break
else:
return 'ERROR'
used_ports.get(eip)[port] = (src_ip, src_port, dst_ip, dst_port, flowid)
#set SNAT flow
egress_flow = [src_ip, src_port, dst_ip, dst_port]
of_handler.set_snat(egress_flow, eip, port, 1000 + flowid, switch)
#set DNAT flow
ingress_flow = [dst_ip, dst_port, eip, port]
of_handler.set_dnat(ingress_flow, src_ip, src_port, 2000 + flowid, switch, switch_port)
#set ROUTE flow
of_handler.set_route(ingress_flow, int(switch) + 1, 3000 + flowid, GWID)
return json.dumps([eip, port])
return 'ERROR'
def __init__(self):
self.db = DBHandler()
self.file_handler = FileHandler()
self.db.create_database()
self.db.create_tables()
self.db.create_room_birthday_idx()
self.db.create_sex_index()
self.insert_data()
def bind_eip(bindings):
"""
Add NAT rules for each internal IP to EIP
"""
global used_ports
handler = DBHandler()
for int_ip in bindings['int_ip']:
#FIXME judge if the int_ip or ext_ip exists first
handler.add(int_ip, bindings['ext_ip'])
used_ports.setdefault(bindings['ext_ip'], {})
def unbind_eip(bindings):
"""
Delete NAT rules for each internal IP to EIP
"""
global used_ports
handler = DBHandler()
for int_ip in bindings['int_ip']:
handler.add(int_ip)
if not len(handler.get(ext_ip=bindings['ext_ip'])):
used_ports.pop(bindings['ext_ip'])
def get_nat():
"""
return all NAT rules
"""
handler = DBHandler()
ret = handler.get()
if ret:
return json.dumps(ret)
else:
return ''
def server_initialize(self, status, port, host_backup, port_backup):
if (status == 'active'):
self.server_tag = "A"
self.active = True
self.db = DBHandler(connection_string_1, "register-share-system-1")
self.start_timer_thread()
else:
self.server_tag = "B"
self.db = DBHandler(connection_string_2, "register-share-system-2")
self.port = port
self.host_backup = host_backup
self.port_backup = port_backup
def register():
db_handler = DBHandler()
if current_user.is_authenticated:
return redirect(url_for("index"))
form = RegistrationForm()
if form.validate_on_submit():
db_handler.add_user(name=form.name.data, password=form.password.data)
flash(flash_messages["new_user"])
return redirect(url_for("login"))
return render_template("register.html", title="Register", form=form)
def download_log_files(self):
jobs = self.service.jobs
blocking_search = {"exec_mode": "blocking"}
query = "search | head 5"
job = jobs.create(query, **blocking_search)
job_results = results.ResultsReader(job.results(count=5))
i = 0
mydb = DBHandler()
for result in job_results:
if True:
mydb.create_log_entry(i, result['_raw'], result['_indextime'],
result['host'], result['source'],
result['sourcetype'])
i += 1
def extract_timestamps_for_db(db_handler: DBHandler, terms: Sequence[str]):
term_to_timestamp = {}
for term in tqdm(terms, desc=db_handler.table_name):
timestamp_counter = Counter()
if db_handler.table_name == 'darkode':
comments, timestamps = db_handler.fetch_comments(term)
count_occurrences(timestamps, comments, term, timestamp_counter)
else:
comments, _ = db_handler.fetch_comments(term)
for comment in comments:
timestamps, texts = parse_comments(comment,
db_handler.table_name)
count_occurrences(timestamps, texts, term, timestamp_counter)
term_to_timestamp[term] = pd.Series(list(timestamp_counter.values()),
list(timestamp_counter.keys()))
return term_to_timestamp
def find_similar_user(user_id):
db_handler = DBHandler()
user_factors = np.load(CURRENT_USER_FACTORS_PATH, allow_pickle=True)
if user_id > user_factors.shape[0]:
most_similar_user_id = np.random.choice(user_factors.shape[0])
else:
user_factor = user_factors[user_id - 1]
dot_product = (user_factor * user_factors).sum(1)
indexes = np.argsort(dot_product)[::-1]
most_similar_user_id = int(indexes[-2])
print(most_similar_user_id, type(most_similar_user_id))
most_similar_user = db_handler.get_user_by_id(user_id=most_similar_user_id)
return most_similar_user
def find_negative_item(user_id):
db_handler = DBHandler()
all_items = db_handler.get_all_items()
items_ids = [item.id for item in all_items]
interacted_items = db_handler.get_interactions_by_id(user_id=user_id)
interacted_items_id = [item.item_id for item in interacted_items]
not_interacted_items_id = list(set(items_ids) - set(interacted_items_id))
not_interacted_items = db_handler.get_items_by_ids(not_interacted_items_id)
if not_interacted_items is None:
return None
else:
not_interacted_item = np.random.choice(not_interacted_items)
return not_interacted_item
def __init__(self, cfg):
self.cfg = cfg
self.downloader = Downloader(cfg)
self.logger = Logger(cfg)
self.parser = AnotherHTMLParser(self.logger)
self.pairs = set()
self.db_handler = DBHandler(cfg)
self._word_dict = None
def expire_port(src_ip, src_port, dst_ip, dst_port, switch, switch_port):
"""
Expire a temporary EIP:port for this flow
"""
handler = DBHandler()
nat_rule = handler.get(int_ip=src_ip)
if nat_rule:
eip = nat_rule[0].get('ext_ip')
for port in used_ports.get(eip):
if used_ports.get(eip).get(port)[:4] == (src_ip, src_port, dst_ip, dst_port):
flow = used_ports.get(eip).pop(port)
flowid = flow.pop(4)
#set flowid unused
used_flowid.remove(flowid)
#rm ROUTE flow
ingress_flow = [dst_ip, dst_port, eip, src_port]
of_handler.set_route(ingress_flow, switch + 1, flowid, GWID, method='DELETE')
return 'OK'
def login():
db_handler = DBHandler()
if current_user.is_authenticated:
return redirect(url_for("index"))
form = LoginForm()
if form.validate_on_submit():
user = db_handler.get_user_by_name(name=form.username.data)
if user is None or not user.check_password(form.password.data):
flash(flash_messages["invalid_login"])
return redirect(url_for("login"))
login_user(user, remember=form.remember_me.data)
next_page = request.args.get("next")
flash(flash_messages["login_ok"])
if not next_page or url_parse(next_page).netloc != "":
next_page = url_for("index")
return redirect(next_page)
return render_template("login.html", title="Sign In", form=form)
def count_predictions(
config: dict, db_name: str, dark_terms: List[str]
) -> Tuple[List[WordWindow], DefaultDict[str, Counter]]:
hdlr = DBHandler(config['data']['db_dir'], db_name)
model = WordPredictModel(config['mask']['model_name'])
word_windows = _fetch_word_windows(dark_terms, hdlr,
config['mask']['window_size'])
counters = _predict_masked_words(word_windows, model,
config['mask']['top_k'])
return word_windows, counters
def main():
# give functionality for creating table, inserting into table, deleting from table, retrieving data from table
conn = create_connection("test_amaris.db")
while True:
db_handler = DBHandler(conn)
print_help("init")
curr = cmd_handler(db_handler)
if curr == False:
break
finish(conn)
def main():
running = True
message_queue = queue.Queue()
print("Creating mqtt class..")
mqtt_thread = MqttConnector()
mqtt_thread.start()
## Start sensors
temp_sensor = TemperatureSensor(message_queue)
#ir_temp_sensor = IRTemperatureSensor(message_queue)
#ir2_temp_sensor = IR2TemperatureSensor(message_queue)
#gps_sensor = GpsSensor(message_queue)
humidity_sensor = HumiditySensor(message_queue)
pressure_sensor = PressureSensor(message_queue)
orientation_sensor = OrientationSensor(message_queue)
acceleration_sensor = AccelerationSensor(message_queue)
imu_sensor = UDPReceiver(cfg.ip_imu, cfg.port_imu, message_queue, "IMU")
seanav_sensor = UDPReceiver(cfg.ip_seanav, cfg.port_seanav, message_queue,
"SEANAV")
temp_sensor.start()
#ir_temp_sensor.start()
#ir2_temp_sensor.start()
#gps_sensor.start()
humidity_sensor.start()
pressure_sensor.start()
orientation_sensor.start()
acceleration_sensor.start()
imu_sensor.start()
seanav_sensor.start()
## Connect to databases
db_handler = DBHandler(local=True, cloud=True)
## Wait for connected to server
while not mqtt_thread.connected:
continue
print("Connected...")
print("Start message sender")
sender = MessageSender(message_queue, mqtt_thread, db_handler)
sender.start()
## Keep from stopping program
while (running):
continue
print("Program end")
def _fetch_word_windows(dark_terms: List[str], hdlr: DBHandler,
window_size: int) -> Union[List[WordWindow], list]:
word_windows = []
for dark_term in tqdm(dark_terms, desc='dark_terms'):
result, _ = hdlr.fetch_comments(dark_term)
for comments in result:
for comment in sent_tokenize(comments):
ww = word_window(clean_str(comment),
dark_term,
size=window_size // 2)
if ww:
left, right = ww
word_windows.append(WordWindow(dark_term, left, right))
return word_windows
def main():
try:
data_handler: DataHandler = DBHandler()
checker: Checker = Checker(data_handler)
# Create the parser
input_path = get_csv_path()
print("Welcome to Vulnerability Checker")
if input_path is None:
print("Scanning the all packages on machine for vulnerabilities...\n")
checker.scan_machine()
else:
print("Scanning the given packages for vulnerabilities...\n")
checker.scan_user_list(input_path)
except User_Exit_Interrupt:
print("\n\nUser stopped the scan. Exiting program...")
sys.exit(0)
except KeyboardInterrupt:
print("\n\nUser stopped the scan. Exiting program...")
sys.exit(0)
def extract_timestamps(config: dict, dark_terms: Sequence[str]):
timestamps = {}
db_names = config['data']['db_names'][:3]
for db_name in db_names:
db_handler = DBHandler(config['data']['db_dir'], db_name)
table_name = db_handler.table_name
timestamps[table_name] = extract_timestamps_for_db(
db_handler, dark_terms)
total = timestamps.get('total', {})
for dark_term in dark_terms:
if dark_term not in total:
series = timestamps[table_name][dark_term]
else:
series1 = timestamps['total'][dark_term]
series2 = timestamps[table_name][dark_term]
series = (series1 + series2).fillna(series2).fillna(series1)
total[dark_term] = series
timestamps['total'] = total
print('writing timestamps for {}'.format(db_name))
pkl.dump(timestamps, open(dict_out, 'wb'))
def main():
# INIT
db = DBHandler()
# id name lvl curr_exp base_health base_damage curr_location inventory
hero_res = start_menu(db)
while hero_res == -1:
hero_res = start_menu(db)
# INVENTORY
t = pypika.Table('inventories')
q = pypika.Query.from_(t).select('*').where(t.id == hero_res[7]).get_sql()
# id capacity size
inv_res = db.select(q)[0]
heros_inv = Inventory(inv_res[0], inv_res[1], inv_res[2], db)
# LOCATIONS
init_locations(db)
for l in LOCATIONS:
if l.id == hero_res[6]:
global CUR_LOCATION
CUR_LOCATION = l
# PYGAME
pygame.init()
pygame.key.set_repeat(100, 300)
clock = pygame.time.Clock()
screen = pygame.display.set_mode([DISPLAY_W, DISPLAY_H], pygame.DOUBLEBUF)
mapp = Map(MAP_W, MAP_H)
mapp.random_fill(create_monsters_list(db))
hero = Character(hero_res[0], hero_res[1], hero_res[2], hero_res[3],
hero_res[4], hero_res[5], heros_inv, round(mapp.w / 2),
round(mapp.h / 2), CUR_LOCATION,
make_pygame_image("assets/new_hero.png"), hero_res[-1],
hero_res[-2])
set_skills_to_hero(hero, db)
constants.BACKGROUND_COLOR = hero.curr_location.rgb
inventory_surf = pygame.Surface((DISPLAY_W - MAP_W, DISPLAY_H / 2))
inventory_surf.fill(INVENTORY_COLOR)
skills_surf = pygame.Surface((DISPLAY_W - MAP_W, DISPLAY_H / 2))
skills_surf.fill(SKILLS_COLOR)
in_inventory_mode = False
in_skills_mode = False
in_skill_direction_choosing_mode = False
target_is_found = False
target = None
screen.fill(constants.BACKGROUND_COLOR)
for i, it in mapp:
if it != 0:
screen.blit(it.img, (it.x * TILE_SIZE, it.y * TILE_SIZE))
if isinstance(it, Monster):
health_bar = pygame.Surface(
(TILE_SIZE * (it.health / it.base_health),
int(TILE_SIZE / 10)))
health_bar.fill(hex_to_rgb('#00FF00'))
screen.blit(health_bar, (it.x * TILE_SIZE, (it.y * TILE_SIZE)))
health_bar = pygame.Surface(
(TILE_SIZE * (hero.curr_health / hero.get_total_health()),
int(TILE_SIZE / 10)))
health_bar.fill(hex_to_rgb('#00FF00'))
screen.blit(health_bar, (hero.x * TILE_SIZE, (hero.y * TILE_SIZE) - 15))
mana_bar = pygame.Surface(
(TILE_SIZE * (hero.curr_mana / hero.total_mana), int(TILE_SIZE / 10)))
mana_bar.fill(hex_to_rgb('#0000FF'))
screen.blit(mana_bar, (hero.x * TILE_SIZE, (hero.y * TILE_SIZE) - 10))
screen.blit(hero.img, (hero.x * TILE_SIZE, hero.y * TILE_SIZE))
screen.blit(inventory_surf, (MAP_W, 0))
screen.blit(skills_surf, (MAP_W, DISPLAY_H / 2))
draw_UI(screen, hero, in_inventory_mode, in_skills_mode,
in_skill_direction_choosing_mode)
pygame.display.update()
running = True
while running and hero.curr_health > 0:
clock.tick(FPS)
for e in pygame.event.get():
if e.type == pygame.QUIT:
running = False
if e.type == pygame.KEYDOWN:
global CUR_ITEM
global CUR_SKILL
if e.key == pygame.K_ESCAPE:
running = False
break
if e.key == pygame.K_i:
if not in_inventory_mode:
in_skills_mode = False
in_inventory_mode = not in_inventory_mode
CUR_ITEM = 0
if e.key == pygame.K_s:
if not in_skills_mode:
in_inventory_mode = False
in_skills_mode = not in_skills_mode
CUR_SKILL = 0
if in_inventory_mode:
if e.key == pygame.K_UP:
if CUR_ITEM != 0:
CUR_ITEM -= 1
if e.key == pygame.K_DOWN:
if CUR_ITEM < hero.inventory.size - 1:
CUR_ITEM += 1
if e.key == pygame.K_g:
index = hero.x + hero.y * mapp.w
mmmap = mapp.cells[index]
if isinstance(mmmap, Item):
if hero.take_item(mmmap):
mapp.cells[index] = 0
if e.key == pygame.K_d:
hero.drop_item(hero.inventory.items[CUR_ITEM])
elif in_skills_mode:
if e.key == pygame.K_UP:
if CUR_SKILL != 0:
CUR_SKILL -= 1
if e.key == pygame.K_DOWN:
if CUR_SKILL < len(hero.skills) - 1:
CUR_SKILL += 1
if e.key == pygame.K_RETURN:
skill = hero.skills[CUR_SKILL]
if skill.cost > hero.curr_mana:
break
if skill.type == 'heal':
skill.use_on_target(hero)
skill_animation(hero, hero.x, hero.y, screen)
hero.curr_mana -= skill.cost
in_skills_mode = False
if skill.type == 'dmg':
in_skill_direction_choosing_mode = True
in_skills_mode = False
elif in_skill_direction_choosing_mode:
if e.key == pygame.K_UP:
ind = hero.x + hero.y * mapp.w
while ind >= 0:
if isinstance(mapp.cells[ind], Monster):
target_is_found = True
target = mapp.cells[ind]
break
ind -= mapp.w
elif e.key == pygame.K_DOWN:
ind = hero.x + hero.y * mapp.w
while ind <= mapp.h * mapp.w:
if isinstance(mapp.cells[ind], Monster):
target_is_found = True
target = mapp.cells[ind]
break
ind += mapp.w
elif e.key == pygame.K_LEFT:
ind = hero.x + hero.y * mapp.w
while ind >= hero.y * mapp.w:
if isinstance(mapp.cells[ind], Monster):
target_is_found = True
target = mapp.cells[ind]
break
ind -= 1
elif e.key == pygame.K_RIGHT:
ind = hero.x + hero.y * mapp.w
while ind <= hero.y * mapp.w + mapp.w:
if isinstance(mapp.cells[ind], Monster):
target_is_found = True
target = mapp.cells[ind]
break
ind += 1
else:
target_is_found = False
target = None
in_skill_direction_choosing_mode = False
if target_is_found:
hero.curr_mana -= skill.cost
skill_animation(hero, target.x, target.y, screen)
hero.skills[CUR_SKILL].use_on_target(target)
if target.health <= 0:
mapp.cells[target.x +
target.y * mapp.w] = target.item
target.item.x = target.x
target.item.y = target.y
leveling(hero, target.exp, db)
target_is_found = False
target = None
in_skill_direction_choosing_mode = False
else:
if e.key == pygame.K_UP:
move(hero, mapp, 0, -1, db)
if e.key == pygame.K_DOWN:
move(hero, mapp, 0, 1, db)
if e.key == pygame.K_LEFT:
move(hero, mapp, -1, 0, db)
if e.key == pygame.K_RIGHT:
move(hero, mapp, 1, 0, db)
if hero.curr_health <= 0:
break
hero.regenerate()
screen.fill(constants.BACKGROUND_COLOR)
for i, it in mapp:
if it != 0:
screen.blit(it.img,
(it.x * TILE_SIZE, it.y * TILE_SIZE))
if isinstance(it, Monster):
health_bar = pygame.Surface(
(TILE_SIZE * (it.health / it.base_health),
int(TILE_SIZE / 10)))
health_bar.fill(hex_to_rgb('#00FF00'))
screen.blit(health_bar,
(it.x * TILE_SIZE, (it.y * TILE_SIZE)))
health_bar = pygame.Surface(
(TILE_SIZE * (hero.curr_health / hero.get_total_health()),
int(TILE_SIZE / 10)))
health_bar.fill(hex_to_rgb('#00FF00'))
screen.blit(health_bar,
(hero.x * TILE_SIZE, (hero.y * TILE_SIZE) - 15))
mana_bar = pygame.Surface(
(TILE_SIZE * (hero.curr_mana / hero.total_mana),
int(TILE_SIZE / 10)))
mana_bar.fill(hex_to_rgb('#0000FF'))
screen.blit(mana_bar,
(hero.x * TILE_SIZE, (hero.y * TILE_SIZE) - 10))
screen.blit(hero.img, (hero.x * TILE_SIZE, hero.y * TILE_SIZE))
screen.blit(inventory_surf, (MAP_W, 0))
screen.blit(skills_surf, (MAP_W, DISPLAY_H / 2))
draw_UI(screen, hero, in_inventory_mode, in_skills_mode,
in_skill_direction_choosing_mode)
pygame.display.update()
if hero.curr_health <= 0:
print('Your character is totally dead :(')
print('Make another one and good luck!')
address_download = args["address_download"]
roads_download = args["roads_download"]
delete_err = args["delete_err"]
move_self_intersect = args["move_self_intersect"]
move_intersect = args["move_intersect"]
dsn = args["dsn"]
intersect = args["intersect"]
address = args["assign_address"]
check_address = args["check_address"]
check_road_rail = args["check_road_rail"]
vacuum = args["vacuum"]
report = args["report"]
index = args["index_data"]
bbox = args["bbox"]
db = DBHandler(dsn)
osm = OSMHandler(bbox)
if setup:
print 'Setting up the database.'
db.setup_db()
if building_download:
print 'Querying OverpassAPI for buildings.'
buildings = osm.query_buildings()
print 'Uploading OSM buildings to Postgres...'
db.upload_osm(buildings, 'osm_buildings')
if address_download:
print 'Querying OverpassAPI for addresses.'
addresses = osm.query_address()
def main():
handler = DBHandler(DB_NAME, DB_USER, DB_PASSWORD)
with handler:
handler.insert_values('Teams', ('Team 2', True, 't2_password'))
print(handler.query_list('SELECT * FROM Teams;'))
handler.insert_values('Bids', ('bid_1', 4.56, False))
handler.insert_values('Bids', ('bid_2', 7.89, False))
print(handler.query_list('SELECT * FROM Bids;'))
handler.update_values('Bids', "bid_id = 'bid_2'", ("price", ),
('1.23', ))
print(handler.query_list('SELECT * FROM Bids;'))
def handle_message(self, message_dict, address):
message_type = message_dict["TYPE"]
self.write_to_log('MESSAGE RECEIVED\t [' + str(address) + ']:\t ' +
str(message_dict))
if (message_type == "INITIALIZATION"):
self.client_list.append(address)
self.semaphore.acquire()
try:
if self.active == True:
# Notify the client about which server is the active one ("A" or "B")
msg = {
"TYPE": "INITIALIZATION-SUCCESS",
"SERVER-TAG": self.server_tag
}
msg_serialized = utils.serialize(msg)
self.sock.sendto(msg_serialized, address)
self.write_to_log('MESSAGE SENT\t\t [' + str(address) +
']:\t ' + str(msg))
finally:
self.semaphore.release()
elif (message_type == "REGISTER"):
success = self.db.add_user(message_dict["NAME"],
message_dict["IP"],
message_dict["PORT"])
if (success == 0):
msg = {"TYPE": "REGISTER-SUCCESS", "RQ#": message_dict["RQ#"]}
self.semaphore.acquire()
try:
if self.active == True:
self.sock.sendto(utils.serialize(msg), address)
self.sock.sendto(utils.serialize(msg),
(self.host_backup, self.port_backup))
self.write_to_log('MESSAGE SENT\t\t [' + str(address) +
']:\t ' + str(msg))
self.write_to_log('MESSAGE SENT\t\t [' +
str((self.host_backup,
self.port_backup)) + ']:\t ' +
str(msg))
finally:
self.semaphore.release()
else:
if (success == 1):
msg = {
"TYPE": "REGISTER-DENIED",
"RQ#": message_dict["RQ#"],
"REASON": "USERNAME ALREADY EXISTS"
}
elif (success == 2):
msg = {
"TYPE": "REGISTER-DENIED",
"RQ#": message_dict["RQ#"],
"REASON": "NOT A VALID USERNAME"
}
elif (success == 3):
msg = {
"TYPE": "REGISTER-DENIED",
"RQ#": message_dict["RQ#"],
"REASON": "DATABASE CONNECTION ERROR"
}
self.semaphore.acquire()
try:
if self.active == True:
self.sock.sendto(utils.serialize(msg), address)
self.sock.sendto(utils.serialize(msg),
(self.host_backup, self.port_backup))
self.write_to_log('MESSAGE SENT\t\t [' + str(address) +
']:\t ' + str(msg))
self.write_to_log('MESSAGE SENT\t\t [' +
str((self.host_backup,
self.port_backup)) + ']:\t ' +
str(msg))
finally:
self.semaphore.release()
elif (message_type == "DE-REGISTER"):
success = self.db.remove_user(message_dict["NAME"])
if (success):
msg_client = {
"TYPE": "DE-REGISTER-SUCCESS",
"RQ#": message_dict["RQ#"]
}
msg_server = {
"TYPE": "DE-REGISTER-SUCCESS",
"RQ#": message_dict["RQ#"],
"NAME": message_dict["NAME"]
}
self.sock.sendto(utils.serialize(msg_client), address)
self.sock.sendto(utils.serialize(msg_server),
(self.host_backup, self.port_backup))
self.write_to_log('MESSAGE SENT\t\t [' + str(address) +
']:\t ' + str(msg_client))
self.write_to_log('MESSAGE SENT\t\t [' +
str((self.host_backup, self.port_backup)) +
']:\t ' + str(msg_server))
else:
msg_client = {
"TYPE": "DE-REGISTER-DENIED",
"RQ#": message_dict["RQ#"]
}
msg_server = {
"TYPE": "DE-REGISTER-DENIED",
"RQ#": message_dict["RQ#"],
"NAME": message_dict["NAME"]
}
self.sock.sendto(utils.serialize(msg_client), address)
self.sock.sendto(utils.serialize(msg_server),
(self.host_backup, self.port_backup))
self.write_to_log('MESSAGE SENT\t\t [' + str(address) +
']:\t ' + str(msg_client))
self.write_to_log('MESSAGE SENT\t\t [' +
str((self.host_backup, self.port_backup)) +
']:\t ' + str(msg_server))
elif (message_type == "UPDATE-SOCKET"):
success = self.db.update_socket(message_dict["NAME"],
message_dict["IP"],
message_dict["PORT"])
if (success):
msg = {
"TYPE": "UPDATE-SOCKET-SUCCESS",
"RQ#": message_dict["RQ#"],
"NAME": message_dict["NAME"],
"IP": message_dict["IP"],
"PORT": message_dict["PORT"]
}
self.sock.sendto(utils.serialize(msg), address)
self.sock.sendto(utils.serialize(msg),
(self.host_backup, self.port_backup))
self.write_to_log('MESSAGE SENT\t\t [' + str(address) +
']:\t ' + str(msg))
self.write_to_log('MESSAGE SENT\t\t [' +
str((self.host_backup, self.port_backup)) +
']:\t ' + str(msg))
else:
msg = {
"TYPE": "UPDATE-SOCKET-DENIED",
"RQ#": message_dict["RQ#"],
"NAME": message_dict["NAME"],
"IP": message_dict["IP"],
"PORT": message_dict["PORT"]
}
self.sock.sendto(utils.serialize(msg), address)
self.sock.sendto(utils.serialize(msg),
(self.host_backup, self.port_backup))
self.write_to_log('MESSAGE SENT\t\t [' + str(address) +
']:\t ' + str(msg))
self.write_to_log('MESSAGE SENT\t\t [' +
str((self.host_backup, self.port_backup)) +
']:\t ' + str(msg))
elif (message_type == "SUBJECTS"):
subjects_list = message_dict["SUBJECT-LIST"].split(",")
print(subjects_list)
success = self.db.update_subjects(message_dict["NAME"],
subjects_list)
if (success):
msg = {
"TYPE": "UPDATE-SUBJECTS-SUCCESS",
"RQ#": message_dict["RQ#"],
"NAME": message_dict["NAME"],
"SUBJECT-LIST": message_dict["SUBJECT-LIST"]
}
self.sock.sendto(utils.serialize(msg), address)
self.sock.sendto(utils.serialize(msg),
(self.host_backup, self.port_backup))
self.write_to_log('MESSAGE SENT\t\t [' + str(address) +
']:\t ' + str(msg))
self.write_to_log('MESSAGE SENT\t\t [' +
str((self.host_backup, self.port_backup)) +
']:\t ' + str(msg))
else:
msg = {
"TYPE": "UPDATE-SUBJECTS-DENIED",
"RQ#": message_dict["RQ#"],
"NAME": message_dict["NAME"],
"SUBJECT-LIST": message_dict["SUBJECT-LIST"]
}
self.sock.sendto(utils.serialize(msg), address)
self.sock.sendto(utils.serialize(msg),
(self.host_backup, self.port_backup))
self.write_to_log('MESSAGE SENT\t\t [' + str(address) +
']:\t ' + str(msg))
self.write_to_log('MESSAGE SENT\t\t [' +
str((self.host_backup, self.port_backup)) +
']:\t ' + str(msg))
elif (message_type == "PUBLISH"):
success = self.db.publish_message(message_dict["NAME"],
message_dict["SUBJECT"],
message_dict["TEXT"])
if (success):
msg = {
"TYPE": "PUBLISH-SUCCESS",
"RQ#": message_dict["RQ#"],
"NAME": message_dict["NAME"],
"SUBJECT": message_dict["SUBJECT"],
"TEXT": message_dict["TEXT"]
}
self.sock.sendto(utils.serialize(msg), address)
self.sock.sendto(utils.serialize(msg),
(self.host_backup, self.port_backup))
self.write_to_log('MESSAGE SENT\t\t [' + str(address) +
']:\t ' + str(msg))
self.write_to_log('MESSAGE SENT\t\t [' +
str((self.host_backup, self.port_backup)) +
']:\t ' + str(msg))
else:
msg = {
"TYPE": "PUBLISH-DENIED",
"RQ#": message_dict["RQ#"],
"NAME": message_dict["NAME"],
"SUBJECT": message_dict["SUBJECT"],
"TEXT": message_dict["TEXT"]
}
self.sock.sendto(utils.serialize(msg), address)
self.sock.sendto(utils.serialize(msg),
(self.host_backup, self.port_backup))
self.write_to_log('MESSAGE SENT\t\t [' + str(address) +
']:\t ' + str(msg))
self.write_to_log('MESSAGE SENT\t\t [' +
str((self.host_backup, self.port_backup)) +
']:\t ' + str(msg))
elif (message_type == "RETRIEVE-TEXTS"):
msg_list = self.db.retrieve_texts(message_dict["NAME"])
if (msg_list != None):
msg = {
"TYPE": "RETRIEVE-SUCCESS",
"RQ#": message_dict["RQ#"],
"NAME": message_dict["NAME"],
"POSTS": msg_list
}
self.sock.sendto(utils.serialize(msg), address)
self.sock.sendto(utils.serialize(msg),
(self.host_backup, self.port_backup))
self.write_to_log('MESSAGE SENT\t\t [' + str(address) +
']:\t ' + str(msg))
self.write_to_log('MESSAGE SENT\t\t [' +
str((self.host_backup, self.port_backup)) +
']:\t ' + str(msg))
else:
msg = {"TYPE": "RETRIEVE-DENIED", "RQ#": message_dict["RQ#"]}
self.sock.sendto(utils.serialize(msg), address)
self.sock.sendto(utils.serialize(msg),
(self.host_backup, self.port_backup))
self.write_to_log('MESSAGE SENT\t\t [' + str(address) +
']:\t ' + str(msg))
self.write_to_log('MESSAGE SENT\t\t [' +
str((self.host_backup, self.port_backup)) +
']:\t ' + str(msg))
elif (message_type == "CHANGE-SERVER"):
# If the server receives this message, it gains control
self.gain_control()
self.start_timer_thread()
elif (message_type == "UPDATE-SERVER"):
self.semaphore.acquire()
try:
if self.active == True:
# If a server receives this message, it means it the inactive server has
# changed its IP and PORT. Thus, we should update these values
self.host_backup = message_dict["IP"]
self.port_backup = message_dict["PORT"]
# We should now reply to the new server that we have taken note of its IP and PORT
# We also send it our IP and PORT number so it knows which one is the active server
msg_reply = {
"TYPE": "UPDATE-SERVER-SUCCESS",
"IP": self.host,
"PORT": self.port,
"SERVER-TAG": self.server_tag,
"CLIENT-LIST": self.client_list
}
msg_reply_serialized = utils.serialize(msg_reply)
self.sock.sendto(msg_reply_serialized, address)
self.write_to_log('MESSAGE SENT\t\t [' + str(address) +
']:\t ' + str(msg_reply))
# else if the inactive server receives this message, it should send a message
# to all the connected clients so they know where the new inactive server is
else:
new_host = message_dict["IP"]
new_port = message_dict["PORT"]
msg_client = {
"TYPE": "UPDATE-SERVER",
"IP": new_host,
"PORT": new_port,
"SERVER-TAG": self.server_tag
}
msg_client_serialized = utils.serialize(msg_client)
for i in self.client_list:
self.sock.sendto(msg_client_serialized, i)
self.write_to_log('MESSAGE SENT\t\t [' + str(i) +
']:\t ' + str(msg_client))
self.write_to_log("SERVER-CLOSED")
print("SERVER-CLOSED")
except:
print("ERROR SENDING UPDATE-SERVER MESSAGE")
msg_error = {"TYPE": "UPDATE-SERVER-DENIED"}
msg_error_serialized = utils.serialize(msg_error)
self.sock.sendto(msg_error_serialized, address)
self.write_to_log('MESSAGE SENT\t\t [' + str(address) +
']:\t ' + str(msg_error))
finally:
self.semaphore.release()
elif (message_type == "END-CONNECTION"):
try:
self.client_list.remove(address)
except:
pass
elif ((message_type == "REGISTER-SUCCESS")
or (message_type == "REGISTER-DENIED")):
pass
elif ((message_type == "DE-REGISTER-SUCCESS")
or (message_type == "DE-REGISTER-DENIED")):
try:
if (message_type == "DE-REGISTER-SUCCESS"):
success = self.db.remove_user(message_dict["NAME"])
except Exception as msg:
print(str(msg))
elif ((message_type == "UPDATE-SOCKET-SUCCESS")
or (message_type == "UPDATE-SOCKET-DENIED")):
try:
if (message_type == "UPDATE-SOCKET-SUCCESS"):
self.db.update_socket(message_dict["NAME"],
message_dict["IP"],
message_dict["PORT"])
except:
pass
elif ((message_type == "UPDATE-SUBJECTS-SUCCESS")
or (message_type == "UPDATE-SUBJECTS-DENIED")):
subjects_list = message_dict["SUBJECT-LIST"].split(",")
try:
if (message_type == "UPDATE-SUBJECTS-SUCCESS"):
self.db.update_subjects(message_dict["NAME"],
subjects_list)
except:
pass
elif ((message_type == "PUBLISH-SUCCESS")
or (message_type == "PUBLISH-DENIED")):
try:
if (message_type == "PUBLISH-SUCCESS"):
self.db.publish_message(message_dict["NAME"],
message_dict["SUBJECT"],
message_dict["TEXT"])
except:
pass
elif ((message_type == "RETRIEVE-SUCCESS")
or (message_type == "RETRIEVE-DENIED")):
try:
if (message_type == "RETRIEVE-SUCCESS"):
self.db.retrieve_texts(message_dict["NAME"])
except:
pass
elif ((message_type == "UPDATE-SERVER-SUCCESS")
or (message_type == "UPDATE-SERVER-DENIED")):
try:
if (message_type == "UPDATE-SERVER-SUCCESS"):
if message_dict["SERVER-TAG"] == "A":
self.server_tag = "B"
self.db = DBHandler(connection_string_2,
"register-share-system-2")
else:
self.server_tag = 'A'
self.db = DBHandler(connection_string_1,
"register-share-system-1")
self.host_backup = message_dict["IP"]
self.port_backup = message_dict["PORT"]
self.client_list = utils.convert(
message_dict["CLIENT-LIST"])
except:
pass
else:
pass
class Server:
def __init__(self):
self.sock = None # socket
self.host = None
self.port = 8888
self.log_file_path = ''
self.active = False # Indicates if the server is active or not
self.host_backup = None # Reference to the IP of the other server
self.port_backup = 0 # Reference to the Port of the other server
self.semaphore = threading.Semaphore(
1) # Semaphore to protect the critical section ("active" variable)
self.client_list = [
] # This is a list of addresses of all the connected clients
self.db = None # Reference to MongoDB database
self.server_tag = "B" # This is simply a tag for the server ("A" or "B")
def change_server(self, address):
msg = {"TYPE": "CHANGE-SERVER", "IP": self.host, "PORT": self.port}
msg_serialized = utils.serialize(msg)
self.sock.sendto(msg_serialized, address)
# Send this message to the other server
self.write_to_log('MESSAGE SENT\t\t [' + str(address) + ']:\t ' +
str(msg))
# Also send this message to all the connected clients
try:
for i in self.client_list:
self.sock.sendto(msg_serialized, i)
self.write_to_log('MESSAGE SENT\t\t [' + str(i) + ']:\t ' +
str(msg))
except:
print("ERROR SENDING CHANGE-SERVER MESSAGE TO ALL CLIENTS")
def gain_control(self):
self.semaphore.acquire()
self.active = True
self.semaphore.release()
def start_timer(self):
# Set the timer to a duration of 120 seconds (2 minutes)
endtime = time.time() + 120
# Busy wait
while time.time() < endtime:
pass
self.semaphore.acquire()
self.active = False
self.semaphore.release()
# We should now send a message to the other server to inform it to take
self.change_server((self.host_backup, self.port_backup))
def start_timer_thread(self):
timer_thread = threading.Thread(target=self.start_timer)
timer_thread.start()
# This function is called the first time we initialize server A or server B (1 active & 1 inactive)
def server_initialize(self, status, port, host_backup, port_backup):
if (status == 'active'):
self.server_tag = "A"
self.active = True
self.db = DBHandler(connection_string_1, "register-share-system-1")
self.start_timer_thread()
else:
self.server_tag = "B"
self.db = DBHandler(connection_string_2, "register-share-system-2")
self.port = port
self.host_backup = host_backup
self.port_backup = port_backup
# This function is called when there are two servers already in the system, but we want to
# change the IP/Port of the inactive server
def new_server_initialize(self, port):
self.port = port
# This function sends an "UPDATE-SERVER" message to both servers that are already in the system
# We refer to these as server A and server B
def update_server(self, host_a, port_a, host_b, port_b):
msg = {"TYPE": "UPDATE-SERVER", "IP": self.host, "PORT": self.port}
msg_serialized = utils.serialize(msg)
try:
self.sock.sendto(msg_serialized, (host_a, port_a))
self.sock.sendto(msg_serialized, (host_b, port_b))
except:
print("FAILED TO SEND UPDATE-SERVER MESSAGE")
def write_to_log(self, msg):
log_file = open(self.log_file_path, "a+")
log_file.write(str(msg) + "\n")
log_file.close()
def initialize_log_file(self):
date_str_temp = str(datetime.datetime.now())
date_str = date_str_temp.replace(
" ", "-") # This replaces spaces in the file path with a '-'
date_str = date_str.replace(
".", "-") # This replaces spaces in the file path with a '-'
date_str = date_str.replace(
":", "-") # This replaces spaces in the file path with a '-'
self.log_file_path = "logs/server/" + "server-" + date_str + ".txt"
self.write_to_log("----- Server Log -----")
def create_socket(self):
try:
# printing the IP address of the host
print(socket.gethostbyname(socket.gethostname()))
# configure host IP address
self.host = socket.gethostbyname(socket.gethostname())
# create the socket
self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.write_to_log(socket.gethostbyname(socket.gethostname()))
self.write_to_log(self.port)
self.write_to_log('Server socket created ')
except OSError as msg:
print('Failed to create socket. Error Code : ' + str(msg))
sys.exit()
def bind_socket(self):
try:
self.sock.bind((self.host, self.port))
print('Socket bind complete')
except OSError as msg:
print('Bind failed. Error Code : ' + str(msg))
sys.exit()
# Process messages (from either client or other server) and take appropriate action
def handle_message(self, message_dict, address):
message_type = message_dict["TYPE"]
self.write_to_log('MESSAGE RECEIVED\t [' + str(address) + ']:\t ' +
str(message_dict))
if (message_type == "INITIALIZATION"):
self.client_list.append(address)
self.semaphore.acquire()
try:
if self.active == True:
# Notify the client about which server is the active one ("A" or "B")
msg = {
"TYPE": "INITIALIZATION-SUCCESS",
"SERVER-TAG": self.server_tag
}
msg_serialized = utils.serialize(msg)
self.sock.sendto(msg_serialized, address)
self.write_to_log('MESSAGE SENT\t\t [' + str(address) +
']:\t ' + str(msg))
finally:
self.semaphore.release()
elif (message_type == "REGISTER"):
success = self.db.add_user(message_dict["NAME"],
message_dict["IP"],
message_dict["PORT"])
if (success == 0):
msg = {"TYPE": "REGISTER-SUCCESS", "RQ#": message_dict["RQ#"]}
self.semaphore.acquire()
try:
if self.active == True:
self.sock.sendto(utils.serialize(msg), address)
self.sock.sendto(utils.serialize(msg),
(self.host_backup, self.port_backup))
self.write_to_log('MESSAGE SENT\t\t [' + str(address) +
']:\t ' + str(msg))
self.write_to_log('MESSAGE SENT\t\t [' +
str((self.host_backup,
self.port_backup)) + ']:\t ' +
str(msg))
finally:
self.semaphore.release()
else:
if (success == 1):
msg = {
"TYPE": "REGISTER-DENIED",
"RQ#": message_dict["RQ#"],
"REASON": "USERNAME ALREADY EXISTS"
}
elif (success == 2):
msg = {
"TYPE": "REGISTER-DENIED",
"RQ#": message_dict["RQ#"],
"REASON": "NOT A VALID USERNAME"
}
elif (success == 3):
msg = {
"TYPE": "REGISTER-DENIED",
"RQ#": message_dict["RQ#"],
"REASON": "DATABASE CONNECTION ERROR"
}
self.semaphore.acquire()
try:
if self.active == True:
self.sock.sendto(utils.serialize(msg), address)
self.sock.sendto(utils.serialize(msg),
(self.host_backup, self.port_backup))
self.write_to_log('MESSAGE SENT\t\t [' + str(address) +
']:\t ' + str(msg))
self.write_to_log('MESSAGE SENT\t\t [' +
str((self.host_backup,
self.port_backup)) + ']:\t ' +
str(msg))
finally:
self.semaphore.release()
elif (message_type == "DE-REGISTER"):
success = self.db.remove_user(message_dict["NAME"])
if (success):
msg_client = {
"TYPE": "DE-REGISTER-SUCCESS",
"RQ#": message_dict["RQ#"]
}
msg_server = {
"TYPE": "DE-REGISTER-SUCCESS",
"RQ#": message_dict["RQ#"],
"NAME": message_dict["NAME"]
}
self.sock.sendto(utils.serialize(msg_client), address)
self.sock.sendto(utils.serialize(msg_server),
(self.host_backup, self.port_backup))
self.write_to_log('MESSAGE SENT\t\t [' + str(address) +
']:\t ' + str(msg_client))
self.write_to_log('MESSAGE SENT\t\t [' +
str((self.host_backup, self.port_backup)) +
']:\t ' + str(msg_server))
else:
msg_client = {
"TYPE": "DE-REGISTER-DENIED",
"RQ#": message_dict["RQ#"]
}
msg_server = {
"TYPE": "DE-REGISTER-DENIED",
"RQ#": message_dict["RQ#"],
"NAME": message_dict["NAME"]
}
self.sock.sendto(utils.serialize(msg_client), address)
self.sock.sendto(utils.serialize(msg_server),
(self.host_backup, self.port_backup))
self.write_to_log('MESSAGE SENT\t\t [' + str(address) +
']:\t ' + str(msg_client))
self.write_to_log('MESSAGE SENT\t\t [' +
str((self.host_backup, self.port_backup)) +
']:\t ' + str(msg_server))
elif (message_type == "UPDATE-SOCKET"):
success = self.db.update_socket(message_dict["NAME"],
message_dict["IP"],
message_dict["PORT"])
if (success):
msg = {
"TYPE": "UPDATE-SOCKET-SUCCESS",
"RQ#": message_dict["RQ#"],
"NAME": message_dict["NAME"],
"IP": message_dict["IP"],
"PORT": message_dict["PORT"]
}
self.sock.sendto(utils.serialize(msg), address)
self.sock.sendto(utils.serialize(msg),
(self.host_backup, self.port_backup))
self.write_to_log('MESSAGE SENT\t\t [' + str(address) +
']:\t ' + str(msg))
self.write_to_log('MESSAGE SENT\t\t [' +
str((self.host_backup, self.port_backup)) +
']:\t ' + str(msg))
else:
msg = {
"TYPE": "UPDATE-SOCKET-DENIED",
"RQ#": message_dict["RQ#"],
"NAME": message_dict["NAME"],
"IP": message_dict["IP"],
"PORT": message_dict["PORT"]
}
self.sock.sendto(utils.serialize(msg), address)
self.sock.sendto(utils.serialize(msg),
(self.host_backup, self.port_backup))
self.write_to_log('MESSAGE SENT\t\t [' + str(address) +
']:\t ' + str(msg))
self.write_to_log('MESSAGE SENT\t\t [' +
str((self.host_backup, self.port_backup)) +
']:\t ' + str(msg))
elif (message_type == "SUBJECTS"):
subjects_list = message_dict["SUBJECT-LIST"].split(",")
print(subjects_list)
success = self.db.update_subjects(message_dict["NAME"],
subjects_list)
if (success):
msg = {
"TYPE": "UPDATE-SUBJECTS-SUCCESS",
"RQ#": message_dict["RQ#"],
"NAME": message_dict["NAME"],
"SUBJECT-LIST": message_dict["SUBJECT-LIST"]
}
self.sock.sendto(utils.serialize(msg), address)
self.sock.sendto(utils.serialize(msg),
(self.host_backup, self.port_backup))
self.write_to_log('MESSAGE SENT\t\t [' + str(address) +
']:\t ' + str(msg))
self.write_to_log('MESSAGE SENT\t\t [' +
str((self.host_backup, self.port_backup)) +
']:\t ' + str(msg))
else:
msg = {
"TYPE": "UPDATE-SUBJECTS-DENIED",
"RQ#": message_dict["RQ#"],
"NAME": message_dict["NAME"],
"SUBJECT-LIST": message_dict["SUBJECT-LIST"]
}
self.sock.sendto(utils.serialize(msg), address)
self.sock.sendto(utils.serialize(msg),
(self.host_backup, self.port_backup))
self.write_to_log('MESSAGE SENT\t\t [' + str(address) +
']:\t ' + str(msg))
self.write_to_log('MESSAGE SENT\t\t [' +
str((self.host_backup, self.port_backup)) +
']:\t ' + str(msg))
elif (message_type == "PUBLISH"):
success = self.db.publish_message(message_dict["NAME"],
message_dict["SUBJECT"],
message_dict["TEXT"])
if (success):
msg = {
"TYPE": "PUBLISH-SUCCESS",
"RQ#": message_dict["RQ#"],
"NAME": message_dict["NAME"],
"SUBJECT": message_dict["SUBJECT"],
"TEXT": message_dict["TEXT"]
}
self.sock.sendto(utils.serialize(msg), address)
self.sock.sendto(utils.serialize(msg),
(self.host_backup, self.port_backup))
self.write_to_log('MESSAGE SENT\t\t [' + str(address) +
']:\t ' + str(msg))
self.write_to_log('MESSAGE SENT\t\t [' +
str((self.host_backup, self.port_backup)) +
']:\t ' + str(msg))
else:
msg = {
"TYPE": "PUBLISH-DENIED",
"RQ#": message_dict["RQ#"],
"NAME": message_dict["NAME"],
"SUBJECT": message_dict["SUBJECT"],
"TEXT": message_dict["TEXT"]
}
self.sock.sendto(utils.serialize(msg), address)
self.sock.sendto(utils.serialize(msg),
(self.host_backup, self.port_backup))
self.write_to_log('MESSAGE SENT\t\t [' + str(address) +
']:\t ' + str(msg))
self.write_to_log('MESSAGE SENT\t\t [' +
str((self.host_backup, self.port_backup)) +
']:\t ' + str(msg))
elif (message_type == "RETRIEVE-TEXTS"):
msg_list = self.db.retrieve_texts(message_dict["NAME"])
if (msg_list != None):
msg = {
"TYPE": "RETRIEVE-SUCCESS",
"RQ#": message_dict["RQ#"],
"NAME": message_dict["NAME"],
"POSTS": msg_list
}
self.sock.sendto(utils.serialize(msg), address)
self.sock.sendto(utils.serialize(msg),
(self.host_backup, self.port_backup))
self.write_to_log('MESSAGE SENT\t\t [' + str(address) +
']:\t ' + str(msg))
self.write_to_log('MESSAGE SENT\t\t [' +
str((self.host_backup, self.port_backup)) +
']:\t ' + str(msg))
else:
msg = {"TYPE": "RETRIEVE-DENIED", "RQ#": message_dict["RQ#"]}
self.sock.sendto(utils.serialize(msg), address)
self.sock.sendto(utils.serialize(msg),
(self.host_backup, self.port_backup))
self.write_to_log('MESSAGE SENT\t\t [' + str(address) +
']:\t ' + str(msg))
self.write_to_log('MESSAGE SENT\t\t [' +
str((self.host_backup, self.port_backup)) +
']:\t ' + str(msg))
elif (message_type == "CHANGE-SERVER"):
# If the server receives this message, it gains control
self.gain_control()
self.start_timer_thread()
elif (message_type == "UPDATE-SERVER"):
self.semaphore.acquire()
try:
if self.active == True:
# If a server receives this message, it means it the inactive server has
# changed its IP and PORT. Thus, we should update these values
self.host_backup = message_dict["IP"]
self.port_backup = message_dict["PORT"]
# We should now reply to the new server that we have taken note of its IP and PORT
# We also send it our IP and PORT number so it knows which one is the active server
msg_reply = {
"TYPE": "UPDATE-SERVER-SUCCESS",
"IP": self.host,
"PORT": self.port,
"SERVER-TAG": self.server_tag,
"CLIENT-LIST": self.client_list
}
msg_reply_serialized = utils.serialize(msg_reply)
self.sock.sendto(msg_reply_serialized, address)
self.write_to_log('MESSAGE SENT\t\t [' + str(address) +
']:\t ' + str(msg_reply))
# else if the inactive server receives this message, it should send a message
# to all the connected clients so they know where the new inactive server is
else:
new_host = message_dict["IP"]
new_port = message_dict["PORT"]
msg_client = {
"TYPE": "UPDATE-SERVER",
"IP": new_host,
"PORT": new_port,
"SERVER-TAG": self.server_tag
}
msg_client_serialized = utils.serialize(msg_client)
for i in self.client_list:
self.sock.sendto(msg_client_serialized, i)
self.write_to_log('MESSAGE SENT\t\t [' + str(i) +
']:\t ' + str(msg_client))
self.write_to_log("SERVER-CLOSED")
print("SERVER-CLOSED")
except:
print("ERROR SENDING UPDATE-SERVER MESSAGE")
msg_error = {"TYPE": "UPDATE-SERVER-DENIED"}
msg_error_serialized = utils.serialize(msg_error)
self.sock.sendto(msg_error_serialized, address)
self.write_to_log('MESSAGE SENT\t\t [' + str(address) +
']:\t ' + str(msg_error))
finally:
self.semaphore.release()
elif (message_type == "END-CONNECTION"):
try:
self.client_list.remove(address)
except:
pass
elif ((message_type == "REGISTER-SUCCESS")
or (message_type == "REGISTER-DENIED")):
pass
elif ((message_type == "DE-REGISTER-SUCCESS")
or (message_type == "DE-REGISTER-DENIED")):
try:
if (message_type == "DE-REGISTER-SUCCESS"):
success = self.db.remove_user(message_dict["NAME"])
except Exception as msg:
print(str(msg))
elif ((message_type == "UPDATE-SOCKET-SUCCESS")
or (message_type == "UPDATE-SOCKET-DENIED")):
try:
if (message_type == "UPDATE-SOCKET-SUCCESS"):
self.db.update_socket(message_dict["NAME"],
message_dict["IP"],
message_dict["PORT"])
except:
pass
elif ((message_type == "UPDATE-SUBJECTS-SUCCESS")
or (message_type == "UPDATE-SUBJECTS-DENIED")):
subjects_list = message_dict["SUBJECT-LIST"].split(",")
try:
if (message_type == "UPDATE-SUBJECTS-SUCCESS"):
self.db.update_subjects(message_dict["NAME"],
subjects_list)
except:
pass
elif ((message_type == "PUBLISH-SUCCESS")
or (message_type == "PUBLISH-DENIED")):
try:
if (message_type == "PUBLISH-SUCCESS"):
self.db.publish_message(message_dict["NAME"],
message_dict["SUBJECT"],
message_dict["TEXT"])
except:
pass
elif ((message_type == "RETRIEVE-SUCCESS")
or (message_type == "RETRIEVE-DENIED")):
try:
if (message_type == "RETRIEVE-SUCCESS"):
self.db.retrieve_texts(message_dict["NAME"])
except:
pass
elif ((message_type == "UPDATE-SERVER-SUCCESS")
or (message_type == "UPDATE-SERVER-DENIED")):
try:
if (message_type == "UPDATE-SERVER-SUCCESS"):
if message_dict["SERVER-TAG"] == "A":
self.server_tag = "B"
self.db = DBHandler(connection_string_2,
"register-share-system-2")
else:
self.server_tag = 'A'
self.db = DBHandler(connection_string_1,
"register-share-system-1")
self.host_backup = message_dict["IP"]
self.port_backup = message_dict["PORT"]
self.client_list = utils.convert(
message_dict["CLIENT-LIST"])
except:
pass
else:
pass
# This function is an ongoing loop that receives messages from clients and the other server
def run(self):
while (True):
d = self.sock.recvfrom(1024)
data = d[0]
addr = d[1]
if not data:
break
client_data = pickle.loads(data) # DESERIALIZED DATA
client_dict = ''
try:
client_dict = ast.literal_eval(str(client_data))
except:
self.write_to_log("ERROR CONVERTING MESSAGE TO DICTIONARY")
self.handle_message(client_dict, addr)
class DataGatherer(object):
def __init__(self, cfg):
self.cfg = cfg
self.downloader = Downloader(cfg)
self.logger = Logger(cfg)
self.parser = AnotherHTMLParser(self.logger)
self.pairs = set()
self.db_handler = DBHandler(cfg)
self._word_dict = None
def read_raw_pairs(self, delimiter=',', limit=0):
path = cfg['train_path']
try:
f = open(path)
except IOError:
self.logger.critical("Can't open file '{}'!".format(path))
sys.exit()
lines = f.read().split('\n')
pairs = set()
i = 0
for line in lines:
if not line:
continue
if limit and i > limit:
break
i += 1
elements = line.split(delimiter)
try:
if elements[2] == 'left':
pair = (elements[0], elements[1])
else:
pair = (elements[1], elements[0])
if pair in pairs:
self.logger.warning('pair {} is duplicate!'.format(pair))
i -= 1
pairs.add(pair)
except IndexError:
raise AssertionError('line {} is incorrect!'.format(line))
return pairs
def read_pairs(self, delimiter=',', limit=0):
path = cfg['train_fixed_path']
try:
f = open(path)
except IOError:
self.logger.critical("Can't open file '{}'!".format(path))
sys.exit()
lines = f.read().split('\n')
pairs = set()
i = 0
for line in lines:
if not line:
continue
if limit and i > limit:
break
i += 1
elements = line.split(delimiter)
try:
pair = tuple(elements)
if pair in pairs:
self.logger.warning('pair {} is duplicate!'.format(pair))
i -= 1
pairs.add(pair)
except IndexError:
raise AssertionError('line {} is incorrect!'.format(line))
return pairs
def exclude_untracked_videos(self, pairs):
ids = set(self.db_handler.get_all_video_ids())
pairs_set = set(pairs)
for pair in pairs:
for youtube_id in pair:
if youtube_id not in ids:
pairs_set.remove(pair)
break
return pairs_set
def rewrite_pairs(self, pairs):
pairs_fixed = self.exclude_untracked_videos(pairs)
f = open(self.cfg['train_fixed_path'], 'wb')
for pair in pairs_fixed:
f.write(','.join(pair) + '\n')
f.close()
def fill_video_catalog(self, pairs, force=False):
lefts_and_rights = zip(*pairs)
ids = set(lefts_and_rights[0] + lefts_and_rights[1])
if not force:
ids_cache = set(self.db_handler.get_all_video_ids())
ids.difference_update(ids_cache)
for i, youtube_id in enumerate(ids):
if i % 100 == 0:
self.logger.info('scanned {} lines.'.format(i))
self.add_video_by_id(youtube_id)
def update_video_catalog(self, limit=None):
ids_cache = set(self.db_handler.get_all_video_ids())
for i, youtube_id in enumerate(ids_cache):
if limit and i > limit:
break
self.update_video_by_id(youtube_id)
def add_video_by_id(self, youtube_id):
html = self.downloader.get_html(youtube_id)
if not self.parser._check_video_availability(html):
return
video_item = Video(youtube_id)
video_item.update(title=self.parser.get_video_title(html))
self.db_handler.add_entry(video_item)
def update_video_by_id(self, youtube_id):
html = self.downloader.get_html(youtube_id)
if not self.parser._check_video_availability(html):
return
video_item = self.db_handler.get_video_by_youtube_id(youtube_id)
try:
video_item.update(
title=self.parser.get_video_title(html),
views=self.parser.get_view_count(html),
likes=self.parser.get_likes_count(html),
dislikes=self.parser.get_dislikes_count(html),
)
except ParseError:
video_item.mark_invalid()
self.db_handler.commit()
def update_rank1s(self, pairs):
videos = self.db_handler.get_all_videos()
rank1_map = self.get_rank1_map(pairs)
for video in videos:
if video.youtube_id in rank1_map:
video.rank1 = rank1_map[video.youtube_id]
else:
self.logger.warning('video {} has no rank calculated!'.format(video.youtube_id))
self.db_handler.db_session.commit()
def update_rank2s(self, catalog, pairs):
videos = self.db_handler.get_all_videos()
rank2_map = self.get_rank2_map(catalog, pairs)
for video in videos:
if video.youtube_id in rank2_map:
video.rank2 = rank2_map[video.youtube_id]
else:
self.logger.warning('video {} has no rank calculated!'.format(video.youtube_id))
self.db_handler.db_session.commit()
def update_views(self, force=False):
if force:
videos = self.db_handler.get_all_videos()
else:
videos = self.db_handler.db_session.query(Video).filter(Video.views == None).all()
for video in videos:
try:
video.views = self.parser.get_view_count(self.downloader.get_html(video.youtube_id))
except ParseError:
pass
self.db_handler.commit()
def get_video_catalog(self):
return self.db_handler.get_all_video_data()
def get_rank1_map(self, pairs):
ids_above, ids_below = zip(*pairs)
rank_map = defaultdict(lambda: 0)
for youtube_id in ids_above:
rank_map[youtube_id] += 1
for youtube_id in ids_below:
rank_map[youtube_id] -= 1
return rank_map
def get_rank2_map(self, catalog, pairs):
chunks = partial_sort(catalog, pairs)
aggregated_ranks = calculate_aggregated_ranks(chunks)
assert len(aggregated_ranks) == len(chunks)
ranked_chunks = zip(aggregated_ranks, chunks)
r_map = {}
for rank, chunk in ranked_chunks:
for youtube_id in chunk:
r_map[youtube_id] = rank
return r_map
def get_char_stat(self):
characters = set()
videos = self.db_handler.get_all_videos()
for video in videos:
if video.title:
characters.update(video.title)
return sorted(list(characters))
def update_lang_stat(self):
videos = self.db_handler.get_all_videos()
for video in videos:
if video.title:
video.lang = get_lang(video.title)
self.db_handler.commit()
def get_all_words(self):
words = defaultdict(lambda: 0)
print 'delimiters: {}'.format(TITLE_DELIMITER)
videos = self.db_handler.get_all_videos()
for video in videos:
for word in extract_words(video.title):
words[prepare_word(word)] += 1
return words
def fill_word_db(self, words):
for w, count in words.iteritems():
word = Word(w, None, count)
self.db_handler.db_session.add(word)
self.db_handler.commit()
def fill_words_for_videos(self):
words = self.db_handler.db_session.query(Word).all()
word_dict = {}
for word in words:
word_dict[word.word] = word
videos = self.db_handler.get_all_videos()
for video in videos:
wordids = set()
for word in extract_words(video.title):
w = prepare_word(word)
if w in word_dict:
wordids.add(word_dict[w].id)
video.wordids = serialize_ids(wordids)
self.db_handler.commit()
def calculate_rank1_for_words(self):
words = self.db_handler.db_session.query(Word).filter(Word.count >= 10).all()
word_dict = {}
for word in words:
word_dict[word.id] = word
rank_dict = defaultdict(lambda: [])
videos = self.db_handler.get_all_videos()
for video in videos:
word_ids = deserialize_ids(video.wordids)
for word_id in word_ids:
if word_id not in word_dict:
continue
rank_dict[word_id].append(video.rank1)
for word_id in rank_dict:
if word_id not in word_dict:
continue
word_dict[word_id].rank1 = mean(rank_dict.setdefault(word_id, [0]))
# kostyl! set rank = 0 for word ''
null_word = self.db_handler.db_session.query(Word).filter(Word.word == '').one()
null_word.rank1 = 0
# --
self.db_handler.commit()
def get_word_dict_by_word(self):
if not self._word_dict:
words = self.db_handler.db_session.query(Word).all()
self._word_dict = {}
for word in words:
self._word_dict[word.word] = word
return self._word_dict
def calculate_title_rank(self, title, f):
word_dict = self.get_word_dict_by_word()
title_words = extract_words(title)
title_rank = sum(f(word_dict[x]) for x in title_words if x in word_dict)
return title_rank
def __init__(self):
super(MainWindow, self).__init__()
loadUi('../ui/MainWindow.ui', self)
self.myDb = DBHandler()
self.updateTableFlag = True
self.FilterConfigButton = self.findChild(QPushButton, 'filterButton')
self.FilterConfigButton.clicked.connect(self.openFilterConfig)
self.settingsConfig = self.findChild(QAction, 'actionSettings')
self.settingsConfig.triggered.connect(self.openSettings)
#Search button functionality ##need to add keyword search still
self.searchSearchButton = self.findChild(QPushButton,
'searchSearchButton')
self.searchSearchButton.clicked.connect(self.openFilterConfig)
# Enter press on qlineedit search tab triggers the search button
self.searchLineEdit = self.findChild(QLineEdit, 'lineEdit_2')
self.searchLineEdit.returnPressed.connect(
self.searchSearchButton.click)
# Enter press on qlineedit graph tab triggers search button
self.graphSearchEdit = self.findChild(QLineEdit, 'graphLineEdit')
#Exit menu option functionality
self.CloseMenuSelect = self.findChild(QAction, 'actionClose_Exit')
self.CloseMenuSelect.setShortcut('Ctrl+Q')
self.CloseMenuSelect.triggered.connect(qApp.quit)
#Export menu option functionality
self.exportConfig = self.findChild(QAction, 'actionExport')
self.exportConfig.setShortcut('Ctrl+E')
self.exportConfig.triggered.connect(self.openExportConfig)
#VectorDBConfig linked to menu option
self.versionControl = self.findChild(QAction, 'actionVersion_Control')
self.versionControl.setShortcut('Ctrl+S')
self.versionControl.triggered.connect(self.openVectDBConfig)
self.GraphTable = self.findChild(QTableWidget, 'tableWidget')
self.graphArea = self.findChild(QWidget, 'graphArea')
self.graphArea.setLayout(QVBoxLayout())
self.graphSearchButton = self.findChild(QPushButton,
'graphSearchButton_2')
# self.graphSearchButton.clicked.connect()
self.currentVectorMenu = self.findChild(QComboBox, 'VectorMenu')
self.currentVectorLabel = self.findChild(QLabel, 'CurrentVector')
self.currentVectorMenu.addItems(self.myDb.get_vector_names())
self.currentVectorLabel.setText(self.currentVectorMenu.currentText())
self.currentVectorMenu.activated.connect(self.vectorSelected)
self.GraphTable.cellClicked.connect(self.entryFieldSelected)
self.GraphTable.cellChanged.connect(self.updateEntryFromTable)
# Events
def node_selected(node):
if self.qgv.manipulation_mode == QGraphVizManipulationMode.Node_remove_Mode:
print("Node {} removed".format(node))
self.saveGraph()
# self.updateTableFromGraph()
else:
print("Node selected {}".format(node))
def edge_selected(edge):
if self.qgv.manipulation_mode == QGraphVizManipulationMode.Edge_remove_Mode:
print("Edge {} removed".format(edge))
else:
print("Edge selected {}".format(edge))
def node_invoked(node):
print("Node double clicked")
def edge_invoked(node):
print("Edge double clicked")
def node_removed(node):
print("Node removed")
def edge_removed(node):
print("Edge removed")
# Create QGraphViz widget
show_subgraphs = True
self.qgv = QGraphViz(show_subgraphs=show_subgraphs,
node_selected_callback=node_selected,
edge_selected_callback=edge_selected,
node_invoked_callback=node_invoked,
edge_invoked_callback=edge_invoked,
node_removed_callback=node_removed,
edge_removed_callback=edge_removed,
hilight_Nodes=True,
hilight_Edges=True)
self.qgv.setStyleSheet("background-color:white;")
# Create A new Graph using Dot layout engine
self.qgv.new(Dot(Graph("Main_Graph"), show_subgraphs=show_subgraphs))
# Adding nodes with an image as its shape
icon_path = os.path.dirname(os.path.abspath(__file__)) + r"\dbicon.png"
# Build the graph (the layout engine organizes where the nodes and connections are)
self.qgv.build()
# Save it to a file to be loaded by Graphviz if needed
self.qgv.save("test.gv")
# Add the QGraphViz object to the layout
self.graphArea.layout().addWidget(self.qgv)
# Add a horizontal layout (a pannel to select what to do)
self.hpanel = QHBoxLayout()
self.graphArea.layout().addLayout(self.hpanel)
# Add buttons to the panel
def save():
fname = QFileDialog.getSaveFileName(self.qgv, "Save", "", "*.json")
if (fname[0] != ""):
self.qgv.saveAsJson(fname[0])
def new():
self.qgv.engine.graph = Graph("MainGraph")
self.qgv.build()
self.qgv.repaint()
def load():
fname = QFileDialog.getOpenFileName(self.qgv, "Open", "", "*.json")
if (fname[0] != ""):
self.qgv.loadAJson(fname[0])
def add_node():
dlg = QDialog()
dlg.ok = False
dlg.node_name = ""
dlg.node_label = ""
dlg.node_color = ""
dlg.node_type = "None"
# Layouts
main_layout = QVBoxLayout()
l = QFormLayout()
buttons_layout = QHBoxLayout()
main_layout.addLayout(l)
main_layout.addLayout(buttons_layout)
dlg.setLayout(main_layout)
leNodeName = QLineEdit()
leNodeLabel = QLineEdit()
cbxNodeType = QComboBox()
leImagePath = QLineEdit()
leNodeColor = QLineEdit()
pbOK = QPushButton()
pbCancel = QPushButton()
cbxNodeType.addItems(["None", "circle", "box"])
pbOK.setText("&OK")
pbCancel.setText("&Cancel")
l.setWidget(0, QFormLayout.LabelRole, QLabel("Node Name"))
l.setWidget(0, QFormLayout.FieldRole, leNodeName)
l.setWidget(1, QFormLayout.LabelRole, QLabel("Node Label"))
l.setWidget(1, QFormLayout.FieldRole, leNodeLabel)
l.setWidget(2, QFormLayout.LabelRole, QLabel("Node Type"))
l.setWidget(2, QFormLayout.FieldRole, cbxNodeType)
l.setWidget(3, QFormLayout.LabelRole, QLabel("Node Image"))
l.setWidget(3, QFormLayout.FieldRole, leImagePath)
l.setWidget(4, QFormLayout.LabelRole, QLabel("Node Color"))
l.setWidget(4, QFormLayout.FieldRole, leNodeColor)
def ok():
dlg.OK = True
dlg.node_name = leNodeName.text()
dlg.node_label = leNodeLabel.text()
if (leImagePath.text()):
dlg.node_type = leImagePath.text()
else:
dlg.node_type = cbxNodeType.currentText()
dlg.node_color = leNodeColor.text()
dlg.close()
def cancel():
dlg.OK = False
dlg.close()
pbOK.clicked.connect(ok)
pbCancel.clicked.connect(cancel)
buttons_layout.addWidget(pbOK)
buttons_layout.addWidget(pbCancel)
dlg.exec_()
# node_name, okPressed = QInputDialog.getText(wi, "Node name","Node name:", QLineEdit.Normal, "")
if dlg.OK and dlg.node_name != '':
self.qgv.addNode(self.qgv.engine.graph,
dlg.node_name,
label=dlg.node_label,
shape=dlg.node_type,
color=dlg.node_color)
self.qgv.build()
def remove_node():
self.qgv.manipulation_mode = QGraphVizManipulationMode.Node_remove_Mode
for btn in self.buttons_list:
btn.setChecked(False)
self.btnRemoveNode.setChecked(True)
def remove_edge():
self.qgv.manipulation_mode = QGraphVizManipulationMode.Edge_remove_Mode
for btn in self.buttons_list:
btn.setChecked(False)
self.btnRemoveEdge.setChecked(True)
def add_edge():
self.qgv.manipulation_mode = QGraphVizManipulationMode.Edges_Connect_Mode
for btn in self.buttons_list:
btn.setChecked(False)
self.btnAddEdge.setChecked(True)
# Add buttons
self.btnNew = QPushButton("New")
self.btnNew.clicked.connect(new)
self.btnOpen = QPushButton("Open")
self.btnOpen.clicked.connect(load)
self.btnSave = QPushButton("Save")
self.btnSave.clicked.connect(save)
self.hpanel.addWidget(self.btnNew)
self.hpanel.addWidget(self.btnOpen)
self.hpanel.addWidget(self.btnSave)
self.buttons_list = []
self.btnAddNode = QPushButton("Add Node")
self.btnAddNode.clicked.connect(add_node)
self.hpanel.addWidget(self.btnAddNode)
self.buttons_list.append(self.btnAddNode)
self.btnRemoveNode = QPushButton("Remove Node")
self.btnRemoveNode.setCheckable(True)
self.btnRemoveNode.clicked.connect(remove_node)
self.hpanel.addWidget(self.btnRemoveNode)
self.buttons_list.append(self.btnRemoveNode)
self.btnAddEdge = QPushButton("Add Edge")
self.btnAddEdge.setCheckable(True)
self.btnAddEdge.clicked.connect(add_edge)
self.hpanel.addWidget(self.btnAddEdge)
self.buttons_list.append(self.btnAddEdge)
self.btnRemoveEdge = QPushButton("Remove Edge")
self.btnRemoveEdge.setCheckable(True)
self.btnRemoveEdge.clicked.connect(remove_edge)
self.hpanel.addWidget(self.btnRemoveEdge)
self.buttons_list.append(self.btnRemoveEdge)
#icon_path = os.path.dirname(os.path.abspath(__file__)) + r"\Resouces\IconDir,100,100"
# n9 = qgv.addNode(qgv.engine.graph, "Node9", label="N9", shape=icon_path)
#drop down menus vector collumn search table
self.SearchTable = self.findChild(QTableWidget, 'tableWidget_2')
self.showMaximized()
# DBHandler.create_vector_entry('../Resources/LocalGraphs/VECTOR_3.json')
self.updateViews()
# Revolet
# Force override doesnt work. must delete model and associated files.
cf = CFHandler('109jikL9APOQLYLWVbh0BPzsKh4Fj1E5y',
'1d4uAconWl18s629ZUMaZlXkopD2Vfzh3',
'1-JiAeAJ_0XWXxkRteRP1my84x57cpVmg',
force_download=False)
# Sum
# Take paper2data (authors, citing authors) from CF (Revolet)
if use_semantic:
from inference.semantic import SemanticHandler
semantic = SemanticHandler('1-n-GephieNUHJ-pXeHIoX-rjMWRtssKj',
'11rB0mV9o5Uk78d8bIjzDR4obnHtn2bwc',
'1-H_P6t33LNMfaAuD7qtj7b9hD-RviLvQ')
db_handler = DBHandler(cf, mock=not use_db)
def get_best_authors(title, author_ids, top=10):
if len(author_ids) > 0:
cf_recommended_authors = cf.get_recommended_authors(author_ids, top)
else:
cf_recommended_authors = []
# TODO: We dont care about the weight but we should
cf_recommended_authors = [
author_score_tuple[0] for author_score_tuple in cf_recommended_authors
]
if title is not None and use_semantic:
semantic_recommended_authors = semantic.get_recommended_authors(
class AttendenceCheckHandler:
def __init__(self):
self.db_handler = DBHandler()
self.current_user = None
self.semester = None
self.curr_offering = None
self.create_instances()
def create_instances(self):
self.db_handler.create_instances()
def login_to_account(self, username, password):
if self.db_handler.login(username, password):
self.current_user = self.db_handler.get_curr_user()
return True
return False
def check_secter_answer(self, username, answer):
return self.db_handler.check_secter_answer(username, answer)
def change_password(self, username, new_password):
self.db_handler.change_password(username, new_password)
def get_data_from_db(self):
self.semester = self.db_handler.get_data_from_db()
def set_curr_exam(self, exam_id):
status, self.curr_offering = self.semester.get_curr_exam(exam_id)
return status
def get_student_ids(self):
return self.curr_offering.get_student_ids()
def get_exam_list(self):
return self.semester.get_exam_list()
def is_student_available(self, student_id):
return self.curr_offering.is_student_available(student_id)
def confirm_attendance(self, student_id):
return self.curr_offering.confirm_attendance(student_id)
def set_present(self, student_id):
self.curr_offering.set_present(student_id)
def are_all_studence_checked(self):
return self.curr_offering.are_all_studence_checked()
def get_offering_prof_info(self):
return self.curr_offering.get_offering_prof_info()
def get_student_info(self, student_id):
return self.curr_offering.get_student_info(student_id)
def prof_confirm(self):
self.curr_offering.prof_confirm()
def remove_curr_offering(self):
self.curr_offering = None
def join_all_threads(self):
self.db_handler.join_all_threads()
def submit_results(self):
all_data_to_post = self.semester.get_data_to_post()
self.db_handler.post_all_present_data(all_data_to_post)
def findChecksecinDB(package, rowlist):
query = f"SELECT {CHECKSEC} from {VULNERABILITY_TABLE_NAME} where {PACKAGE} = '{package}' and {CHECKSEC} != 'NULL'"
db = DBHandler()
result = db._execute_select(query)
rowlist.extend(result)
import os
from db_handler import DBHandler
from flask import Flask, request
from data_processor import DataProcessor
from flaskthreads import AppContextThread
weather_app = Flask(__name__)
env_config = os.getenv("APP_SETTINGS", "config.DevelopmentConfig")
weather_app.config.from_object(env_config)
data_processor = DataProcessor(weather_app)
db_handler = DBHandler.get_instance(weather_app)
weather_app.app_context().push()
@weather_app.route('/', methods=['GET'])
def welcome_to_service():
return "Welcome to my weather service!"
@weather_app.route('/pre_process', methods=['GET'])
def pre_process():
thread = AppContextThread(target=data_processor.process_files)
thread.start()
return "Pre-processing csv files..."
@weather_app.route('/weather/data', methods=['GET'])
def get_data_by_location():
data = request.get_json()
try:
if 'lon' in data and 'lat' in data:
def __init__(self):
self.db_handler = DBHandler()
self.current_user = None
self.semester = None
self.curr_offering = None
self.create_instances()
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import telegram
from datetime import datetime
import logging
from telegram.ext import Updater, CommandHandler, CallbackQueryHandler
from telegram import InlineKeyboardButton, InlineKeyboardMarkup
from getToken import getToken
from db_handler import DBHandler
from getChatId import getId
CHAT_ID = getId()
startTime = datetime.now()
db = DBHandler()
def error(bot, update, error):
#change the logging level between INFO - normal mode or DEBUG - verbose mode
logging.basicConfig(
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
level=logging.DEBUG)
logger = logging.getLogger(__name__)
logger.warning('Update "%s" caused error "%s"' % (update, error))
#checkUser checks is the user is already registered for d2-bot,
#verify that only users from a certain chat have access
def checkUser(bot, update):
load_config,
COUNTRIES,
SCORE_THRESHOLD,
RUMOR_THRESHOLD,
USEFUL_THRESHOLD,
SENTIMENT_THRESHOLD,
ITOPICS,
ECOUNTRY_ICOUNTRIES_MAP,
)
logger = logging.getLogger(__file__)
cfg = load_config()
meta_data_handler = MetaDataHandler()
db_handler = DBHandler(**cfg["db_handler"])
log_handler = LogHandler(**cfg["log_handler"])
twitter_handler = TwitterHandler(**cfg["twitter_handler"])
def update_database(do_tweet: bool = False):
logger.debug("Add automatically categorized pages.")
data_path = cfg["data"]["article_list"]
cache_file = f'{cfg["log_handler"]["log_dir"]}/offset.txt'
if os.path.exists(cache_file):
with open(cache_file) as f:
offset = int(f.read().strip())
else:
offset = 0
logger.debug(f"Skip the first {offset} lines.")
maybe_tweeted_ds = []
roads_download = args["roads_download"]
simplify = args["simplify"]
dsn = args["dsn"]
assign_address = args["assign_address"]
check_address = args["check_address"]
check_building = args["check_building"]
check_road_rail = args["check_road_rail"]
vacuum = args["vacuum"]
report = args["report"]
index = args["index_data"]
bbox = args["bbox"]
add_fields = args["add_fields"]
dedupe = args["dedupe_address"]
convert_poly = args["convert_poly"]
db = DBHandler(dsn)
osm = OSMHandler(bbox)
if setup:
print 'Setting up the database.'
db.setup_db()
if building_download:
print 'Querying OverpassAPI for buildings.'
buildings = osm.query_buildings()
print 'Uploading OSM buildings to Postgres...'
db.insert_osm(buildings, 'osm_buildings')
if address_download:
print 'Querying OverpassAPI for addresses.'
addresses = osm.query_address()
