def stored_procedure( script_name, *args ): try: sha = REDIS.get( 'lua_function:%s' % script_name ) return REDIS.evalsha( sha, len(args), *args ) except Exception as err: LOG.debug( '%s failed to run stored procedure, with error %s and args %s' % (script_name, str(err), str(args)) ) return str(err)
def simple_query(self, query, params, condition=''):
"""
Simple database query
Input parameters:
query - query number (one from QUERY_... constants)
params - parameter tuple
when only one parameter is required specify as (parameter,)
Return value:
success:
- First row of query result
failure:
- raise error
"""
if self.__cnx is None:
raise DbException.CONNECTION_FAILED
try:
complete_query = query.__add__(condition)
self.__cursor.execute(complete_query, params)
result = self.__cursor.fetchone()
self.__cnx.commit()
if result is None:
raise DbException.QUERY_NO_RESULT
except mysql.connector.Error:
LOG.error("Query: '%s', parameters (%s)" % (complete_query, str(params)))
raise DbException.MYSQL_CONNECTOR
return result
def test_read_file_mixed_whitespace(mock_file_directory):
LOG.info("test_read_file_from_mock_dir()")
file_contents = "a ab abc abcd abcde"
expected = file_contents.split()
test_file = os.path.join(mock_file_directory, "abcde_whitespace.txt")
actual = read_file(test_file)
assert actual == expected
def login_as_admin():
LOG.info("login_as_admin")
response = Auth().login(APP_URL, ADMIN_USER, ADMIN_PASSWORD)
assert response.ok
access_token = response.json()["access_token"]
yield access_token
def create_comment(self, app_url, access_token, message):
LOG.info("create_comment")
request_headers = build_request_headers(access_token)
payload = {"text": message}
LOG.debug(f"Request payload: {payload}")
response = SESSION.post(f"{app_url}{self.comment_url}/",
headers=request_headers, params=payload)
return response
def test_transpose(transposer_fixture, filename, expected_longest,
expected_reverse):
LOG.info(f"test_transpose_single_file({ filename })")
(longest, reversed_word
) = transposer_fixture.get_longest_and_transposed_word_from_file(
os.path.join(ROOT_DIR, "sample_txt_files", filename))
assert longest == expected_longest
assert reversed_word == expected_reverse
def test_get_txt_files_in_directory_bad_path(mock_file_directory):
LOG.info("test_get_txt_files_in_directory_bad_path()")
# make sure the bad file path doesn't exist
bad_file_path = os.path.join(mock_file_directory, "nonexistant_directory")
if os.path.exists(bad_file_path):
os.remove(bad_file_path)
with pytest.raises(FileNotFoundError) as fe:
get_txt_files_in_directory(bad_file_path)
def test_reverse_string_other_data_types_raise_value_error(input):
LOG.info(
f"test_reverse_string_other_data_types_raise_value_error({ input })")
with pytest.raises(TypeError) as error_info:
reverse_string(input)
exception_actual = error_info.value
LOG.debug(exception_actual)
exception_expected = f"reverse_string() expects a string but got a { type(input) }"
assert str(exception_actual) == exception_expected
def main():
"""
The main program starts here
"""
configure_logger()
LOG.info("Starting Python RPC Server Daemon...")
loop = asyncio.get_event_loop()
loop.add_signal_handler(signal.SIGTERM, functools.partial(shutdown, loop))
RPCS().Listening.run_forever(session_handler, nb_session_handler)
def test_reverse_string_long():
LOG.info("test_reverse_string_long()")
input = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et " \
"dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip " \
"ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu " \
"fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia " \
"deserunt mollit anim id est laborum. "
expected = input[::-1]
actual = reverse_string(input)
assert expected == actual
def load_label(self, index):
labels = []
try:
with open(self.label_files[index], 'r') as f:
for l in f.readlines():
target = l.split(' ')
labels.append([self.class_to_idx(target[0]), int(target[1]), int(target[2]), int(target[3]), int(target[4])])
except:
LOG.error('annotation file {} non-existent'.format(self.label_files[index]))
return labels
def test_read_file(tmpdir):
LOG.info("test_read_file()")
file_contents = "a\nab\nabc\nabcd\nabcde"
expected = file_contents.split()
# create temp file for testing
test_file = tmpdir.join("test_file.txt")
test_file.write(file_contents)
# read temp file
actual = read_file(test_file)
LOG.debug(actual)
assert actual == expected
def getGame(room):
g = gameDB.games.find({'room': room})
try:
game = g[0]
except IndexError as e:
LOG.info("Game: " + room + " Does not yet exist, creating new game")
game = createGame(room)
del game['_id'] # Remove the object id as it is not JSON serializable
return game
def login(self, app_url, username, password):
LOG.info("login")
request_headers = {
"Content-Type": "application/x-www-form-urlencoded",
"accept": "application/json"
}
payload = {"username": username, "password": password}
LOG.debug(f"Request payload: {payload}")
response = SESSION.post(f"{app_url}{self.auth_url}/login",
headers=request_headers,
data=payload)
return response
def log(self, ip, msg, resultcode, cid, stage):
""" Logs CLIENT connection to the database using inform parameters """
try:
# Always [try to] log to the log table
self.__cursor.execute(Queries.INSERT_CLIENTLOG,
(cid, resultcode, ip, msg))
self.__cursor.fetchone()
self.__cnx.commit()
except mysql.connector.Error as err:
LOG.error("Database: Error during LOG, %s" % err.msg)
def update_northbound_handler(session):
try:
event = get_element('event', session, data_type=str)
if (event == 'CONNECTION_REQUEST'
and session['client_id'] in RPCS.Northbound_Queue):
# Northbridge session is waiting for a response,
# return 'error' via stream socket to communicate CPE disconnection
result = 'error'
session['nb_response_stream'].send(result.encode())
except KeyError:
LOG.debug(
"Event is CONNECTION_REQUEST but Northbound session was not found!"
)
def build_request_headers(access_token,
accept_type="application/json",
**kwargs):
LOG.info("build_request_headers")
headers = {
"Authorization": f"Bearer {access_token}",
"accept": accept_type
}
if "content_type" in kwargs:
headers["Content-Type"] = kwargs["content_type"]
LOG.debug(f"Request headers: {headers}")
return headers
def test_cud_comment(login_as_admin):
LOG.info("test_cud_comment")
response = Comments().create_comment(APP_URL, login_as_admin, "first post")
assert response.ok
response_data = response.json()
comment_id = response_data["id"]
LOG.debug(response_data)
assert response_data["comment_text"] == "first post"
response = Comments().update_comment(APP_URL,
login_as_admin,
comment_id,
message="updated to second post",
likes=3)
assert response.ok
response_data = response.json()
LOG.debug(response_data)
assert response_data["comment_text"] == "updated to second post"
assert response_data["likes"] == 3
response = Comments().delete_comment(APP_URL, login_as_admin, comment_id)
assert response.ok
response_data = response.json()
LOG.debug(response_data)
assert response_data["detail"] == f"Deleted comment {comment_id}"
def update_comment(self, app_url, access_token, comment_id, **kwargs):
LOG.info("update_comment")
request_headers = build_request_headers(access_token,
content_type="application/json")
payload = {}
if "message" in kwargs:
payload["comment_text"] = kwargs["message"]
if "likes" in kwargs:
payload["likes"] = kwargs["likes"]
LOG.debug(f"Request payload: {payload}")
response = SESSION.put(f"{app_url}{self.comment_url}/{comment_id}",
headers=request_headers, json=payload)
return response
def handle_response(self, session, message):
""" Process the return from the set RPC
an error response is already handled
"""
result = misc.get_element('result', message)
if result == 0:
LOG.debug("Response to 'set': Success")
pass
elif result == 1:
LOG.debug("Response to 'set': Error")
pass
else:
# Unknown result
raise ClientMethodException('handle response', 'RpcSet')
def userJoiningRoom(message):
LOG.info("User is attemping to join a game. Message: " + str(message))
username = message['username']
room = message['room']
color = message['color']
# Try to add the user to the multiplayer game
try:
addUser(room, username, color, request.sid)
join_room(room)
LOG.info("User: "******" has joined room: " + str(room))
game = getGame(room)
emit('joined', game, broadcast=True, room=room)
except ColorTakenException as e: # Username already taken
emit('join-error', str(e))
except UsernameTakenException as e: # Color already taken
emit('join-error', str(e))
def configure_logger():
# Configure the logger to log both file and console using appropiate format
root_logger = LOG.getLogger()
root_logger.setLevel(LOG.NOTSET)
file_handler = LOG.FileHandler(os.path.abspath(LOGGING_FILENAME))
file_handler.setLevel(LOGGING_LEVEL_LOGFILE)
file_formatter = LOG.Formatter('%(asctime)s> [%(levelname)s] %(message)s',
datefmt='%a %b %d %H:%M:%S %Y')
file_handler.setFormatter(file_formatter)
root_logger.addHandler(file_handler)
console_handler = LOG.StreamHandler(sys.stdout)
console_handler.setLevel(LOGGING_LEVEL_CONSOLE)
console_formatter = LOG.Formatter('[%(levelname)s] (%(module)s:%(lineno)d)'
' %(message)s')
console_handler.setFormatter(console_formatter)
root_logger.addHandler(console_handler)
def create_user(self,
app_url,
access_token,
username,
password,
role="user"):
LOG.info("create_user")
request_headers = build_request_headers(
access_token, content_type="application/json")
payload = {
"username": username,
"password_hash": password,
"roles": role
}
LOG.debug(f"Request payload: {payload}")
response = SESSION.post(f"{app_url}{self.user_url}",
headers=request_headers,
json=payload)
return response
def disconnect(self):
"""
Disconnect from database
Input parameter: cnx = connection
"""
if self.__cnx is None:
LOG.error("Error: disconnecting when not connected")
return
try:
# Try to clean up database cursor object. If that fails the garbage
# collector will automatically deal with it after connection has
# been closed.
self.__cursor.close()
except mysql.connector.Error as err:
LOG.error("Error closing cursor: %s" % err.msg)
finally:
# Closing the connection with the database server may lead to
# exceptions but they are handled in such a way that they never lead
# to an error disturbing normal operation. Closing the connection
# can thus safely be done here.
self.__cnx.close()
def connect(self):
"""
Connect to the database and create a cursor
No input parameters
"""
config = {
'user': MYSQL_USER,
'password': MYSQL_PASSWORD,
'database': MYSQL_DATABASE,
'unix_socket': '/run/mysqld/mysqld.sock',
'raise_on_warnings': False,
'charset': 'latin1',
'collation': 'latin1_general_ci',
'buffered': False,
}
try:
self.__cnx = mysql.connector.connect(**config)
except mysql.connector.Error as err:
if err.errno == errorcode.ER_ACCESS_DENIED_ERROR:
LOG.error("Cannot connect to database: No access")
elif err.errno == errorcode.ER_BAD_DB_ERROR:
LOG.error("Cannot connect to database: Unknown database")
else:
LOG.error("Cannot connect to database: Unknown error")
raise DbException.CONNECTION_FAILED
self.__cursor = self.__cnx.cursor(prepared=True)
def test_user_permissions(login_as_admin):
LOG.info("test_user_permissions")
# Create new user and assign "user" role
new_username = "******"
new_password = "******"
new_user_roles = "user"
response = Users().create_user(APP_URL, login_as_admin, new_username,
new_password)
assert response.ok
response_data = response.json()
new_user_id = response_data["id"]
assert response_data["username"] == new_username
assert response_data["roles"] == "user"
# Login as the newly created user
response = Auth().login(APP_URL, new_username, new_password)
assert response.ok
response_data = response.json()
access_token = response_data["access_token"]
# Check the new user can get his own info
response = Users().get_current_user(APP_URL, access_token)
assert response.ok
assert response.json()["username"] == new_username
assert response.json()["roles"] == new_user_roles
# Check that the newly created user CAN NOT create other users because
# it doesn't have admin privileges
response = Users().create_user(APP_URL, access_token, "tony", "montana")
assert not response.ok
# Check that the newly created user CAN NOT delete other users because
# it doesn't have admin privileges
response = Users().delete_user(APP_URL, access_token, new_user_id)
assert not response.ok
# Finally, delete the newly created user but this time use the admin account
response = Users().delete_user(APP_URL, login_as_admin, new_user_id)
assert response.ok
def addUser(room, username, color, sid):
LOG.info("Adding user: "******" to room: " + room)
game = getGame(room)
if username in game['game']['users'].keys():
raise UsernameTakenException("Username is already taken.")
for user in game['game']['users']:
if game['game']['users'][user]['color'] == color:
raise ColorTakenException("Another user already has choosen " +
color)
user = {'sid': sid, 'room': room, 'username': username}
LOG.info("Added new user " + str(user))
userDB.users.insert(user)
game['game']['users'][username] = {
'color': color,
'totalGuesses': 0,
'correctGuesses': 0,
'averageDistance': 0,
'totalDistance': 0
}
updateGame(room, game)
def mock_file_directory(tmpdir_factory):
mock_file_dir = tmpdir_factory.mktemp("files")
# create text file per challenge example
abcde_txt = mock_file_dir.join("abcde.txt")
abcde_txt.write("a\nab\nabc\nabcd\nabcde")
# challenge example in random order
abcde_random_txt = mock_file_dir.join("abcde_random.txt")
abcde_random_txt.write("abc\nabcde\nab\na\nabcd ")
# separated by other whitespace characters
abcde_whitespace_txt = mock_file_dir.join("abcde_whitespace.txt")
abcde_whitespace_txt.write("a ab \tabc\f abcd \rabcde")
# restricted permission file (unsuccessful)
# restricted_txt = mock_file_dir.join("restricted.txt")
# restricted_txt.write("This file should not be accessible.\n")
# restricting the permissions on this file did not successfully trigger the PermissionError on read
# os.chmod(restricted_txt, 400)
LOG.debug(f"Created mock file directory at { mock_file_dir }")
return mock_file_dir
def load_mosaic(self, index):
# mosaic增强
labels4 = []
s = self.img_size
# mosaic增强四部分图片的相交点
yc, xc = [int(random.uniform(-x, 2 * s + x)) for x in self.mosaic_border]
# 随机出其他三张图片
indices = [index] + [random.randint(0, len(self.img_files) - 1) for _ in range(3)]
for i, index in enumerate(indices):
# Load image
img = cv2.imread(self.img_files[index])
assert img is not None, LOG.error('Image not found' + self.img_files[index])
h, w = img.shape[:2]
# place img in img4
if i == 0: # top left
img4 = np.full((s * 2, s * 2, img.shape[2]), 114, dtype=np.uint8) # base image with 4 tiles
x1a, y1a, x2a, y2a = max(xc - w, 0), max(yc - h, 0), xc, yc # xmin, ymin, xmax, ymax (large image)
x1b, y1b, x2b, y2b = w - (x2a - x1a), h - (y2a - y1a), w, h # xmin, ymin, xmax, ymax (small image)
elif i == 1: # top right
x1a, y1a, x2a, y2a = xc, max(yc - h, 0), min(xc + w, s * 2), yc
x1b, y1b, x2b, y2b = 0, h - (y2a - y1a), min(w, x2a - x1a), h
elif i == 2: # bottom left
x1a, y1a, x2a, y2a = max(xc - w, 0), yc, xc, min(s * 2, yc + h)
x1b, y1b, x2b, y2b = w - (x2a - x1a), 0, max(xc, w), min(y2a - y1a, h)
elif i == 3: # bottom right
x1a, y1a, x2a, y2a = xc, yc, min(xc + w, s * 2), min(s * 2, yc + h)
x1b, y1b, x2b, y2b = 0, 0, min(w, x2a - x1a), min(y2a - y1a, h)
img4[y1a:y2a, x1a:x2a] = img[y1b:y2b, x1b:x2b] # img4[ymin:ymax, xmin:xmax]
padw = x1a - x1b
padh = y1a - y1b
# Labels
x = load_label(index)
labels = x.copy()
if x.size > 0: # xyxy format
labels[:, 1] = x[:, 1] + padw
labels[:, 2] = x[:, 2] + padh
labels[:, 3] = x[:, 3] + padw
labels[:, 4] = x[:, 4] + padh
labels4.append(labels)
# Concat/clip labels
if len(labels4):
labels4 = np.concatenate(labels4, 0)
np.clip(labels4[:, 1:], 0, 2 * s, out=labels4[:, 1:])
return img4, labels4
def addGuess(room, username, guess):
LOG.info("User: "******" Room: " + room + " Guess: " + str(guess))
game = getGame(room)
game['game']['question']['guesses'][username] = guess
prevScore = game['game']['users'][username]
if (guess['Correct'] == 'True'):
game['game']['users'][username][
'totalGuesses'] = prevScore['totalGuesses'] + 1
game['game']['users'][username][
'correctGuesses'] = prevScore['correctGuesses'] + 1
game['game']['users'][username]['averageDistance'] = prevScore[
'totalDistance'] / game['game']['users'][username]['totalGuesses']
else:
game['game']['users'][username][
'totalGuesses'] = prevScore['totalGuesses'] + 1
game['game']['users'][username][
'totalDistance'] = prevScore['totalDistance'] + guess['Distance']
game['game']['users'][username]['averageDistance'] = game['game'][
'users'][username]['totalDistance'] / game['game']['users'][
username]['totalGuesses']
updateGame(room, game)
def checkAnswer(message):
LOG.info("User has submitted their guess. Message: " + str(message))
country = message['country']
guess = message['guess']
username = message['username']
room = message['room']
# Check the user's guess by checking with the singleplayer backend
check_url = f'http://geoquiz:5000/game/checkAnswer?country={country}&guess={guess}'
response = requests.get(check_url).json()
response['GuessID'] = guess
response['country'] = country
addGuess(room, username, response)
game = getGame(room)
emit('answer-response', game, broadcast=True, room=room)
# If everyone has answered the question, wait 5 seconds and get a new question
if(len(game['game']['users']) == len(game['game']['question']['guesses'])):
LOG.info("Everyone has answered the question sending new question in 5 seconds")
sleep(5)
addNewQuestion(room)
game = getGame(room)
emit('new-question', game, broadcast=True, room=room)
def run():
LOG.info("starting katipo sweeper")
while True:
outcome = stored_procedure("release_holds")
LOG.info(outcome)
sleep(1)
import os.path
import glob
from config import LOG, REDIS
# load all the lua scripts into redis
# storing keys and sha's
LOG.info( "loading stored procedures" )
for script in glob.glob( os.path.join( 'lua', '*.lua' ) ):
try:
script_code = open( script, "r" ).read()
script_name, script_type = os.path.splitext( script )
script_name = os.path.basename( script_name )
REDIS.set( 'lua_function:%s' % script_name, REDIS.register_script( script_code ).sha )
LOG.debug( 'loaded %s' % script_name )
except Exception as err:
LOG.debug( '%s failed to load, with error %s' % (script_name, str(err)) )
def pairs_to_dictionary( array ):
result = {}
if len( array ) % 2 != 0:
response = array.pop()
if type( response ) == list:
while len( response ) > 0:
key = response.pop(0)
value = response.pop(0)
result[ key ] = value
if len( array ) > 0:
for key, value in pairs_to_dictionary([array]).iteritems():
if value > 0:
