def transfer(puzzle, cleanup=True, skip_status_update=False):
"""
Transfer the puzzle data from Redis to the database. If the cleanup flag is
set the Redis data for the puzzle will be removed afterward.
"""
current_app.logger.info("transferring puzzle: {0}".format(puzzle))
cur = db.cursor()
query = """select * from Puzzle where (id = :puzzle)"""
(result, col_names) = rowify(
cur.execute(query, {
"puzzle": puzzle
}).fetchall(), cur.description)
if not result:
# Most likely because of a database switch and forgot to run this script
# between those actions.
# TODO: Raise an error here and let the caller decide how to handle it.
current_app.logger.warning(
"Puzzle {} not in database. Skipping.".format(puzzle))
return
puzzle_data = result[0]
puzzle_previous_status = puzzle_data["status"]
if not skip_status_update:
r = requests.patch(
"http://{HOSTAPI}:{PORTAPI}/internal/puzzle/{puzzle_id}/details/".
format(
HOSTAPI=current_app.config["HOSTAPI"],
PORTAPI=current_app.config["PORTAPI"],
puzzle_id=puzzle_data["puzzle_id"],
),
json={
"status": MAINTENANCE,
},
)
if r.status_code != 200:
# TODO: Raise an error here and let the caller decide how to handle it.
current_app.logger.warning(
"Puzzle details api error when setting status to maintenance {}"
.format(puzzle_data["puzzle_id"]))
return
(all_pieces, col_names) = rowify(
cur.execute(read_query_file("select_all_piece_props_for_puzzle.sql"), {
"puzzle": puzzle
}).fetchall(),
cur.description,
)
# Save the redis data to the db if it has changed
changed_pieces = []
pcstacked = set(map(int, redis_connection.smembers(f"pcstacked:{puzzle}")))
pcfixed = set(map(int, redis_connection.smembers(f"pcfixed:{puzzle}")))
for piece in all_pieces:
has_changes = False
pieceFromRedis = redis_connection.hgetall("pc:{puzzle}:{id}".format(
puzzle=puzzle, id=piece["id"]))
# The redis data may be empty so skip updating the db
if len(pieceFromRedis) == 0:
continue
# Compare redis data with db for any changes
for (prop, colname) in [
("x", "x"),
("y", "y"),
("r", "r"),
("g", "parent"),
("", "status"),
]:
if colname == "status":
if piece["id"] in pcstacked:
redis_piece_prop = 2
if piece["id"] in pcfixed:
redis_piece_prop = 1
else:
redis_piece_prop = pieceFromRedis.get(prop)
redis_piece_prop = (int(redis_piece_prop) if isinstance(
redis_piece_prop, str) else redis_piece_prop)
if redis_piece_prop != piece[colname]:
current_app.logger.debug("{} has {} changes. {} != {}".format(
piece["id"], colname, redis_piece_prop, piece[colname]))
piece[colname] = redis_piece_prop
has_changes = True
if has_changes:
changed_pieces.append(piece)
if len(changed_pieces) != 0:
r = requests.patch(
"http://{HOSTAPI}:{PORTAPI}/internal/puzzle/{puzzle_id}/pieces/".
format(
HOSTAPI=current_app.config["HOSTAPI"],
PORTAPI=current_app.config["PORTAPI"],
puzzle_id=puzzle_data["puzzle_id"],
),
json={"piece_properties": changed_pieces},
)
if r.status_code != 200:
raise Exception(
"Puzzle pieces api error. Failed to patch pieces. {}".format(
r))
if cleanup:
deletePieceDataFromRedis(redis_connection, puzzle, all_pieces)
cur.close()
if not skip_status_update:
r = requests.patch(
"http://{HOSTAPI}:{PORTAPI}/internal/puzzle/{puzzle_id}/details/".
format(
HOSTAPI=current_app.config["HOSTAPI"],
PORTAPI=current_app.config["PORTAPI"],
puzzle_id=puzzle_data["puzzle_id"],
),
json={
"status": puzzle_previous_status,
},
)
if r.status_code != 200:
# TODO: Raise an error here and let the caller decide how to handle it.
current_app.logger.warning("Puzzle details api error")
return
def do_task(self):
super().do_task()
made_change = False
cur = db.cursor()
for (low, high) in SKILL_LEVEL_RANGES:
in_queue_puzzle_count = cur.execute(
read_query_file("get_in_queue_puzzle_count.sql"),
{
"low": low,
"high": high
},
).fetchone()[0]
if in_queue_puzzle_count <= self.minimum_count:
(result, col_names) = rowify(
cur.execute(
read_query_file("select_random_puzzle_to_rebuild.sql"),
{
"status": COMPLETED,
"low": low,
"high": high
},
).fetchall(),
cur.description,
)
if result:
for completed_puzzle in result:
puzzle = completed_puzzle["id"]
current_app.logger.debug(
"found puzzle {id}".format(**completed_puzzle))
# Update puzzle status to be REBUILD and change the piece count
pieces = random.randint(
max(
int(current_app.config["MINIMUM_PIECE_COUNT"]),
max(completed_puzzle["pieces"] - 400, low),
),
min(high - 1, completed_puzzle["pieces"] + 400),
)
r = requests.patch(
"http://{HOSTAPI}:{PORTAPI}/internal/puzzle/{puzzle_id}/details/"
.format(
HOSTAPI=current_app.config["HOSTAPI"],
PORTAPI=current_app.config["PORTAPI"],
puzzle_id=completed_puzzle["puzzle_id"],
),
json={
"status": REBUILD,
"pieces": pieces,
"queue": QUEUE_END_OF_LINE,
},
)
if r.status_code != 200:
current_app.logger.warning(
"Puzzle details api error. Could not set puzzle status to rebuild. Skipping {puzzle_id}"
.format(**completed_puzzle))
continue
completed_puzzle["status"] = REBUILD
completed_puzzle["pieces"] = pieces
# Delete any piece data from redis since it is no longer needed.
query_select_all_pieces_for_puzzle = (
"""select * from Piece where (puzzle = :puzzle)""")
(all_pieces, col_names) = rowify(
cur.execute(query_select_all_pieces_for_puzzle, {
"puzzle": puzzle
}).fetchall(),
cur.description,
)
deletePieceDataFromRedis(redis_connection, puzzle,
all_pieces)
job = self.queue.enqueue_call(
func="api.jobs.pieceRenderer.render",
args=([completed_puzzle]),
result_ttl=0,
timeout="24h",
)
archive_and_clear(puzzle)
made_change = True
cur.close()
if made_change:
self.log_task()
def do_task(self):
super().do_task()
made_change = False
cur = db.cursor()
in_queue_puzzles_in_piece_groups = current_app.config[
"MINIMUM_IN_QUEUE_PUZZLES_IN_PIECE_GROUPS"].copy()
in_queue_puzzles_in_piece_groups.reverse()
for (low, high, minimum_count) in map(
lambda x: (x[0], x[1], in_queue_puzzles_in_piece_groups.pop()),
current_app.config["SKILL_LEVEL_RANGES"],
):
if minimum_count == 0:
continue
in_queue_puzzle_count = cur.execute(
read_query_file("get_in_queue_puzzle_count.sql"),
{
"low": low,
"high": high
},
).fetchone()[0]
if in_queue_puzzle_count <= minimum_count:
(result, col_names) = rowify(
cur.execute(
read_query_file("select_random_puzzle_to_rebuild.sql"),
{
"status": COMPLETED,
"low": max(0, low - 500),
"high": high + 500,
},
).fetchall(),
cur.description,
)
if not result:
# try again with wider range of puzzle piece counts
(result, col_names) = rowify(
cur.execute(
read_query_file(
"select_random_puzzle_to_rebuild.sql"),
{
"status": COMPLETED,
"low": max(0, low - 2000),
"high": high + 2000,
},
).fetchall(),
cur.description,
)
if result:
for completed_puzzle in result:
puzzle = completed_puzzle["id"]
current_app.logger.debug(
"found puzzle {id}".format(**completed_puzzle))
# Update puzzle status to be REBUILD and change the piece count
low_range = max(
int(current_app.config["MINIMUM_PIECE_COUNT"]),
min(
max(low, (high - 400)),
max(low, (completed_puzzle["pieces"] - 400)),
),
)
high_range = min(
high,
max((low + 400),
(completed_puzzle["pieces"] + 400)))
pieces = random.randint(low_range, high_range)
sleep(API_REQUESTS_LIMIT_RATE)
r = requests.patch(
"http://{HOSTAPI}:{PORTAPI}/internal/puzzle/{puzzle_id}/details/"
.format(
HOSTAPI=current_app.config["HOSTAPI"],
PORTAPI=current_app.config["PORTAPI"],
puzzle_id=completed_puzzle["puzzle_id"],
),
json={
"status": REBUILD,
"pieces": pieces,
"queue": QUEUE_END_OF_LINE,
},
)
if r.status_code != 200:
current_app.logger.warning(
"Puzzle details api error. Could not set puzzle status to rebuild. Skipping {puzzle_id}"
.format(**completed_puzzle))
continue
completed_puzzle["status"] = REBUILD
completed_puzzle["pieces"] = pieces
# Delete any piece data from redis since it is no longer needed.
query_select_all_pieces_for_puzzle = (
"""select * from Piece where (puzzle = :puzzle)""")
(all_pieces, col_names) = rowify(
cur.execute(query_select_all_pieces_for_puzzle, {
"puzzle": puzzle
}).fetchall(),
cur.description,
)
deletePieceDataFromRedis(redis_connection, puzzle,
all_pieces)
job = self.queue.enqueue(
"api.jobs.pieceRenderer.render",
[completed_puzzle],
result_ttl=0,
job_timeout="24h",
)
archive_and_clear(puzzle)
made_change = True
cur.close()
if made_change:
self.log_task()
def post(self):
args = {}
if request.form:
args.update(request.form.to_dict(flat=True))
puzzle_id = args.get("puzzle_id")
if not puzzle_id:
abort(400)
# Check pieces arg
try:
pieces = int(
args.get("pieces", current_app.config["MINIMUM_PIECE_COUNT"]))
except ValueError as err:
abort(400)
if pieces < current_app.config["MINIMUM_PIECE_COUNT"]:
abort(400)
user = int(
current_app.secure_cookie.get(u"user")
or user_id_from_ip(request.headers.get("X-Real-IP")))
cur = db.cursor()
result = cur.execute(
fetch_query_string(
"select_puzzle_for_puzzle_id_and_status_and_not_recent.sql"),
{
"puzzle_id": puzzle_id,
"status": COMPLETED
},
).fetchall()
if not result:
# Puzzle does not exist or is not completed status.
# Reload the page as the status may have been changed.
cur.close()
return redirect(
"/chill/site/front/{puzzle_id}/".format(puzzle_id=puzzle_id))
(result, col_names) = rowify(result, cur.description)
puzzleData = result[0]
puzzle = puzzleData["id"]
userCanRebuildPuzzle = cur.execute(
fetch_query_string("select-user-rebuild-puzzle-prereq.sql"),
{
"user": user,
"puzzle": puzzle,
"pieces": pieces
},
).fetchall()
if not userCanRebuildPuzzle:
cur.close()
abort(400)
original_puzzle_id = puzzleData["original_puzzle_id"]
# Get the adjusted piece count depending on the size of the original and
# the minimum piece size.
original_puzzle_dir = os.path.join(
current_app.config["PUZZLE_RESOURCES"], original_puzzle_id)
# TODO: get path of original.jpg via the PuzzleFile query
# TODO: use requests.get to get original.jpg and run in another thread
imagefile = os.path.join(original_puzzle_dir, "original.jpg")
im = Image.open(imagefile)
(width, height) = im.size
im.close()
max_pieces_that_will_fit = int((old_div(width, MIN_PIECE_SIZE)) *
(old_div(height, MIN_PIECE_SIZE)))
# The user points for rebuilding the puzzle is decreased by the piece
# count for the puzzle. Use at least minimum piece count (20) points for
# smaller puzzles. Players that own a puzzle instance do not decrease
# any points (dots) if the puzzle is complete.
point_cost = max(
current_app.config["MINIMUM_PIECE_COUNT"],
min(
max_pieces_that_will_fit,
pieces,
current_app.config["MAX_POINT_COST_FOR_REBUILDING"],
),
)
if not (puzzleData["owner"] == user and puzzleData["puzzle_id"]
== puzzleData["original_puzzle_id"]):
cur.execute(
fetch_query_string("decrease-user-points.sql"),
{
"user": user,
"points": point_cost
},
)
# Update puzzle status to be REBUILD and change the piece count
cur.execute(
fetch_query_string("update_status_puzzle_for_puzzle_id.sql"),
{
"puzzle_id": puzzle_id,
"status": REBUILD,
"pieces": pieces,
"queue": QUEUE_REBUILD,
},
)
puzzleData["status"] = REBUILD
puzzleData["pieces"] = pieces
db.commit()
# Delete any piece data from redis since it is no longer needed.
(all_pieces, col_names) = rowify(
cur.execute(
fetch_query_string("select_all_piece_ids_for_puzzle.sql"),
{
"puzzle": puzzle
},
).fetchall(),
cur.description,
)
cur.close()
deletePieceDataFromRedis(redis_connection, puzzle, all_pieces)
job = current_app.createqueue.enqueue_call(
func="api.jobs.pieceRenderer.render",
args=([puzzleData]),
result_ttl=0,
timeout="24h",
)
job = current_app.cleanupqueue.enqueue_call(
func="api.jobs.timeline_archive.archive_and_clear",
kwargs=({
"puzzle": puzzle
}),
result_ttl=0,
timeout="24h",
)
purge_route_from_nginx_cache(
"/chill/site/front/{puzzle_id}/".format(puzzle_id=puzzle_id),
current_app.config.get("PURGEURLLIST"),
)
return redirect(
"/chill/site/front/{puzzle_id}/".format(puzzle_id=puzzle_id))
def post(self):
args = {}
if request.form:
args.update(request.form.to_dict(flat=True))
puzzle_id = args.get("puzzle_id")
if not puzzle_id:
abort(400)
# Check pieces arg
try:
pieces = int(
args.get("pieces", current_app.config["MINIMUM_PIECE_COUNT"]))
except ValueError as err:
abort(400)
if pieces < current_app.config["MINIMUM_PIECE_COUNT"]:
abort(400)
user = int(
current_app.secure_cookie.get(u"user")
or user_id_from_ip(request.headers.get("X-Real-IP")))
cur = db.cursor()
result = cur.execute(
fetch_query_string(
"select_puzzle_for_puzzle_id_and_status_and_not_recent.sql"),
{
"puzzle_id": puzzle_id,
"status": COMPLETED
},
).fetchall()
if not result:
# Puzzle does not exist or is not completed status.
# Reload the page as the status may have been changed.
cur.close()
return redirect(
"/chill/site/front/{puzzle_id}/".format(puzzle_id=puzzle_id))
(result, col_names) = rowify(result, cur.description)
puzzleData = result[0]
puzzle = puzzleData["id"]
userCanRebuildPuzzle = cur.execute(
fetch_query_string("select-user-rebuild-puzzle-prereq.sql"),
{
"user": user,
"puzzle": puzzle,
"pieces": pieces
},
).fetchall()
if not userCanRebuildPuzzle:
cur.close()
abort(400)
if (puzzleData["permission"] == PRIVATE and
puzzleData["original_puzzle_id"] == puzzleData["puzzle_id"]):
current_app.logger.warning(
"Original puzzles that are private can not be rebuilt")
cur.close()
abort(400)
# The user points for rebuilding the puzzle is decreased by the piece
# count for the puzzle. Use at least minimum piece count (20) points for
# smaller puzzles. Players that own a puzzle instance do not decrease
# any points (dots) if the puzzle is complete.
point_cost = max(
current_app.config["MINIMUM_PIECE_COUNT"],
min(
pieces,
current_app.config["MAX_POINT_COST_FOR_REBUILDING"],
),
)
if not (puzzleData["owner"] == user and puzzleData["puzzle_id"]
== puzzleData["original_puzzle_id"]):
cur.execute(
fetch_query_string("decrease-user-points.sql"),
{
"user": user,
"points": point_cost
},
)
# Update puzzle status to be REBUILD and change the piece count
cur.execute(
fetch_query_string("update_status_puzzle_for_puzzle_id.sql"),
{
"puzzle_id": puzzle_id,
"status": REBUILD,
"pieces": pieces,
"queue": QUEUE_REBUILD,
},
)
puzzleData["status"] = REBUILD
puzzleData["pieces"] = pieces
db.commit()
# Delete any piece data from redis since it is no longer needed.
(all_pieces, col_names) = rowify(
cur.execute(
fetch_query_string("select_all_piece_ids_for_puzzle.sql"),
{
"puzzle": puzzle
},
).fetchall(),
cur.description,
)
cur.close()
deletePieceDataFromRedis(redis_connection, puzzle, all_pieces)
delete_puzzle_resources(
puzzle_id,
is_local_resource=not puzzleData["preview_full"].startswith("http")
and not puzzleData["preview_full"].startswith("//"),
exclude_regex=r"(original|preview_full).([^.]+\.)?jpg")
job = current_app.createqueue.enqueue(
"api.jobs.pieceRenderer.render",
[puzzleData],
result_ttl=0,
job_timeout="24h",
)
job = current_app.cleanupqueue.enqueue(
"api.jobs.timeline_archive.archive_and_clear",
puzzle,
result_ttl=0,
job_timeout="24h",
)
purge_route_from_nginx_cache(
"/chill/site/front/{puzzle_id}/".format(puzzle_id=puzzle_id),
current_app.config.get("PURGEURLLIST"),
)
return redirect(
"/chill/site/front/{puzzle_id}/".format(puzzle_id=puzzle_id))