def convert_wildcard_query_value_query_key(query_unit): """ Handle the case where we got a wildcard operator in query value :param query_unit: A parsed query unit of quadra-tuple :return: key of query dict to be used in mongo query """ db_type, attr, comp_op, query_value = query_unit assert "*" in query_value assert "*" not in attr, "Sorry, only ONE wildcard operator at a time." assert isinstance(query_value, str) assert query_value[0] == '"' and query_value[-1] == '"' assert comp_op == "", "Sorry, combined usage of comparison and" \ " wildcard operators is not allowed." query_value = query_value[1:-1] value_regex = "^" + query_value.replace("*", ".*") + "$" book_db, author_db = connect_to_mongo() target_db = book_db if db_type == "book" else author_db match_ids = [] # #### "$where" not supported in MongoDB Atlas free tier # in {"$where": f"this.amount.toString().match({value_regex})"} # so we will have to manually extract id of matching objects. for dic in target_db.find(): instance_val = str(dic[attr]) if re.findall(value_regex, instance_val): match_ids.append(dic["_id"]) return {"_id": {"$in": match_ids}}
def test_valid_one_dict_update(self): """ Test valid json file of a single dict can be used to update value of existing object in database. First modify one author's url and then recover it. """ path = JSON_PATH + "legal_one_author.json" _, author_db = connect_to_mongo() with open(path, "r+") as file: author_dic = json.load(file) author_id = author_dic["_id"] true_author_url = author_dic["author_url"] query_key = {"_id": author_id} update_val = {"$set": {"author_url": "duckduckgo.com"}} self.assertTrue(list(author_db.find(query_key)) != []) # modify author url to duckduckgo author_db.update_one(query_key, update_val) author_url_modified = author_db.find_one(query_key)["author_url"] self.assertEqual(author_url_modified, "duckduckgo.com") # recover author url insert_into_db(path, db_type="author") author_url_recovered = author_db.find_one(query_key)["author_url"] self.assertEqual(author_url_recovered, true_author_url)
def test_valid_many_dict_update(self): """ Test valid json file of a list of dicts can be used to update value of existing object in database. First modify books' url and then recover them. """ path = JSON_PATH + "legal_many_books.json" book_db, _ = connect_to_mongo() with open(path, "r+") as file: book_dics = json.load(file) true_book_urls = [] # first modify all target book_urls in db for book_dic in book_dics: book_id = book_dic["_id"] true_book_urls.append(book_dic["book_url"]) query_key = {"_id": book_id} update_val = {"$set": {"book_url": "duckduckgo.com"}} book_db.update_one(query_key, update_val) self.assertTrue(list(book_db.find(query_key)) != []) # recover book url insert_into_db(path, db_type="book") for i, book_dic in enumerate(book_dics): book_id = book_dic["_id"] true_url = true_book_urls[i] query_key = {"_id": book_id} self.assertTrue(book_db.find_one(query_key)["book_url"], true_url)
def api_author(): """ Users can send GET/POST/PUT/DELETE requests to https://host/api/author?{_id, update_attr*} This function handles the backend behavior to response to these requests. """ _, author_db = connect_to_mongo() if request.method == "GET": try: query_id = request.args["_id"] result = list(author_db.find({"_id": query_id})) except: abort(400, "ID not provided.") if not result: # no matching abort(400, "No matching results in author database.") return jsonify(result) elif request.method == "POST": check_json_in_body(request) # check json is properly passed dict_list = request.json # This must be valid json as sanity test is done. if len(dict_list) > 1: abort( 400, "Please send POST request" " to /api/authors to upload many authors.") updater.write_given_dict_list_to_db(dict_list, author_db) return """Status Code [200] : Upload succeeded.""" elif request.method == "PUT": check_json_in_body(request) try: update_key = {"_id": request.json["_id"]} except: abort(400, "_id not provided.") matches = list(author_db.find(update_key)) if not matches: abort(400, "Target instance ID not found in DB.") update_val = {k: v for k, v in request.json.items() if k != "_id"} for key in update_val.keys(): # check to-be-updated attribute exists if key not in updater.AUTHOR_ATTRS: abort(400, "Bad attempt to update non-existing attributes.") if update_val == {}: abort(400, "Empty update value.") author_db.update_one(update_key, {"$set": update_val}) return """Status code [200] : Update succeeded.""" elif request.method == "DELETE": try: query_id = request.args["_id"] except: abort(400, "ID not provided.") if not list(author_db.find({"_id": query_id})): abort(400, "Target ID not in author_DB.") author_db.delete_one({"_id": query_id}) return """Status code [200] : Delete succeeded.""" else: abort(404, "Related resource not found")
def api_authors(): """ Handles clients requests to upload multiple authors at a time. """ _, author_db = connect_to_mongo() check_json_in_body(request) # check json is properly passed dict_list = request.json # This must be valid json as sanity test is done. if len(dict_list) == 1: abort( 400, "Please send POST request" " to /api/author to upload a single author.") updater.write_given_dict_list_to_db(dict_list, author_db) return """Status Code [200] : Upload succeeded."""
def test_client_search_with_id(self): """ Test server can response to existing id correctly. :return: """ book_db, author_db = connect_to_mongo() response1 = requests.get(HOST + "api/book", params={"_id": "58128"}) expected1 = list(book_db.find({"_id": "58128"})) self.assertEqual(response1.json(), expected1) response2 = requests.get(HOST + "api/author", params={"_id": "45372"}) expected2 = list(author_db.find({"_id": "45372"})) self.assertEqual(response2.json(), expected2)
def build_graph(): """ The enter interface for main-subcommand draw. :return: """ book_db, author_db = connect_to_mongo() books, authors = list(book_db.find({})), list(author_db.find({})) index2book = {i: books[i]["book_url"] for i in range(len(books))} book2index = {book: i for i, book in index2book.items()} index2author = {i: books[i]["author_url"] for i in range(len(authors))} author2index = {author: i for i, author in index2author.items()} index_dict_wrapper = tuple( [index2book, book2index, index2author, author2index]) adjacency_matrix = build_adjacency_matrix(books, authors, index_dict_wrapper) draw_graph(adjacency_matrix, books, authors)
def test_valid_single_dict_create(self): """ Test valid json file that contains a single dictionary could be inserted, by deleting one existing author and reinsert it. """ path = JSON_PATH + "legal_one_author.json" _, author_db = connect_to_mongo() with open(path, "r+") as file: author_dic = json.load(file) author_id = author_dic["_id"] self.assertTrue(list(author_db.find({"_id": author_id})) != []) author_db.delete_one({"_id": author_id}) self.assertTrue(list(author_db.find({"_id": author_id})) == []) insert_into_db(path, db_type="author") self.assertTrue(list(author_db.find({"_id": author_id})) != [])
def test_server_upload_and_delete(self): """ Test that server can handle upload and delete requests correctly. :return: """ book_db, _ = connect_to_mongo() book_id = "3735293" self.assertTrue(list(book_db.find({"_id": book_id}))) # existing book requests.delete(HOST + "api/book", params={"_id": book_id}) # delete request self.assertFalse(list(book_db.find({"_id": book_id }))) # not existing after del post_body = load_json_file("legal_one_book.json") requests.post(HOST + "api/book", json=post_body) self.assertTrue(list(book_db.find({"_id": book_id }))) # existing after uploading
def test_valid_many_dict_create(self): """ Test valid json file that contains a list of dictionaries could be inserted, by deleting many existing books and reinserting. """ path = JSON_PATH + "legal_many_books.json" book_db, _ = connect_to_mongo() with open(path, "r+") as file: book_dics = json.load(file) for _, book_dic in enumerate(book_dics): book_id = book_dic["_id"] self.assertTrue(list(book_db.find({"_id": book_id})) != []) book_db.delete_one({"_id": book_id}) self.assertTrue(list(book_db.find({"_id": book_id})) == []) insert_into_db(path, "book") for _, book_dic in enumerate(book_dics): book_id = book_dic["_id"] self.assertTrue(list(book_db.find({"_id": book_id})) != [])
def execute_parsed_query(query_units, logic_op): """ Given a set of parsed query units and logic connection, search through the MongoDB and return matches as a list. :param query_units: One or two unit(s) of parsed query. :param logic_op: logic connection for query units :return: matching results as a list """ assert len(query_units) != 0 book_db, author_db = connect_to_mongo() db_type, _, _, _ = query_units[0] db = book_db if db_type == "book" else author_db if len(query_units) == 1: mongo_query_key = convert_to_mongo_query_key(query_units[0]) return list(db.find(mongo_query_key)) assert logic_op != "NA" mongo_query_key1 = convert_to_mongo_query_key(query_units[0]) mongo_query_key2 = convert_to_mongo_query_key(query_units[1]) result_mongo_query = {"$and": [mongo_query_key1, mongo_query_key2]} \ if logic_op == "AND" else {"$or": [mongo_query_key1, mongo_query_key2]} return list(db.find(result_mongo_query))
def insert_into_db(src_json, db_type): """ Safely insert the entities from json file into remote mongoDB. Json files are required to: (1) The json file exists; (2) Be in good shape, parse-able (either one dic or list of dic); (3) Every single stored object has no missing keys as stated above; (4) Not existed in the database; to be inserted into the database. :param src_json: the NAME json file, stored in the ../JSON directory. Do not add path! :param db_type: either book or author """ book_db, author_db = connect_to_mongo() attrs_to_check = BOOK_ATTRS if db_type == "book" else AUTHOR_ATTRS db_to_update = book_db if db_type == "book" else author_db dictionary_list = load_json_file(src_json) # make sure there are no missing attributes check_missing_attributes(dictionary_list, attrs_to_check) # Write into target db write_given_dict_list_to_db(dictionary_list, db_to_update)
def test_client_elastic_search(self): """ Test server can handle valid elastic search requests correctly. :return: """ book_db, _ = connect_to_mongo() query1 = "book.rating_count : > 500 AND book.rating_count : < 1000" response1 = requests.get(HOST + "api/search", params={"q": query1}) expected1 = list( book_db.find({ "$and": [{ "rating_count": { "$gt": 500 } }, { "rating_count": { "$lt": 1000 } }] })) self.assertEqual(response1.json(), expected1) query2 = "book.review_count : < 30 OR book.rating_value : < 3.5" response2 = requests.get(HOST + "api/search", params={"q": query2}) expected2 = list( book_db.find({ "$or": [{ "review_count": { "$lt": 30 } }, { "rating_value": { "$lt": 3.5 } }] })) self.assertEqual(response2.json(), expected2)
def scrape_start(is_new, start_url, max_book=200, max_author=50, progress_dir=None): """ Scraping either from new url or continue last progress. :param is_new: whether there is a new starting url :param start_url: if is_new, then start url should be provided :param max_book: max number of books to scrape :param max_author: max number of author to scrape :param progress_dir: the directory of previously saved progress """ bfs_queue, visited_books, visited_authors =\ construct_bfs_info(is_new, start_url, progress_dir) book_db, author_db = connect_to_mongo() continuous_failure = 0 while len(bfs_queue) != 0: book_recorded = book_db.count() author_recorded = author_db.count() if book_recorded >= max_book and author_recorded >= max_author: print(f"currently there are {book_recorded} books," f" and {author_recorded} authors recorded." f" Both max criterions are reached.\n" f" Set larger max_author or max_book" f" to continue scraping") break # scraping done book_url = bfs_queue.pop(0) if book_url in visited_books: continue # book already visited try: # scrape the information of the current book print(f"\n\n\n\n Working on {book_recorded + 1} / {max_book} -" f" Scraping {book_url}\n" + SEP) book_dict = book_scraper.scrape_book(book_url) assert book_dict is not None book_id = book_dict.get("book_id") assert book_id is not None # use book_id as storage key assert book_dict.get("book_title") is not None author_url = book_dict.get("author_url") assert author_url is not None # make sure author is found book_dict["_id"] = book_id if not list(book_db.find({"_id": book_id})): book_db.insert_one(book_dict) # make sure no duplicate visited_books.add(book_url) bfs_queue.extend(book_dict["similar_book_urls"]) # scrape the information of author of the book if author_url in visited_authors: print("Author already recorded.") print(f"Currently recorded {book_recorded + 1} books," f"and {author_recorded} authors.") continuous_failure = 0 continue sleep(3) # don't make the IP got blocked author_dict = author_scraper.scrape_author(author_url) assert author_dict is not None, "author_dict is None" author_id = author_dict.get("author_id") assert author_dict.get("author_name") is not None,\ "author_name is None" # indicates scraping failed author_dict["_id"] = author_id if not list(author_db.find({"_id": author_id })): # make sure no duplicate author_db.insert_one(author_dict) visited_authors.add(author_url) # update bfs queue and visited records only if scraping succeeded save_progress(bfs_queue, visited_books, visited_authors, progress_dir) continuous_failure = 0 # reset failure count sleep(3) print(f"Currently recorded {book_recorded + 1} books, " f"and {author_recorded + 1} authors.") except: continuous_failure += 1 if continuous_failure >= 5: break # out IP are likely to be blocked print("Scraping failed ...")