def test_add_missing_columns():
create_test_tables()
db = MySQLDatabase(schema='qmdb_test', env='test')
db.columns_netflix_genres = {
'genreid': 'int(10) unsigned',
'genre_name': 'varchar(128)',
'test_column': 'int(10) unsigned',
'movies_updated': 'varchar(32)'
}
db.connect()
db.add_missing_columns('netflix_genres')
db.c.execute("""
SELECT
column_name,
column_type
FROM information_schema.columns
WHERE table_name='netflix_genres'
AND table_schema='qmdb_test'
""")
actual_cols = db.c.fetchall()
actual_cols_dict = {
d['column_name']: d['column_type']
for d in actual_cols
}
assert actual_cols_dict == db.columns_netflix_genres
assert list(actual_cols_dict.keys()) == list(db.columns_netflix_genres)
def test_set_netflix_genres():
create_test_tables()
db = MySQLDatabase(schema='qmdb_test', env='test')
db.netflix_genres = {
1: {
'genre_names': ['All Action', 'Belgian Movies'],
'movies_updated': arrow.get('2018-02-05 23:01:58+01:00')
},
3: {
'genre_names': ['Japanese Anime'],
'movies_updated': None
}
}
db.set_netflix_genres()
db.connect()
db.load_netflix_genres()
db.close()
assert db.netflix_genres == {
1: {
'genre_names': ['All Action', 'Belgian Movies'],
'movies_updated': arrow.get('2018-02-05 23:01:58+01:00')
},
3: {
'genre_names': ['Japanese Anime'],
'movies_updated': None
}
}
def test_get_movies_for_genres():
create_test_tables(variant='updates')
db = MySQLDatabase(schema='qmdb_test', env='tst')
db.netflix_genres = {
1: {
'genre_names': ['Action', 'Sci-Fi'],
'movies_updated': arrow.get('2018-03-28')
},
2: {
'genre_names': ['Sci-Fi', 'Drama'],
'movies_updated': arrow.get('2018-03-20')
},
3: {
'genre_names': ['Drama'],
'movies_updated': None
},
4: {
'genre_names': ['Romance'],
'movies_updated': arrow.get('2018-03-17')
}
}
netflix_scraper = NetflixScraper(db)
netflix_scraper.get_movies_for_genres()
assert (arrow.now() - db.netflix_genres[1]['movies_updated']).days == 4
assert arrow.now() == db.netflix_genres[2]['movies_updated']
assert arrow.now() == db.netflix_genres[3]['movies_updated']
assert arrow.now() == db.netflix_genres[4]['movies_updated']
assert [
call[0][0]
for call in netflix_scraper.get_movies_for_genre.call_args_list
] == [3, 4, 2]
assert db.movies[1234].netflix_rating == 4.8
remove_test_tables(db)
def test_get_authurl():
create_test_tables(variant='updates')
db = MySQLDatabase(schema='qmdb_test', env='tst')
netflix_scraper = NetflixScraper(db)
netflix_scraper.get_authurl()
assert isinstance(netflix_scraper.authURL, str)
assert len(netflix_scraper.authURL) > 20
def test_database_init_from_scratch_existing():
create_test_tables()
db = MySQLDatabase(schema='qmdb_test', env='test')
assert list(db.movies.keys()) == [1234, 49141]
db = MySQLDatabase(schema='qmdb_test', from_scratch=True, env='test')
assert list(db.movies.keys()) == []
remove_test_tables(db)
def test_get_movies_for_genres_norequests():
create_test_tables(variant='updates')
db = MySQLDatabase(schema='qmdb_test', env='tst')
db.netflix_genres = {
1: {
'genre_names': ['Action', 'Sci-Fi'],
'movies_updated': arrow.get('2018-03-28')
},
2: {
'genre_names': ['Sci-Fi', 'Drama'],
'movies_updated': arrow.get('2018-03-20')
},
3: {
'genre_names': ['Drama'],
'movies_updated': None
},
4: {
'genre_names': ['Romance'],
'movies_updated': arrow.get('2018-03-17')
}
}
netflix_scraper = NetflixScraper(db)
netflix_scraper.get_movies_for_genres()
assert db.netflix_genres[1]['movies_updated'] == arrow.get('2018-03-28')
assert db.netflix_genres[2]['movies_updated'] == arrow.get('2018-03-20')
assert db.netflix_genres[3]['movies_updated'] is None
assert db.netflix_genres[4]['movies_updated'] == arrow.get('2018-03-17')
assert [
call[0][0]
for call in netflix_scraper.get_movies_for_genre.call_args_list
] == [3]
assert db.save_movies.call_count == 0
assert db.movies[1234].netflix_rating is None
remove_test_tables(db)
def test_add_columns(mocker):
create_test_tables()
db = MySQLDatabase(schema='qmdb_test', env='test')
mocker.patch.object(db, 'add_column')
db.add_columns([{
'column_name': 'test_column1',
'column_datatype': 'mediumint'
}, {
'column_name': 'test_column2',
'column_datatype': 'bigint'
}],
table_name=['movies_copy', 'movies_copy2'])
assert db.add_column.call_count == 4
assert db.add_column.call_args_list[3][0] == ('test_column2', 'bigint')
assert db.add_column.call_args_list[3][1] == {
'table_name': 'movies_copy2',
'after': None,
'first': None
}
db.add_columns([{
'column_name': 'test_column1',
'column_datatype': 'mediumint'
}, {
'column_name': 'test_column2',
'column_datatype': 'bigint'
}],
table_name='movies_copy')
assert db.add_column.call_count == 6
assert db.add_column.call_args_list[5][0] == ('test_column2', 'bigint')
assert db.add_column.call_args_list[5][1] == {
'table_name': 'movies_copy',
'after': None,
'first': None
}
remove_test_tables(db)
def test_get_existing_movie():
create_test_tables()
db = MySQLDatabase(schema='qmdb_test', env='test')
movie = db.get_movie(1234)
assert movie.title == 'The Matrix'
assert movie.year == 1999
remove_test_tables(db)
def test_get_movies_for_genre():
create_test_tables(variant='updates')
db = MySQLDatabase(schema='qmdb_test', env='tst')
db.netflix_genres = {1: None}
netflix_scraper = NetflixScraper(db)
movies = netflix_scraper.get_movies_for_genre(2)
assert movies == [1, 1]
remove_test_tables(db)
def test_database_init_existing_file():
create_test_tables()
db = MySQLDatabase(schema='qmdb_test', env='test')
assert db.c is not None
assert isinstance(db.movies, dict)
print(db.movies.keys())
assert list(db.movies.keys()) == [1234, 49141]
remove_test_tables(db)
def test_do_unogs_request():
create_test_tables(variant='updates')
db = MySQLDatabase(schema='qmdb_test', env='tst')
netflix_scraper = NetflixScraper(db)
rjson = netflix_scraper.do_unogs_request(
"https://unogs-unogs-v1.p.mashape.com/api.cgi?t=genres")
assert rjson['COUNT'] == '517'
assert len(rjson['ITEMS']) == 517
remove_test_tables(db)
def test_update_movies(mocker):
mocker.patch.object(Updater, 'get_all_next_updates', lambda x, y, **kwargs: d)
mocker.patch.object(time, 'sleep', lambda x: None)
mocker.patch.object(Updater, 'update_source', lambda x, y, z: None)
create_test_tables(variant='updates')
db = MySQLDatabase(schema='qmdb_test', env='tst')
updater = Updater()
updater.update_movies(db, weibull_lambda=10000)
# TODO: change this unit test to test get_update_sequence, assert timings of next updates
remove_test_tables(db)
def test_update_source():
create_test_tables()
db = MySQLDatabase(schema='qmdb_test', env='tst')
updater = Updater()
crit_id = 1234
updater.update_source(db, {'source': 'criticker', 'crit_id': crit_id})
assert db.movies[crit_id].criticker_updated >= arrow.now().shift(minutes=-1)
updater.update_source(db, {'source': 'omdb', 'crit_id': crit_id})
assert db.movies[crit_id].omdb_updated >= arrow.now().shift(minutes=-1)
remove_test_tables(db)
def test_movie_to_dict_languages():
create_test_tables()
db = MySQLDatabase(schema='qmdb_test', env='test')
movie = Movie({'crit_id': 1234, 'languages': ['English', 'Spanish']})
d = db.movie_to_dict_languages(movie)
assert d == {
'crit_id': 1234,
'n_rows': 2,
'language': ['English', 'Spanish'],
'rank': [1, 2]
}
def test_load_languages(mocker):
create_test_tables()
db = MySQLDatabase(schema='qmdb_test', env='test')
mocker.patch.object(db, 'load_or_initialize', lambda x: None)
movies = {1234: dict(), 49141: dict()}
db.connect()
db.load_languages(movies)
db.close()
assert movies[1234]['languages'] == ['English', 'French']
assert movies[49141]['languages'] == ['English']
remove_test_tables(db)
def test_create_insert_multiple_records_sql():
create_test_tables()
db = MySQLDatabase(schema='qmdb_test', env='test')
d = {
'crit_id': 123,
'n_rows': 3,
'rank': [1, 2, 3],
'tagline': ['Hello', 'Hi', 'Hey']
}
sql, values = db.create_insert_multiple_records_sql('taglines', d)
remove_test_tables(db)
def test_get_update_sequence():
create_test_tables(variant='updates')
db = MySQLDatabase(schema='qmdb_test', env='tst')
updater = Updater()
updater.get_movies_stats(db)
seq = updater.get_all_next_updates(db, weibull_lambda=10000)
assert len(seq) == 48
for e in seq:
assert isinstance(e, dict)
assert len([e for e in seq if e['source'] == 'criticker']) == 5
assert len([e for e in seq if e['source'] == 'omdb']) == 3
assert len([e for e in seq if e['source'] == 'ptp']) == 5
remove_test_tables(db)
def test_update_multiple_records():
create_test_tables()
db = MySQLDatabase(schema='qmdb_test', env='test')
d = {
'crit_id': 1234,
'n_rows': 2,
'language': ['English', 'Spanish'],
'rank': [1, 2]
}
db.update_multiple_records('languages', d)
db.load()
assert db.movies[1234].languages == ['English', 'Spanish']
assert db.movies[49141].languages == ['English']
remove_test_tables(db)
def test_get_movies(mocker):
create_test_tables()
db = MySQLDatabase(schema='qmdb_test', env='test')
crit_scraper = CritickerScraper()
crit_scraper.get_movies(db, start_popularity=8)
save_movies_call_args = db.save_movies.call_args_list[0][0]
assert save_movies_call_args[0] == [1, 1, 1]
assert crit_scraper.get_movies_of_popularity.call_args_list[0][1] ==\
{'debug': False, 'min_year': 2013, 'popularity': 10}
assert crit_scraper.get_movies_of_popularity.call_args_list[1][1] ==\
{'debug': False, 'min_year': 2016, 'popularity': 9}
assert crit_scraper.get_movies_of_popularity.call_args_list[2][1] ==\
{'debug': False, 'min_year': 2018, 'popularity': 8}
remove_test_tables(db)
def test_load_netflix_genres():
create_test_tables()
db = MySQLDatabase(schema='qmdb_test', env='test')
db.connect()
db.load_netflix_genres()
assert db.netflix_genres == {
1: {
'genre_names': ['All Action', 'All Anime'],
'movies_updated': arrow.get('2018-02-04 23:01:58+01:00')
},
2: {
'genre_names': ['All Anime'],
'movies_updated': None
}
}
def test_load_persons(mocker):
create_test_tables()
db = MySQLDatabase(schema='qmdb_test', env='test')
mocker.patch.object(db, 'load_or_initialize', lambda x: None)
movies = {1234: dict(), 49141: dict()}
db.connect()
db.load_persons(movies)
db.close()
assert [e['name'] for e in movies[1234]['director']
] == ['Lana Wachowski', 'J.J. Abrams']
assert [e['name'] for e in movies[1234]['cast']
] == ['Anthony Hopkins', 'Tom Cruise']
assert [e['name']
for e in movies[49141]['director']] == ['Steven Spielberg']
assert [e['name'] for e in movies[49141]['cast']] == ['Natalie Portman']
remove_test_tables(db)
def test_add_weird_movie():
create_test_tables()
db = MySQLDatabase(schema='qmdb_test', env='test')
new_movie = {
'crit_id': 60326,
'crit_popularity_page': 79,
'crit_url': 'https://www.criticker.com/film/alg305-cengi/',
'title': 'Çalgı çengi',
'year': 2011,
'date_added': arrow.now()
}
db.set_movie(Movie(new_movie))
db = MySQLDatabase(schema='qmdb_test', env='test')
assert 60326 in list(db.movies.keys())
assert db.movies[60326].title == 'Çalgı çengi'
remove_test_tables(db)
def test_add_new_movie():
create_test_tables()
db = MySQLDatabase(schema='qmdb_test', env='test')
new_movie = {
'crit_id': 12345,
'crit_popularity_page': 1,
'crit_url': 'blahblah',
'title': 'Pulp Fiction',
'poster_url': 'http://blahblah',
'date_added': arrow.now()
}
db.set_movie(Movie(new_movie))
db = MySQLDatabase(schema='qmdb_test', env='test')
assert 12345 in list(db.movies.keys())
assert db.movies[12345].title == 'Pulp Fiction'
remove_test_tables(db)
def test_parse_store_load_persons():
create_test_tables()
db = MySQLDatabase(schema='qmdb_test', env='test')
imdb_scraper = IMDBScraper()
imdbid = '3315342'
movie_info = imdb_scraper.process_main_info(imdbid)
movie_info.update({
'crit_id': 1234,
'title': 'Logan',
'year': 2017,
'crit_url': 'http://www.criticker.com/film/Logan/',
'date_added': '2018-01-01'
})
movie = Movie(movie_info)
db.set_movie(movie)
db = MySQLDatabase(schema='qmdb_test', env='test')
assert db.movies[1234].cast[0]['name'] == 'Hugh Jackman'
assert db.movies[1234].director[0]['name'] == 'James Mangold'
assert db.movies[1234].writer[1]['name'] == 'Scott Frank'
def test_add_column():
create_test_tables()
db = MySQLDatabase(schema='qmdb_test', env='test')
db.connect()
db.add_column('test_column',
'mediumint',
table_name='movies',
after='crit_id')
db.c.execute("""
SELECT column_name,
column_type
FROM information_schema.columns
WHERE table_name='movies' and table_schema='qmdb_test'
""")
columns = db.c.fetchall()
assert columns[1]['column_name'] == 'test_column'
assert columns[1]['column_type'] == 'mediumint(9)'
db.add_column('imdbid', 'integer', table_name='movies')
db.close()
remove_test_tables(db)
def test_get_movies_for_genre_page():
create_test_tables(variant='updates')
db = MySQLDatabase(schema='qmdb_test', env='tst')
netflix_scraper = NetflixScraper(db)
with requests_mock.Mocker() as m:
url = 'https://unogs-unogs-v1.p.mashape.com/aaapi.cgi?q={query}-!1800,2050-!0,5-!0,10-!10673-!Any-!Any-!Any-' \
'!Any-!{downloadable}&t=ns&cl=67&st=adv&ob=Relevance&p=1&sa=and'
headers = {'X-RateLimit-requests-Remaining': '100'}
m.get(url,
text=read_file('test/fixtures/unogs-get-genre-page.html'),
headers=headers)
nr_pages, movies = netflix_scraper.get_movies_for_genre_page(
10673, country_code=67, pagenr=1)
assert nr_pages == 1
assert len(movies) == 59
assert set(movies[0].keys()) == {
'netflix_id', 'netflix_title', 'netflix_rating', 'imdbid', 'crit_id',
'unogs_updated'
}
remove_test_tables(db)
def test_create_imdbid_to_crit_id_dict():
create_test_tables()
db = MySQLDatabase(schema='qmdb_test', env='test')
db.movies = {
1: Movie({
'crit_id': 1,
'imdbid': 101
}),
2: Movie({
'crit_id': 2,
'imdbid': 102
}),
3: Movie({
'crit_id': 3,
'imdbid': 102
}),
4: Movie({
'crit_id': 4,
'imdbid': None
})
}
db.create_imdbid_to_crit_id_dict()
assert db.imdbid_to_critid == {101: 1, 102: {2, 3}}
def test_unogs_movie_info_to_dict():
create_test_tables(variant='updates')
db = MySQLDatabase(schema='qmdb_test', env='tst')
netflix_scraper = NetflixScraper(db)
movie_info = netflix_scraper.unogs_movie_info_to_dict({
'netflixid': '80192014',
'download': '0',
'imdbid': 'tt4797160',
'rating': '0',
'title': 'Larceny'
})
assert set(movie_info.keys()) == {
'netflix_id', 'netflix_title', 'netflix_rating', 'imdbid',
'unogs_updated'
}
assert movie_info['netflix_id'] == 80192014
assert movie_info['netflix_title'] == 'Larceny'
assert movie_info['netflix_rating'] == 0.0
assert movie_info['imdbid'] == 4797160
movie_info = netflix_scraper.unogs_movie_info_to_dict({
'netflixid': '80192014',
'download': '0',
'imdbid': 'notfound',
'rating': '0',
'title': 'Larceny'
})
assert movie_info is None
movie_info = netflix_scraper.unogs_movie_info_to_dict({
'netflixid': '80192014',
'download': '0',
'imdbid': 'tt4797160',
'rating': '',
'title': 'Larceny'
})
assert movie_info['netflix_rating'] is None
remove_test_tables(db)
def test_get_genre_ids():
create_test_tables(variant='updates')
db = MySQLDatabase(schema='qmdb_test', env='tst')
netflix_scraper = NetflixScraper(db)
netflix_scraper.get_genre_ids()
assert db.netflix_genres == {
1: {
'genre_names': ['All Action'],
'movies_updated': None
},
2: {
'genre_names': ['All Action', 'All Anime'],
'movies_updated': None
},
3: {
'genre_names': ['All Anime'],
'movies_updated': None
},
4: {
'genre_names': ['All Action'],
'movies_updated': None
}
}
remove_test_tables(db)
def test_update_existing_movie():
create_test_tables()
db = MySQLDatabase(schema='qmdb_test', env='test')
existing_movie = {
'crit_id': 1234,
'title': 'The Matrix 2',
'genres': ['Action', 'Sci-Fi']
}
movie = db.movies[existing_movie['crit_id']]
movie.update_from_dict(existing_movie)
db.set_movie(movie)
db.connect()
db.c.execute("select * from genres where crit_id = %s",
[existing_movie['crit_id']])
results = db.c.fetchall()
db.close()
assert set([result['genre'] for result in results]) == {'Action', 'Sci-Fi'}
assert db.movies[1234].year == 1999
db = MySQLDatabase(schema='qmdb_test', env='test')
assert 1234 in list(db.movies.keys())
assert db.movies[1234].title == 'The Matrix 2'
assert db.movies[1234].year == 1999
assert db.movies[1234].genres == ['Action', 'Sci-Fi']
remove_test_tables(db)
