def validate_music_archive(delete=False):
"""get all tracks from db
check each track, creates a list of songs which are not fully or wrong downloaded
this list does not include not downloaded tracks
Used function `chimera.utils.check_track_length` for file comparison"""
wrong_tracks = []
session = create_session()
dbtracks = session.query(DBTrack).all()
for dbtrack in dbtracks:
if check_track_length(dbtrack) == False:
wrong_tracks.append(dbtrack)
length_file = get_file_length_seconds(
os.path.join(cc.root_path, dbtrack.path))
err_msg = f'WRONG: {dbtrack} DEEZER: {dbtrack.remote_duration} LOCAL: {length_file}'
write_outfile_wrong(err_msg)
print(err_msg)
if delete:
try:
# remove from disk
os.remove(os.path.join(cc.root_path, dbtrack.path))
except FileNotFoundError as e:
pass
# remove from db
session.delete(dbtrack)
session.commit()
def wrapper(*args, **kwargs):
try:
return f(*args, **kwargs)
except OperationalError as e:
if 'database is locked' in e.args[0]:
time.sleep(random.randint(1, 10))
# get session for this thread
session = create_session()
session.rollback()
return f(*args, **kwargs)
def update_tracks_in_db(deezer):
"""
Temporary function to update `DBTrack.remote_duration`
"""
session = create_session()
dbtracks = session.query(DBTrack).all()
for dbtrack in dbtracks:
if dbtrack.service is None:
print(f'Updating Track: {dbtrack.id}')
dbtrack.service = 'deezer'
session.commit()
def add(track):
"""
Add Deezer Track to database
Searches for existing track with track.remote_id if not found
creates a new entry
"""
session = create_session()
_track = session.query(
db.models.DBTrack).filter_by(remote_id=track.song_id).first()
if _track is None:
# track not existing, create new
dbtrack = db.models.DBTrack(name=track.title,
artist=track.artist,
album=track.album.title,
release_date=track.album.date,
isrc=track.isrc,
spotify_id=track.spotify_id if hasattr(
track, 'spotify_id') else None,
remote_id=track.song_id,
path=track.path.full_file_name,
remote_duration=track.duration,
service=track.service)
session.add(dbtrack)
session.commit()
else:
# track exists
_track.name = track.title
_track.artist = track.artist
_track.album = track.album.title
_track.release_date = track.album.date
_track.isrc = track.isrc
_track.spotify_id = track.spotify_id if hasattr(track,
'spotify_id') else None
_track.remote_id = track.song_id
_track.path = track.path.full_file_name
_track.remote_duration = track.duration
_track.service = track.service
session.commit()
chrome_options.add_argument("--window-size=1920x1080")
self.driver = webdriver.Chrome(chrome_options=chrome_options, executable_path='./chromedriver')
def get_driver(self):
return self.driver
def close(self):
self.driver.quit()
if __name__ == '__main__':
#parser = argparse.ArgumentParser(description='Scrape restaurant reviews from Tripadvisor')
#args = parser.parse_args()
db = create_session()
scraper = TripadvisorScraper()
driver = scraper.get_driver()
filepath = './data/locations.txt'
with open(filepath) as fp:
line = fp.readline()
while line:
restaurants_scraper = TripadvisorRestaurantsScraper(driver, line)
restaurants_urls = restaurants_scraper.fetch_restaurants()
for restaurant_url in restaurants_urls:
restaurant_scraper = TripadvisorRestaurantScraper(driver, restaurant_url)
restaurant = restaurant_scraper.fetch_restaurant()
if restaurant:
insert_restaurant_data(db, restaurant)
review_scraper = TripadvisorReviewScraper(db, driver, restaurant_url)
def sp_track(sptrack: SpotifyTrack, session, task=None, dlthread=None):
"""converst sp track to active session track
deezer supports isrc lookup"""
track = None
# 1. check db
db_session = create_session()
track = db_session.query(DBTrack).filter_by(isrc=sptrack.isrc).first()
if track:
if os.path.isfile(os.path.join(cc.root_path, track.path)):
if task:
task.finished = True
return DownloadResponse(False,
'Track already in db',
file_name=os.path.join(
cc.root_path, track.path),
status_code=102)
else:
track = None # maybe different fix
# 2. ISRC lookup
if str(session) == 'DEEZER':
track = session.search_isrc(sptrack.isrc)
# 3. no isrc match, search on services
if track is None:
services = {str(session).lower(): session}
track = search_track_on_services(sptrack.title,
sptrack.artist,
sptrack.album.title,
**services,
isrc=sptrack.isrc,
verbose=False if dlthread else True)
# if track found track is no longer false, check again
if track is None:
if task:
task.finished = True
return DownloadResponse(True, 'Track not found on services!')
# 4.a) copy over playlist details
track.is_playlist = sptrack.is_playlist
track.playlist_name = sptrack.playlist_name
track.playlist_index = sptrack.playlist_index
track.playlist_length = sptrack.playlist_length
# 4.b) deezer download
if str(session) == 'DEEZER':
ds = any_track(track,
session,
overwrite=cc.dl_playlist_overwrite,
add_to_db=cc.dl_playlist_add_to_db,
check_db=cc.dl_playlist_check_db,
lyrics=cc.tag_lyrics,
save_lyrics=cc.save_lyrics,
task=task,
dlthread=dlthread)
return ds
# 4.c) everything else
else:
ds = any_track(track,
session,
overwrite=cc.dl_playlist_overwrite,
add_to_db=cc.dl_playlist_add_to_db,
check_db=cc.dl_playlist_check_db,
task=task,
dlthread=dlthread)
return ds
def search_dbtrack(track):
db_session = create_session()
return db_session.query(DBTrack).filter_by(remote_id=track.song_id).first()