def deferred_consume_artist_albums(artist_url, response_url=None):
try:
existing_albums = list_model.get_list()
artist_albums = bandcamp.scrape_bandcamp_album_ids_from_artist_page(artist_url)
new_album_ids = [album_id for album_id in artist_albums if album_id not in existing_albums]
if response_url and new_album_ids:
requests.post(response_url,
data=json.dumps({'text': f':full_moon: found {len(new_album_ids)} new albums to process...'}))
elif response_url:
requests.post(response_url, data=json.dumps({'text': f':new_moon: found no new albums to process'}))
except DatabaseError as e:
print('[db]: failed to check existing items')
print(f'[db]: {e}')
except NotFoundError:
print(f'[scraper]: no albums found for artist at {artist_url}')
if response_url:
requests.post(response_url, data=json.dumps({'text': ':red_circle: failed to find any albums'}))
else:
for new_album_id in new_album_ids:
try:
list_model.add_to_list(new_album_id)
deferred_process_album_details.delay(str(new_album_id))
except DatabaseError as e:
print(f'[db]: failed to update list with {new_album_id} from {artist_url}')
print(f'[db]: {e}')
if response_url and new_album_ids:
requests.post(response_url,
data=json.dumps({'text': f':full_moon_with_face: done processing artist albums'}))
def deferred_add_new_album_details(album_id, added, album, artist, channel, img, tags, url, users):
try:
if album_id not in list_model.get_list():
list_model.add_to_list(album_id)
albums_model.add_to_albums(album_id, artist=artist, name=album, url=url, img=img, channel=channel)
if added:
albums_model.update_album_added(album_id, added)
if not img:
deferred_process_album_cover.delay(album_id)
if tags is not None:
if isinstance(tags, str):
tags = ast.literal_eval(tags)
deferred_process_tags.delay(album_id, tags)
else:
deferred_process_album_tags.delay(album_id)
if users is not None:
if isinstance(users, str):
users = ast.literal_eval(users)
deferred_process_users.delay(album_id, users)
deferred_check_album_url.delay(album_id)
except DatabaseError as e:
print(f'[db]: failed to add new album details for [{album_id}] {album} by {artist}')
print(f'[db]: {e}')
else:
print(f'[db]: added new album details for [{album_id}] {album} by {artist}')
def deferred_add_new_album_details(album_id, added, album, artist, channel,
img, tags, url, users):
try:
if album_id not in list_model.get_list():
list_model.add_to_list(album_id)
albums_model.add_to_albums(album_id,
artist=artist,
name=album,
url=url,
img=img,
channel=channel)
if added:
albums_model.update_album_added(album_id, added)
if not img:
deferred_process_album_cover.delay(album_id)
if tags is not None:
if isinstance(tags, str):
tags = ast.literal_eval(tags)
deferred_process_tags.delay(album_id, tags)
else:
deferred_process_album_tags.delay(album_id)
if users is not None:
if isinstance(users, str):
users = ast.literal_eval(users)
deferred_process_users.delay(album_id, users)
deferred_check_album_url.delay(album_id)
except DatabaseError as e:
print(
f'[db]: failed to add new album details for [{album_id}] {album} by {artist}'
)
print(f'[db]: {e}')
else:
print(
f'[db]: added new album details for [{album_id}] {album} by {artist}'
)
def api_list_albums():
try:
return flask.jsonify(list_model.get_list()), 200
except DatabaseError as e:
print('[db]: failed to get list')
print(f'[db]: {e}')
return flask.jsonify({'text': 'failed'}), 500
def api_list_albums():
try:
return flask.jsonify(list_model.get_list()), 200
except DatabaseError as e:
print('[db]: failed to get list')
print(f'[db]: {e}')
return flask.jsonify({'text': 'failed'}), 500
def deferred_consume_artist_albums(artist_url, response_url=None):
try:
existing_albums = list_model.get_list()
artist_albums = bandcamp.scrape_bandcamp_album_ids_from_artist_page(
artist_url)
new_album_ids = [
album_id for album_id in artist_albums
if album_id not in existing_albums
]
if response_url and new_album_ids:
requests.post(
response_url,
data=json.dumps({
'text':
f':full_moon: found {len(new_album_ids)} new albums to process...'
}))
elif response_url:
requests.post(response_url,
data=json.dumps({
'text':
f':new_moon: found no new albums to process'
}))
except DatabaseError as e:
print('[db]: failed to check existing items')
print(f'[db]: {e}')
except NotFoundError:
print(f'[scraper]: no albums found for artist at {artist_url}')
if response_url:
requests.post(response_url,
data=json.dumps({
'text':
':red_circle: failed to find any albums'
}))
else:
for new_album_id in new_album_ids:
try:
list_model.add_to_list(new_album_id)
deferred_process_album_details.delay(str(new_album_id))
except DatabaseError as e:
print(
f'[db]: failed to update list with {new_album_id} from {artist_url}'
)
print(f'[db]: {e}')
if response_url and new_album_ids:
requests.post(
response_url,
data=json.dumps({
'text':
f':full_moon_with_face: done processing artist albums'
}))
def deferred_consume(url, scrape_function, callback, channel='', tags=None, slack_token=None, response_url=None):
try:
album_id = scrape_function(url)
except NotFoundError:
print(f'[scraper]: no album id found at {url}')
else:
if slack_token:
slack = slacker.Slacker(slack_token)
try:
if album_id not in list_model.get_list():
try:
callback(album_id)
except DatabaseError as e:
print(f'[db]: failed to perform {callback.__name__}')
print(f'[db]: {e}')
if response_url:
requests.post(response_url, data=json.dumps({'text': ':red_circle: failed to update list'}))
elif slack_token and channel:
slack.chat.post_message(f'{channel}', ':red_circle: failed to update list')
else:
if response_url:
requests.post(response_url, data=json.dumps(
{'text': f':full_moon: added album to list: {url}', 'unfurl_links': True}))
elif slack_token and channel:
slack.chat.post_message(f'{channel}', f':full_moon: added album to list: {url}',
unfurl_links=True)
deferred_process_album_details.delay(str(album_id), channel, slack_token)
elif response_url:
requests.post(response_url, data=json.dumps(
{'text': f':new_moon: album already in list: {url}', 'unfurl_links': True}))
elif slack_token and channel:
slack.chat.post_message(f'{channel}', f':new_moon: album already in list: {url}', unfurl_links=True)
if tags:
deferred_process_tags.delay(str(album_id), tags)
except DatabaseError as e:
print('[db]: failed to check existing items')
print(f'[db]: {e}')
def check_for_new_albums():
return [
str(album_id)
for album_id in set(get_list()).difference(set(get_album_ids()))
if album_id is not None
]
def deferred_consume(url,
scrape_function,
callback,
channel='',
tags=None,
slack_token=None,
response_url=None):
try:
album_id = scrape_function(url)
except NotFoundError:
print(f'[scraper]: no album id found at {url}')
else:
if slack_token:
slack = slacker.Slacker(slack_token)
try:
if album_id not in list_model.get_list():
try:
callback(album_id)
except DatabaseError as e:
print(f'[db]: failed to perform {callback.__name__}')
print(f'[db]: {e}')
if response_url:
requests.post(response_url,
data=json.dumps({
'text':
':red_circle: failed to update list'
}))
elif slack_token and channel:
slack.chat.post_message(
f'{channel}', ':red_circle: failed to update list')
else:
if response_url:
requests.post(
response_url,
data=json.dumps({
'text':
f':full_moon: added album to list: {url}',
'unfurl_links': True
}))
elif slack_token and channel:
slack.chat.post_message(
f'{channel}',
f':full_moon: added album to list: {url}',
unfurl_links=True)
deferred_process_album_details.delay(
str(album_id), channel, slack_token)
elif response_url:
requests.post(response_url,
data=json.dumps({
'text':
f':new_moon: album already in list: {url}',
'unfurl_links': True
}))
elif slack_token and channel:
slack.chat.post_message(
f'{channel}',
f':new_moon: album already in list: {url}',
unfurl_links=True)
if tags:
deferred_process_tags.delay(str(album_id), tags)
except DatabaseError as e:
print('[db]: failed to check existing items')
print(f'[db]: {e}')
def check_for_new_albums():
return [
str(album_id)
for album_id in set(get_list()).difference(set(get_album_ids()))
if album_id is not None
]