def do_push_artists(self):
# patch credentials
if not request.headers.get('Authorization'):
abort(401)
else:
auth = request.headers['Authorization'].lstrip('Basic ')
username, password = base64.b64decode(auth).split(':')
if username and password:
conf.CHIRPRADIO_AUTH = '%s %s' % (username, password)
chirpradio.connect()
else:
abort(401)
dry_run = False
# reload artists from file
artists._init()
# Find all of the library artists
all_library_artists = set(artists.all())
# Find all of the artists in the cloud.
all_chirpradio_artists = set()
mapped = 0
t1 = time.time()
for art in models.Artist.fetch_all():
if art.revoked:
continue
std_name = artists.standardize(art.name)
if std_name != art.name:
#print "Mapping %d: %s => %s" % (mapped, art.name, std_name)
mapped += 1
art.name = std_name
idx = search.Indexer()
idx._transaction = art.parent_key()
idx.add_artist(art)
if not dry_run:
idx.save()
all_chirpradio_artists.add(art.name)
to_push = list(all_library_artists.difference(all_chirpradio_artists))
Messages.add_message("Pushing %d artists" % len(to_push), 'warning')
while to_push:
# Push the artists in batches of 50
this_push = to_push[:50]
to_push = to_push[50:]
idx = search.Indexer()
for name in this_push:
#print name
art = models.Artist.create(parent=idx.transaction, name=name)
idx.add_artist(art)
if not dry_run:
idx.save()
#print "+++++ Indexer saved"
Messages.add_message("Artist push complete. OK!", 'success')
def modify_tags_and_save(user, obj, to_add, to_remove):
"""Modify the set of tags attached to an object, and save to the datastore.
Args:
user: The User object of the person responsible for this change
to the tags.
obj: The object (either an Album or a Track) containing the tags.
to_add: A sequence of tags to add to the object.
to_remove: A sequence of tags to remove from the object.
Returns:
True if a modified version of the object was saved, or False if
no changes were necessary.
"""
to_add = list(set(to_add).difference(obj.current_tags))
to_remove = list(set(to_remove).intersection(obj.current_tags))
if not (to_add or to_remove):
return False
obj.current_tags = list(
set(obj.current_tags).union(to_add).difference(to_remove))
tag_edit = models.TagEdit(parent=obj, subject=obj, author=user,
added=to_add, removed=to_remove)
# The two objects are in the same entity group, so saving them
# is an all-or-nothing operation.
AutoRetry(db).save([obj, tag_edit])
# Update search indexer.
idx = search.Indexer(obj.parent_key())
for tag in to_remove:
idx.remove_key(obj.key(), 'tag', tag)
for tag in to_add:
idx.add_key(obj.key(), 'tag', tag)
idx.save()
return True
def main_generator():
chirpradio.connect()
dry_run = False
# Find all of the library artists
all_library_artists = set(artists.all())
# Find all of the artists in the cloud.
all_chirpradio_artists = set()
mapped = 0
t1 = time.time()
for art in models.Artist.fetch_all():
if art.revoked:
continue
std_name = artists.standardize(art.name)
if std_name != art.name:
cprint(u"Mapping {}: {} => {}".format(mapped, art.name, std_name))
mapped += 1
art.name = std_name
idx = search.Indexer()
idx._transaction = art.parent_key()
idx.add_artist(art)
if not dry_run:
idx.save()
all_chirpradio_artists.add(art.name)
yield
to_push = list(all_library_artists.difference(all_chirpradio_artists))
cprint("Pushing %d artists" % len(to_push))
while to_push:
# Push the artists in batches of 50
this_push = to_push[:50]
to_push = to_push[50:]
idx = search.Indexer()
for name in this_push:
cprint(name)
art = models.Artist.create(parent=idx.transaction, name=name)
idx.add_artist(art)
if not dry_run:
idx.save()
cprint("+++++ Indexer saved")
yield
def main():
parser = optparse.OptionParser(usage='%prog')
parser.add_option('--clear-datastore', action='store_true')
(options, args) = parser.parse_args()
startup_appengine(clear_datastore=options.clear_datastore)
from djdb import search, models
import datetime
idx = search.Indexer()
# Create some test artists.
art1 = models.Artist(name=u"Fall, The",
parent=idx.transaction,
key_name="art1")
art2 = models.Artist(name=u"Eno, Brian",
parent=idx.transaction,
key_name="art2")
# Create some test albums.
alb1 = models.Album(title=u"This Nation's Saving Grace",
album_id=12345,
import_timestamp=datetime.datetime.now(),
album_artist=art1,
num_tracks=123,
parent=idx.transaction)
alb2 = models.Album(title=u"Another Green World",
album_id=67890,
import_timestamp=datetime.datetime.now(),
album_artist=art2,
num_tracks=456,
parent=idx.transaction)
for i, track_title in enumerate(
(u"Spider And I", u"Running To Tie Your Shoes", u"Kings Lead Hat")):
idx.add_track(
models.Track(ufid="test3-%d" % i,
album=alb2,
sampling_rate_hz=44110,
bit_rate_kbps=128,
channels="mono",
duration_ms=789,
title=track_title,
track_artist=art2,
track_num=i + 1,
parent=idx.transaction))
idx.add_artist(art1)
idx.add_artist(art2)
idx.add_album(alb1)
idx.add_album(alb2)
idx.save() # saves all objects
print "created some artists and stuff"
def test_basic_indexing_and_search(self):
key1 = db.Key.from_path("kind_Foo", "key1")
key2 = db.Key.from_path("kind_Foo", "key2")
key3 = db.Key.from_path("kind_Bar", "key3")
key4 = db.Key.from_path("kind_Bar", "key4")
idx = search.Indexer()
idx.add_key(key1, "f1", u"alpha beta")
idx.add_key(key2, "f2", u"alpha delta")
idx.save()
idx = search.Indexer()
idx.add_key(key3, "f1", u"alpha gamma")
idx.add_key(key4, "f2", u"alaska")
idx.save()
self.assertEqual(set([(key1, "f1"), (key2, "f2"), (key3, "f1")]),
search.fetch_keys_for_one_term("alpha"))
self.assertEqual(
set([(key1, "f1"), (key2, "f2")]),
search.fetch_keys_for_one_term("alpha", entity_kind="kind_Foo"))
self.assertEqual(set([(key1, "f1"), (key3, "f1")]),
search.fetch_keys_for_one_term("alpha", field="f1"))
self.assertEqual(set([(key1, "f1")]),
search.fetch_keys_for_one_term("beta"))
self.assertEqual(0, len(search.fetch_keys_for_one_term("unknown")))
self.assertEqual(set([(key1, "f1"), (key2, "f2"), (key3, "f1")]),
search.fetch_keys_for_one_prefix("alpha"))
self.assertEqual(
set([(key1, "f1"), (key2, "f2"), (key3, "f1"), (key4, "f2")]),
search.fetch_keys_for_one_prefix("al"))
self.assertEqual(set([(key2, "f2"), (key4, "f2")]),
search.fetch_keys_for_one_prefix("al", field="f2"))
self.assertEqual(0, len(search.fetch_keys_for_one_prefix("unknown")))
def flush(list_of_pending_albums):
if not list_of_pending_albums:
return
idx = search.Indexer()
for alb in list_of_pending_albums:
process_one_album(idx, alb)
# This runs as a batch job, so set a very long deadline.
while True:
try:
rpc = db.create_rpc(deadline=120)
idx.save(rpc=rpc)
return
except urllib2.URLError:
#print "Retrying indexer flush"
pass
def setUp(self):
idx = search.Indexer()
# Create some test artists.
art = models.Artist(name=u"beatles",
parent=idx.transaction,
key_name="ss-art1")
idx.add_artist(art)
art = models.Artist(name=u"beatnicks",
parent=idx.transaction,
key_name="ss-art2")
idx.add_artist(art)
art = models.Artist(name=u"beatnuts",
parent=idx.transaction,
key_name="ss-art3")
idx.add_artist(art)
idx.save()
def bootstrap(request):
"""Inject test library data into the datastore."""
# We never want to do this in prod!
if in_prod():
return http.HttpResponse(status=403)
# First, inject the artist names.
search.create_artists(_ARTIST_NAMES)
# Now build up a bunch of random albums for each artist.
counter = 1
for art in models.Artist.all().fetch(100):
for _ in range(random.randint(1, 10)):
# 10% are multi-disc sets
discs = [None]
if random.randint(1, 10) == 1:
discs = range(1, random.randint(2, 5))
idx = search.Indexer()
for disc_num in discs:
counter += 1
alb = models.Album(
title=random_phrase(),
#label=_LABELS[random.randint(0, len(_LABELS) - 1)],
#year=_YEARS[random.randint(0, len(_YEARS) - 1)],
disc_number=disc_num,
album_id=counter,
import_timestamp=datetime.datetime.now(),
album_artist=art,
num_tracks=random.randint(2, 12),
parent=idx.transaction,
)
idx.add_album(alb)
for i in range(alb.num_tracks):
trk = models.Track(ufid="%d:%d" % (counter, i),
album=alb,
title=random_phrase(),
track_num=i + 1,
sampling_rate_hz=44100,
bit_rate_kbps=128,
channels=u"stereo",
duration_ms=random.randint(
60000, 300000),
parent=idx.transaction)
idx.add_track(trk)
idx.save()
return http.HttpResponseRedirect("/djdb/")
def test_object_indexing(self):
idx = search.Indexer()
# Create some test artists.
art1 = models.Artist(name=u"Fall, The",
parent=idx.transaction,
key_name="art1")
art2 = models.Artist(name=u"Eno, Brian",
parent=idx.transaction,
key_name="art2")
# Create some test single-artist albums.
alb1 = models.Album(title=u"This Nation's Saving Grace",
year=1985,
album_id=12345,
label=u"Some Label",
import_timestamp=datetime.datetime.now(),
album_artist=art1,
num_tracks=123,
parent=idx.transaction)
trk1 = []
for i, track_title in enumerate(
(u"Mansion", u"Bombast", u"Cruiser's Creek", u"What You Need",
u"Spoiled Victorian Child", u"L.A.")):
trk1.append(
models.Track(ufid="test1-%d" % i,
album=alb1,
sampling_rate_hz=44110,
bit_rate_kbps=320,
channels="stereo",
duration_ms=123,
title=track_title,
track_num=i + 1,
parent=idx.transaction))
alb2 = models.Album(title=u"Another Green World",
album_id=67890,
label=u"Some Label",
import_timestamp=datetime.datetime.now(),
album_artist=art2,
num_tracks=456,
parent=idx.transaction)
trk2 = []
for i, track_title in enumerate(
(u"Sky Saw", u"Over Fire Island", u"St. Elmo's Fire",
u"In Dark Trees", u"The Big Ship")):
trk2.append(
models.Track(ufid="test2-%d" % i,
album=alb2,
sampling_rate_hz=44110,
bit_rate_kbps=192,
channels="joint_stereo",
duration_ms=456,
title=track_title,
track_num=i + 1,
parent=idx.transaction))
# Create a test album that is a compilation.
alb3 = models.Album(title=u"R&B Gold: 1976",
album_id=76543,
label=u"Some Label",
import_timestamp=datetime.datetime.now(),
is_compilation=True,
num_tracks=789,
parent=idx.transaction)
trk3_art = []
trk3 = []
for i, (artist_name, track_title) in enumerate(
((u"Earth, Wind & Fire", u"Sing A Song"),
(u"Diana Ross", u"Love Hangover"), (u"Aretha Franklin",
u"Something He Can Feel"),
(u"KC & the Sunshine Band",
u"(Shake, Shake, Shake) Shake Your Booty"))):
art = models.Artist(name=artist_name,
key_name=artist_name,
parent=idx.transaction)
trk3_art.append(art)
trk3.append(
models.Track(ufid="test3-%d" % i,
album=alb3,
sampling_rate_hz=44110,
bit_rate_kbps=128,
channels="mono",
duration_ms=789,
title=track_title,
track_artist=art,
track_num=i + 1,
parent=idx.transaction))
# Now index everything we just created.
idx.add_artist(art1)
idx.add_artist(art2)
for art in trk3_art:
idx.add_artist(art)
idx.add_album(alb1)
idx.add_album(alb2)
idx.add_album(alb3)
for trk in trk1 + trk2 + trk3:
idx.add_track(trk)
idx.save() # This also saves all of the objects.
# Now do some test searches.
# This query matches the album and all of the tracks.
expected = {alb1.key(): set(["title"])}
self.assertEqual(
expected,
search.fetch_keys_for_query_string(u"nations",
entity_kind="Album"))
for t in trk1:
expected[t.key()] = set(["album"])
self.assertEqual(expected,
search.fetch_keys_for_query_string(u"nations"))
# The query "fire" should match:
# * Two of the songs from "Another Green World"
# * The band "Earth, Wind & Fire"
# * The EW&F track from the compilation.
expected = {
trk2[1].key(): set(["title"]),
trk2[2].key(): set(["title"]),
trk3_art[0].key(): set(["name"]),
trk3[0].key(): set(["artist"]),
}
self.assertEqual(expected, search.fetch_keys_for_query_string(u"fire"))
def test_search_using_queries(self):
key1 = db.Key.from_path("kind_Foo", "key1")
key2 = db.Key.from_path("kind_Foo", "key2")
key3 = db.Key.from_path("kind_Foo", "key3")
key4 = db.Key.from_path("kind_Bar", "key4")
key5 = db.Key.from_path("kind_Bar", "key5")
key6 = db.Key.from_path("kind_Bar", "key6")
key7 = db.Key.from_path("kind_Bar", "key7")
idx = search.Indexer()
idx.add_key(key1, "f1", u"alpha beta")
idx.add_key(key2, "f2", u"alpha delta")
idx.add_key(key3, "f1", u"alaska beta")
idx.add_key(key4, "f2", u"beta delta")
idx.add_key(key5, "f1", u"alpha alaska")
idx.add_key(key6, "f2", u"delta gamma")
# an indexed value ending in a stop word:
idx.add_key(key7, "stop-word-prefix", u"something in")
idx.save()
# Check that some simple queries are handled correctly.
self.assertEqual(
{
key1: set(["f1"]),
key2: set(["f2"]),
key5: set(["f1"])
}, search.fetch_keys_for_query_string(u"alpha"))
self.assertEqual(
{
key2: set(["f2"]),
key4: set(["f2"]),
key6: set(["f2"])
}, search.fetch_keys_for_query_string(u"delta"))
self.assertEqual(
{
key1: set(["f1"]),
key2: set(["f2"]),
key3: set(["f1"]),
key5: set(["f1"])
}, search.fetch_keys_for_query_string(u"al*"))
self.assertEqual({key1: set(["f1"])},
search.fetch_keys_for_query_string(u"beta alpha"))
self.assertEqual({
key1: set(["f1"]),
key3: set(["f1"])
}, search.fetch_keys_for_query_string(u"al* beta"))
self.assertEqual({
key2: set(["f2"]),
key5: set(["f1"])
}, search.fetch_keys_for_query_string(u"al* -beta"))
self.assertEqual({
key4: set(["f2"]),
key6: set(["f2"])
}, search.fetch_keys_for_query_string(u"delta -al*"))
# Make sure we can run a prefix search on a stop word
# (this is necessary for autocomplete searches)
self.assertEqual({key7: set(["stop-word-prefix"])},
search.fetch_keys_for_query_string(u"something i*"))
# Check that entity kind restrictions are respected.
self.assertEqual({
key1: set(["f1"]),
key2: set(["f2"])
}, search.fetch_keys_for_query_string(u"alpha",
entity_kind="kind_Foo"))
self.assertEqual({key5: set(["f1"])},
search.fetch_keys_for_query_string(
u"al*", entity_kind="kind_Bar"))
self.assertEqual({key2: set(["f2"])},
search.fetch_keys_for_query_string(
u"al* -beta", entity_kind="kind_Foo"))
# Check that searches against unknown terms are handled properly.
self.assertEqual({}, search.fetch_keys_for_query_string(u"nosuchterm"))
self.assertEqual({},
search.fetch_keys_for_query_string(u"nosuchterm*"))
self.assertEqual(
{}, search.fetch_keys_for_query_string(u"alpha nosuchterm"))
self.assertEqual(
{}, search.fetch_keys_for_query_string(u"alpha nosuchterm*"))
self.assertEqual(
{
key1: set(["f1"]),
key2: set(["f2"]),
key5: set(["f1"])
}, search.fetch_keys_for_query_string(u"alpha -nosuchterm"))
self.assertEqual(
{
key1: set(["f1"]),
key2: set(["f2"]),
key5: set(["f1"])
}, search.fetch_keys_for_query_string(u"alpha -nosuchterm*"))
# Check that None is returned for various invalid/bogus queries.
self.assertEqual(None, search.fetch_keys_for_query_string(u""))
self.assertEqual(None, search.fetch_keys_for_query_string(u"+,,,*"))
self.assertEqual(None, search.fetch_keys_for_query_string(u"-foo"))