def test_request(es_cdlc):
assert_that(list(CustomDLC.search('definitely not here'))).is_empty()
cdlcs = list(CustomDLC.search('paradise'))
assert_that(cdlcs).is_length(1)
assert_that(
cdlcs[0].full_title).is_equal_to('ZUN - Paradise ~ Deep Mountain')
def try_write(self, cdlc: dict) -> Any:
continuous_from = cdlc.pop(CONTINUOUS_FROM, None)
is_continuous = self.__continuous and continuous_from is not None and continuous_from <= self.start_from()
cdlc_id = cdlc.get('id', None)
c = CustomDLC(_id=cdlc_id, from_auto_index=is_continuous, **cdlc)
c.save()
LOG.warning('Indexed CDLC #%s.', cdlc_id)
return None if c.direct_download else c
def test_load_links(tl):
# this simulates the scenario where link was empty due to error
CustomDLC(_id=49841).update(direct_download='')
with HTTMock(customsforge):
tl.load_links()
assert_that([hit.link for hit in CustomDLC.search()]) \
.is_length(6) \
.contains_only('magical_link')
def test_loading_continued(tl):
# pretend we loaded cdlcs 2 days before TEST_DATE
for cdlc_id in ['49706', '12990', '49792', '49841']:
CustomDLC(_id=cdlc_id).update(from_auto_index=True)
with HTTMock(customsforge):
tl.load()
hits = list(CustomDLC.search().filter('term', from_auto_index=True).filter('term', direct_download='magical_link'))
# the only updated cdlcs are from the last two days (one each), and the latest cdlc before
assert_that(hits).is_length(3)
def test_properties(es_cdlc):
cdlc = CustomDLC.get(49874)
assert_that(
cdlc.full_title).is_equal_to("Trey Parker - Jackin' It In San Diego")
assert_that(
cdlc.link).is_equal_to('https://customsforge.com/process.php?id=49874')
cdlc.update(direct_download='direct_link')
same_cdlc = list(CustomDLC.search().query('match', id='49874'))
assert_that(same_cdlc).is_length(1)
assert_that(same_cdlc[0].link).is_equal_to('direct_link')
def execute(self, user: User, alias: str, args: str,
respond: ResponseHook) -> bool:
"""
Same as !request, but automatically picks from ALL possible matches. This includes matches that cannot
be picked with !pick due to lower relevance. Usually best used with artists, e.g. !random acdc
"""
with self._queue:
match = CustomDLC.random(args, *self.__exclusions())
if not match:
without_exclusions = CustomDLC.random_pool(query=args).count()
message = f'Everything already played or enqueued' if without_exclusions else f'No matches'
return respond.to_sender(f'{message} for <{args}>')
request = Match(user, args, match)
return self._enqueue_request(user, request, respond)
def test_loading_from_start(tl):
with HTTMock(customsforge):
tl.load()
assert_that([hit.link for hit in CustomDLC.search().filter('term', from_auto_index=True)]) \
.is_length(6) \
.contains_only('magical_link')
def domains() -> FrozenSet[str]:
"""
:returns set of domains for all links in the current index.
"""
return frozenset(
extract(hit.link).registered_domain
for hit in CustomDLC.search().scan())
def test_last_auto_index_time(es_cdlc):
assert_that(CustomDLC.latest_auto_time()).is_none()
# this cdlc IS NOT the earliest in the index, so it will be considered non-continuous
CustomDLC(_id=49792).update(from_auto_index=True)
assert_that(CustomDLC.latest_auto_time()).is_none()
assert_that(CustomDLC.earliest_not_auto()).is_equal_to(1589092730)
# this cdlc IS the earliest in the index, so it AND ONLY IT will be considered continuous
CustomDLC(_id=49706).update(from_auto_index=True)
assert_that(CustomDLC.latest_auto_time()).is_equal_to(1589092730)
assert_that(CustomDLC.earliest_not_auto()).is_equal_to(1589162625)
# after this, all cdlcs marked as indexed are continuous, since no unmarked cdlcs are in between
CustomDLC(_id=12990).update(from_auto_index=True)
assert_that(CustomDLC.latest_auto_time()).is_equal_to(1589249216)
assert_that(CustomDLC.earliest_not_auto()).is_equal_to(1589377216)
def try_write(self, cdlc: dict) -> Any:
cdlc_id = cdlc.get('id', None)
art = cdlc.get('art', '')
try:
CustomDLC(_id=cdlc_id).update(art=art)
except NotFoundError as e:
LOG.warning('Art update failed for CDLC #%s, probably because it has not been loaded yet.', cdlc_id)
debug_ex(e, 'fix my sins', LOG, silent=True)
return None
def find(query: str, results: int = None) -> List[CustomDLC]:
"""
Searches for matching CDLCs in the index.
"""
s = CustomDLC.search(query)
result = list(s[:results] if results else s)
if not result:
LOG.warning('No CDLCs matching <%s> were found.', query)
for hit in result:
LOG.warning('(%5.2f) Found CDLC#%05d %s: <%s>', hit.meta.score, hit.id,
hit, hit.link)
return result
def read_all(self, start_from: int = 0) -> Iterator[dict]:
LOG.warning('Loading elastic index CDLCs from %s (%s).', loading_from(start_from), self.__describe_mode())
timestamp_range = {'gte': start_from} if start_from else {}
s = CustomDLC.search()
if self.__continuous:
s = s.filter('term', from_auto_index=True).sort('snapshot_timestamp').params(preserve_order=True)
timestamp_range['lte'] = self.start_from()
if timestamp_range:
s = s.filter('range', snapshot_timestamp=timestamp_range)
for hit in s.scan():
cdlc = hit.to_dict()
cdlc.pop('from_auto_index', None)
if self.__continuous:
cdlc[CONTINUOUS_FROM] = start_from or 0
yield cdlc
def execute(self, user: User, alias: str, args: str,
respond: ResponseHook) -> bool:
"""
Adds any matching songs to the request queue. Query is taken as a literal search string. Can be empty.
If multiple matches are found, you will be able to use !pick to choose the most relevant ones.
The following rules do NOT apply to ADMIN rank:
* If you already had a song in the queue, replaces it instead.
* If the song is already in the queue, does not add it.
* If the song has been played already, does not add it.
"""
matches = list(CustomDLC.search(query=args)[:self.__max_search])
if not matches:
return respond.to_sender(f'No matches for <{args}>')
playable = list(filter(lambda match: match.is_playable, matches))
if not playable:
unplayable = '; '.join(match.short for match in matches)
return respond.to_sender(
f'Matches for <{args}> not playable: {unplayable}')
request = Match(user, args, *playable[:self.__max_pick])
return self._enqueue_request(user, request, respond)
def __enter__(self):
self.__start_from = CustomDLC.latest_auto_time() if self.__continuous else None
def domain_all(domain: str) -> Iterator[str]:
for hit in CustomDLC.search().scan():
if extract(hit.link).registered_domain == domain:
LOG.warning('Found CDLC#%05d <%s>', hit.id, hit)
yield hit.link
def to_direct_link(self, cdlc_id: Union[str, int]) -> str:
c = CustomDLC.get(cdlc_id, ignore=[404])
return c.direct_download if c else None
def update(self, from_write: Any, direct_link: str):
cdlc_id = from_write
if cdlc_id and direct_link:
CustomDLC(_id=cdlc_id).update(direct_download=direct_link)
LOG.warning('Indexed direct link for CDLC #%s.', cdlc_id)
def test_partial_update_for_non_existent_document(es_cdlc):
try:
CustomDLC(_id=100000).update(id=100000)
assert False # exception should be thrown!
except NotFoundError:
assert_that(CustomDLC.get(100000, ignore=[404])).is_none()
def test_playable(es_cdlc):
assert_that(CustomDLC.get(
49706).is_playable).is_true() # official is still playable
assert_that(CustomDLC.get(49792).is_playable).is_false() # no pc
assert_that(
CustomDLC.get(49841).is_playable).is_false() # no lead or rhythm
def test_random(es_cdlc):
assert_that(CustomDLC.random('definitely not here')).is_none()
assert_that(CustomDLC.random().id).is_in(12990, 49874, 49886)
def test_filtered_pools(es_cdlc):
assert_that(list(CustomDLC.playable())).is_length(
4) # 2 mock CDLCs out of 6 are not playable
assert_that(list(CustomDLC.random_pool())).is_length(
3) # 1 more CDLC is official