def check_for_work(self):
if self.currently_fetching:
defer.returnValue(None)
self.currently_fetching = True
while metadata_db[self.keyify('d', 'queue')]:
obj_id = metadata_db[self.keyify('d', 'queue')][0]
metadata_db[self.keyify('d', 'queue')] = metadata_db[self.keyify('d', 'queue')][1:]
if self.keyify('m', obj_id) in metadata_db:
log.msg('%s Skipping %s' % (self.id, obj_id))
continue
log.msg('Fetching %s info for %s' % (self.id, obj_id))
# timeout of 10minutes
metadata_db[self.keyify('m', obj_id)] = yield self.fetch_metadata(obj_id)
metadata_db[self.keyify('d', 'items')] = metadata_db.get(self.keyify('d', 'items'), []) + [obj_id]
metadata_db.sync()
log.msg('Fetched %s info for %s' % (self.id, obj_id))
self.currently_fetching = False
def get_obj(self, obj_id):
result = metadata_db.get(self.keyify('m', obj_id), {})
result['href'] = self.get_url(obj_id)
if result.get('cover', None):
result['cover'] = self.get_cover_url(result['cover'])
return result
