def redacted_archiver_fetch_missing():
start = time.time()
state = RedactedArchiverState.objects.get()
if not state.is_fetch_missing_enabled:
return
client = RedactedClient()
tracker = TrackerRegistry.get_plugin(RedactedTrackerPlugin.name,
'redacted_archiver_fetch_missing')
realm = Realm.objects.get(name=tracker.name)
num_fetched = 0
not_found = set()
while True:
torrent_missing_info = Torrent.objects.filter(
~Q(info_hash__in=not_found),
realm=realm,
torrent_info=None,
).first()
if torrent_missing_info is None:
break
try:
red_data = client.get_torrent_by_info_hash(
torrent_missing_info.info_hash)
except RedactedTorrentNotFoundException:
not_found.add(torrent_missing_info.info_hash)
LogEntry.warning(
'Unable to fetch missing metadata for Redacted torrent {} with info hash {} '
'downloaded in {} in managed {}. Please check and/or remove from client.'
.format(
torrent_missing_info.name,
torrent_missing_info.info_hash,
torrent_missing_info.download_path,
torrent_missing_info.client,
))
continue
tracker_id = str(red_data['torrent']['id'])
fetch_torrent(realm, tracker, str(tracker_id))
num_fetched += 1
allowed_time = (settings.REDACTED_ARCHIVER_FETCH_MISSING_INTERVAL -
settings.REDACTED_ARCHIVER_FETCH_MISSING_SLEEP - 3)
if time.time() - start >= allowed_time:
break
time.sleep(settings.REDACTED_ARCHIVER_FETCH_MISSING_SLEEP)
remaining_missing = Torrent.objects.filter(realm=realm,
torrent_info=None).count()
time_taken = time.time() - start
ComponentStatus.update_status(
'redacted_archiver_fetch_missing',
ComponentStatus.STATUS_GREEN,
'Completed Redacted fetch missing run. Fetched {} torrents in {:.3f} s. Not found: {}. Remaining: {}.'
.format(num_fetched, time_taken, len(not_found), remaining_missing),
)
def handle(self, *args, **options):
self.request_cache = RedactedRequestCache()
self.tracker = TrackerRegistry.get_plugin(RedactedTrackerPlugin.name)
self.realm = Realm.objects.get(name=self.tracker.name)
if options[TRANSCODE_TYPE_REDBOOK_FLAC]:
print('Scanning for Redbook FLAC transcodes...')
self._scan_torrents(
torrents=list(
Torrent.objects.filter(
realm=self.realm,
progress=1,
torrent_info__redacted_torrent__encoding=RedactedTorrent
.ENCODING_24BIT_LOSSLESS,
torrent_info__redacted_torrent__remaster_year__gt=0,
)),
transcode_type=TRANSCODE_TYPE_REDBOOK_FLAC,
match_fn=lambda t: t['encoding'] == 'Lossless',
auto_create=options['auto_create'],
)
if options[TRANSCODE_TYPE_MP3_V0]:
print('Scanning for MP3 V0 transcodes...')
self._scan_torrents(
torrents=list(
Torrent.objects.filter(
realm=self.realm,
progress=1,
torrent_info__redacted_torrent__format=RedactedTorrent.
FORMAT_FLAC,
torrent_info__redacted_torrent__remaster_year__gt=0,
)),
transcode_type=TRANSCODE_TYPE_MP3_V0,
match_fn=lambda t: t['encoding'] == 'V0 (VBR)',
auto_create=options['auto_create'],
)
if options[TRANSCODE_TYPE_MP3_320]:
print('Scanning for MP3 320 transcodes...')
self._scan_torrents(
torrents=list(
Torrent.objects.filter(
realm=self.realm,
progress=1,
torrent_info__redacted_torrent__format=RedactedTorrent.
FORMAT_FLAC,
torrent_info__redacted_torrent__remaster_year__gt=0,
)),
transcode_type=TRANSCODE_TYPE_MP3_320,
match_fn=lambda t: t['encoding'] == '320',
auto_create=options['auto_create'],
)
def bibliotik_archiver_download_torrent():
start = time.time()
state = BibliotikArchiverState.objects.get()
if not state.is_download_enabled:
return
bibliotik_torrent, num_remaining = get_bibliotik_torrent_for_archiving()
if not bibliotik_torrent:
logger.info('Bibliotik torrent download - nothing to download.')
return
tracker = TrackerRegistry.get_plugin('bibliotik',
'bibliotik_archive_download_torrent')
realm = Realm.objects.get(name=tracker.name)
download_location = realm.get_preferred_download_location()
if not download_location:
logger.error('No download location for realm {}.', tracker.name)
return
tracker_id = bibliotik_torrent.torrent_info.tracker_id
torrent_info = fetch_torrent(realm, tracker, tracker_id)
if torrent_info.is_deleted:
logger.info('Bibliotik torrent {} already deleted.', tracker_id)
return
logger.info('Downloading Bibliotik torrent {}.', tracker_id)
add_torrent_from_tracker(
tracker=tracker,
tracker_id=tracker_id,
download_path_pattern=download_location.pattern,
force_fetch=False,
)
time_taken = time.time() - start
ComponentStatus.update_status(
'bibliotik_archiver_download',
ComponentStatus.STATUS_GREEN,
'Completed Bibliotik archiver download torrent run in {:.3f} s. Remaining: {}.'
.format(time_taken, num_remaining - 1),
)
def bibliotik_archiver_metadata():
start = time.time()
state = BibliotikArchiverState.objects.get()
if not state.is_metadata_enabled:
return
client = BibliotikClient()
tracker = TrackerRegistry.get_plugin(BibliotikTrackerPlugin.name,
'bibliotik_archiver_metadata')
realm = Realm.objects.get(name=tracker.name)
search_results = parse_search_results(client.search(''))
max_tracker_id = search_results[0]['tracker_id']
logger.info('Bibliotik max tracker id: {}.', max_tracker_id)
num_scraped = 0
# last_meta_tracker_id was the last one processed, so resume from the next.
for tracker_id in range(state.last_meta_tracker_id + 1,
max_tracker_id + 1):
try:
fetch_torrent(realm, tracker, tracker_id)
logger.info('Bibliotik torrent {} fetched.', tracker_id)
except BibliotikTorrentNotFoundException:
logger.info('Bibliotik torrent {} not found.', tracker_id)
state.last_meta_tracker_id = tracker_id
state.save(update_fields=('last_meta_tracker_id', ))
num_scraped += 1
allowed_time = (settings.BIBLIOTIK_ARCHIVER_METADATA_INTERVAL -
settings.BIBLIOTIK_ARCHIVER_METADATA_SLEEP - 4)
if time.time() - start >= allowed_time:
break
time.sleep(settings.BIBLIOTIK_ARCHIVER_METADATA_SLEEP)
time_taken = time.time() - start
ComponentStatus.update_status(
'bibliotik_archiver_metadata',
ComponentStatus.STATUS_GREEN,
'Completed Bibliotik archiver metadata run with {} torrents in {:.3f} s. Progress: {} / {}.'
.format(num_scraped, time_taken, state.last_meta_tracker_id,
max_tracker_id),
)
def ready(self):
super().ready()
from trackers.registry import TrackerRegistry
from plugins.redacted.tracker import RedactedTrackerPlugin
TrackerRegistry.register(RedactedTrackerPlugin())
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.client = RedactedClient()
self.tracker = TrackerRegistry.get_plugin(RedactedTrackerPlugin.name,
'redacted_uploader_step')
self.realm = Realm.objects.get(name=self.tracker.name)
def __init__(self):
self.client = RedactedClient()
self.tracker = TrackerRegistry.get_plugin(RedactedTrackerPlugin.name,
'redacted_refresh_metadata')
self.realm = Realm.objects.get(name=self.tracker.name)
def create_transcode_project(tracker_id, transcode_type):
if transcode_type not in TRANSCODE_TYPES:
raise APIException(
'Unknown transcode type. Supported types: {}'.format(
TRANSCODE_TYPES),
code=status.HTTP_400_BAD_REQUEST,
)
tracker = TrackerRegistry.get_plugin(RedactedTrackerPlugin.name,
'transcode_torrent')
realm = Realm.objects.get(name=RedactedTrackerPlugin.name)
download_location = realm.get_preferred_download_location()
if not download_location:
raise APIException(
'No download location available for realm {}'.format(realm.name),
code=status.HTTP_400_BAD_REQUEST,
)
torrent_info = fetch_torrent(
realm=realm,
tracker=tracker,
tracker_id=tracker_id,
force_fetch=True,
)
try:
torrent = torrent_info.torrent
except Torrent.DoesNotExist:
torrent = add_torrent_from_tracker(
tracker=tracker,
tracker_id=tracker_id,
download_path_pattern=download_location.pattern,
force_fetch=False,
)
torrent_group = torrent_info.redacted_torrent.torrent_group
project = Project.objects.create(
media_type=Project.MEDIA_TYPE_MUSIC,
project_type='redacted_transcode_{}'.format(transcode_type),
name='{} - {} ({})'.format(
get_shorter_joined_artists(torrent_group.music_info,
torrent_group.name),
torrent_group.name,
transcode_type,
),
source_torrent=torrent,
)
project.steps.append(
ProjectStep(executor_name=RedactedTorrentSourceExecutor.name, ))
project.steps.append(
ProjectStep(
executor_name=SoxProcessExecutor.name,
executor_kwargs={
'target_sample_rate':
SoxProcessExecutor.TARGET_SAMPLE_RATE_44100_OR_4800,
'target_bits_per_sample': 16,
'target_channels': 2,
},
))
if transcode_type in {TRANSCODE_TYPE_MP3_V0, TRANSCODE_TYPE_MP3_320}:
project.steps.append(
ProjectStep(
executor_name=LAMETranscoderExecutor.name,
executor_kwargs={
'bitrate': {
TRANSCODE_TYPE_MP3_V0: MusicMetadata.ENCODING_V0,
TRANSCODE_TYPE_MP3_320: MusicMetadata.ENCODING_320,
}[transcode_type],
},
))
project.steps.append(
ProjectStep(executor_name=RedactedCheckFileTags.name, ))
project.steps.append(
ProjectStep(
executor_name=CreateTorrentFileExecutor.name,
executor_kwargs={
'announce': RedactedClient().get_announce(),
'extra_info_keys': {
'source': 'RED',
}
},
))
project.steps.append(
ProjectStep(executor_name=RedactedUploadTranscodeExecutor.name, ))
project.steps.append(ProjectStep(
executor_name=FinishUploadExecutor.name, ))
project.save_steps()
# If the torrent is complete, launch it. Otherwise the torrent_finished receiver will start it when received.
if torrent.progress == 1:
project_run_all.delay(project.id)
return project
def ready(self):
super().ready()
from trackers.registry import TrackerRegistry
from plugins.bibliotik.tracker import BibliotikTrackerPlugin
TrackerRegistry.register(BibliotikTrackerPlugin())