def sync_files(self): if os.path.exists(self.path): files = [wm_unicode(f) for f in os.listdir(self.path)] else: os.mkdir(self.path, 0777) os.chmod(self.path, 0777) files = [] files_added = [] if not any(u'.torrent' in f for f in files): files_added.append(u'torrent') torrent_path = os.path.join(wm_str(self.path), wm_str(self.what_torrent.torrent_file_name)) with open(torrent_path, 'wb') as file: file.write(self.what_torrent.torrent_file_binary) os.chmod(torrent_path, 0777) if not any(u'ReleaseInfo2.txt' == f for f in files): files_added.append(u'ReleaseInfo2.txt') release_info_path = os.path.join(self.path.encode('utf-8'), u'ReleaseInfo2.txt') with open(release_info_path.decode('utf-8'), 'w') as file: file.write(self.what_torrent.info) os.chmod(os.path.join(release_info_path.decode('utf-8')), 0777) if files_added: LogEntry.add(None, u'info', u'Added files {0} to {1}' .format(', '.join(files_added), self))
def get_metadata_batch(cls, what_torrent, trans_torrent, force_update): torrent_path = trans_torrent.path cache_lines = list(what_torrent.whatfilemetadatacache_set.all()) if len(cache_lines) and not force_update: for cache_line in cache_lines: cache_line.path = os.path.join(torrent_path, cache_line.filename) return sorted(cache_lines, key=lambda c: c.path) cache_lines = {c.filename_sha256: c for c in cache_lines} abs_rel_filenames = [] for dirpath, dirnames, filenames in os.walk(wm_str(torrent_path)): unicode_dirpath = wm_unicode(dirpath) unicode_filenames = [wm_unicode(f) for f in filenames] for filename in unicode_filenames: if os.path.splitext(filename)[1].lower() in [u'.flac', u'.mp3']: abs_path = os.path.join(unicode_dirpath, filename) rel_path = os.path.relpath(abs_path, torrent_path) abs_rel_filenames.append((abs_path, rel_path)) abs_rel_filenames.sort(key=lambda f: f[1]) filename_hashes = {f[0]: hashlib.sha256(wm_str(f[1])).hexdigest() for f in abs_rel_filenames} hash_set = set(filename_hashes.values()) old_cache_lines = [] for cache_line in cache_lines.itervalues(): if cache_line.filename_sha256 not in hash_set: old_cache_lines.append(cache_line) dirty_cache_lines = [] result = [] for abs_path, rel_path in abs_rel_filenames: try: file_mtime = os.path.getmtime(wm_str(abs_path)) cache = cache_lines.get(filename_hashes[abs_path]) if cache is None: cache = WhatFileMetadataCache( what_torrent=what_torrent, filename_sha256=filename_hashes[abs_path], filename=rel_path[:400], file_mtime=0 ) cache.path = abs_path if abs(file_mtime - cache.file_mtime) <= 1: result.append(cache) continue cache.fill(abs_path, file_mtime) dirty_cache_lines.append(cache) result.append(cache) except Exception as ex: print 'Failed:', abs_path, ex if old_cache_lines or dirty_cache_lines: with transaction.atomic(): for cache_line in old_cache_lines: cache_line.delete() for cache_line in dirty_cache_lines: cache_line.save() return result
def edit_upload(request, upload_id): qobuz_upload = QobuzUpload.objects.get(id=upload_id) if request.method == 'POST': form = EditUploadForm(request.POST, qobuz_upload=qobuz_upload) if form.is_valid(): qobuz_upload.artists = form.cleaned_data['artists'] qobuz_upload.album_name = form.cleaned_data['album_name'] tracks = [] for i, track in enumerate(qobuz_upload.track_data): qobuz_track = qobuz_upload.album_data['tracks']['items'][i] title_field = 'track_{0}_{1}_title'.format(qobuz_track['media_number'], i + 1) artists_field = 'track_{0}_{1}_artists'.format(qobuz_track['media_number'], i + 1) tracks.append({ 'media_number': qobuz_track['media_number'], 'artists': form.cleaned_data[artists_field], 'title': form.cleaned_data[title_field], }) qobuz_upload.track_data_json = ujson.dumps(tracks) qobuz_upload.save() else: form = EditUploadForm(qobuz_upload=qobuz_upload) download_error = '' if qobuz_upload.download_task_id: async_result = AsyncResult(qobuz_upload.download_task_id) if async_result.state == states.PENDING: download_status = 'Waiting to start' elif async_result.state == states.STARTED: download_status = 'Started' elif async_result.state == states.SUCCESS: download_status = 'Completed' elif async_result.state == states.FAILURE: download_status = 'Failed' download_error = '{0}: {1}'.format(type(async_result.result).__name__, async_result.result.message) else: download_status = 'Unknown Status' else: download_status = 'not_started' try: spectral_files = sorted(os.listdir(wm_str(qobuz_upload.spectrals_path))) except OSError: spectral_files = [] try: cover_files = sorted([f for f in os.listdir(wm_str(qobuz_upload.temp_media_path)) if f.endswith('.jpg')], reverse=True) except OSError: cover_files = [] data = { 'upload': qobuz_upload, 'form': form, 'download_status': download_status, 'download_error': download_error, 'spectral_files': spectral_files, 'cover_files': cover_files, } return render(request, 'qobuz/edit_upload.html', data)
def subfolder_move(self, subfolder, torrent_id): if not self.error_move: print u'Skipping {}..'.format(torrent_id) return print u'Moving {} to {} subfolder..'.format(torrent_id, subfolder) s = wm_str(os.path.join(self.wm_media, torrent_id)) # source d = wm_str(os.path.join(self.wm_media, subfolder, torrent_id)) # dest safe_makedirs(os.path.dirname(d)) try: shutil.move(s, d) except Exception as e: shutil.rmtree(d) raise e
def move_files(self): print u'Moving files to new directory...' if 'files' in self.torrent_info['info']: for f in self.get_unicode_torrent_files(): f_path = os.path.join(self.data_path, *f['path']) f_dest_path = os.path.join(self.dest_path, *f['path']) safe_makedirs(os.path.dirname(f_dest_path)) shutil.move(wm_str(f_path), wm_str(f_dest_path)) else: f_path = os.path.join(self.data_path, wm_unicode(self.torrent_info['info']['name'])) f_dest_path = os.path.join(self.dest_path, wm_unicode( self.torrent_info['info']['name'])) safe_makedirs(os.path.dirname(f_dest_path)) shutil.move(wm_str(f_path), wm_str(f_dest_path))
def find_existing_torrent_group(self): if self.existing_new_group is not None: return existing_group_id = None group_id = self.what_torrent_info['group']['id'] try: mapping = TorrentGroupMapping.objects.get(what_group_id=group_id) mapping_group = self.what.request('torrentgroup', id=mapping.pth_group_id)['response'] if mapping_group['group']['id'] != mapping.pth_group_id: raise Exception('NOOOOO THIS CANNOT HAPPEN {} {}!'.format( mapping_group['group']['id'], mapping.pth_group_id, )) existing_group_id = mapping.pth_group_id print 'Found torrent group mapping with {}'.format(existing_group_id) except TorrentGroupMapping.DoesNotExist: pass except BadIdException: print 'Mapping has gone bad, deleting...' mapping.delete() if existing_group_id is None: group_year = self.what_torrent_info['group']['year'] group_name = html_parser.unescape(self.what_torrent_info['group']['name']).lower() search_str = '{} {}'.format(wm_str(group_name), wm_str(str(group_year))) results = self.what.request('browse', searchstr=search_str)['response']['results'] for result in results: if html_parser.unescape(result['groupName']).lower() == group_name and \ result['groupYear'] == group_year: if not existing_group_id: existing_group_id = result['groupId'] print 'Found existing group', existing_group_id else: print 'Multiple matching existing groups!!!!!!!!!!' existing_group_id = None break if existing_group_id is None: existing_group_id = raw_input(u'Enter existing group id (empty if non-existent): ') if existing_group_id: TorrentGroupMapping.objects.get_or_create( what_group_id=self.what_torrent_info['group']['id'], pth_group_id=existing_group_id ) if existing_group_id: self.existing_new_group = self.existing_new_group = self.what.request( 'torrentgroup', id=existing_group_id)['response']
def __init__(self, what, location_mapping, data, flac_only): self.flac_only = flac_only self.what = what self.location_mapping = location_mapping self.data = data self.what_torrent = self.data['what_torrent'] self.what_torrent_info = ujson.loads(self.what_torrent['info']) self.full_location = os.path.join( wm_str(self.data['location']['path']), str(self.what_torrent['id']), ) self.torrent_dict = bencode.bdecode( b64decode(self.what_torrent['torrent_file'])) self.torrent_name = self.torrent_dict['info']['name'] self.torrent_new_name = self.torrent_name self.torrent_dir_path = os.path.join( self.full_location.encode('utf-8'), self.torrent_name) self.new_torrent = None self.log_files = set() self.log_files_full_paths = [] self.torrent_file_new_data = None self.torrent_new_infohash = None self.payload = None self.payload_files = None self.existing_new_group = None self.full_new_location = None
def __init__(self, what, location_mapping, data, flac_only): self.flac_only = flac_only self.what = what self.location_mapping = location_mapping self.data = data self.what_torrent = self.data['what_torrent'] self.what_torrent_info = ujson.loads(self.what_torrent['info']) self.full_location = os.path.join( wm_str(self.data['location']['path']), str(self.what_torrent['id']), ) self.torrent_dict = bencode.bdecode(b64decode(self.what_torrent['torrent_file'])) self.torrent_name = self.torrent_dict['info']['name'] self.torrent_new_name = self.torrent_name self.torrent_dir_path = os.path.join(self.full_location.encode('utf-8'), self.torrent_name) self.new_torrent = None self.log_files = set() self.log_files_full_paths = [] self.torrent_file_new_data = None self.torrent_new_infohash = None self.payload = None self.payload_files = None self.existing_new_group = None self.full_new_location = None
def handle(self, *args, **options): if not self.check_args(args): print u'Pass the torrent data directory as a first argument, ' \ u'a path to the .torrent file as a second.' return self.data_path, self.torrent_path = [wm_unicode(i) for i in args] with open(wm_str(self.torrent_path), 'rb') as f: self.torrent_info = bencode.bdecode(f.read()) if options['base_dir']: self.data_path = os.path.join(self.data_path, wm_unicode(self.torrent_info['info']['name'])) print u'Checking to see if torrent is already loaded into WM..' masters = list(ReplicaSet.get_what_master().transinstance_set.all()) try: TransTorrent.objects.get(instance__in=masters, info_hash=self.info_hash) print u'Torrent already added to WM. Skipping...' return False except TransTorrent.DoesNotExist: pass self.what_torrent = WhatTorrent.get_or_create(self.pseudo_request, info_hash=self.info_hash) if not self.check_files(): return self.move_files() print 'Adding torrent to WM...' manage_torrent.add_torrent(self.pseudo_request, self.trans_instance, self.download_location, self.what_torrent.id) print 'Done!'
def upload_cover_to_whatimg(request, upload_id): qobuz_upload = QobuzUpload.objects.get(id=upload_id) with open(os.path.join(wm_str(qobuz_upload.temp_media_path), 'folder.jpg'), 'rb') as f: data = f.read() qobuz_upload.what_img_cover = whatimg.upload_image_from_memory('4460', data) qobuz_upload.save() return redirect(edit_upload, upload_id)
def download_torrent_file(request, upload_id): qobuz_upload = QobuzUpload.objects.get(id=upload_id) file_name = qobuz_upload.torrent_name + '.torrent' file_path = os.path.join(qobuz_upload.temp_media_path, file_name) f = open(wm_str(file_path), 'rb') response = HttpResponse(f, content_type='application/x-bittorrent') response['Content-Disposition'] = 'attachment; filename="' + file_name + '"' return response
def view_spectral(request, upload_id): try: qobuz_upload = QobuzUpload.objects.get(id=upload_id) path = os.path.join(qobuz_upload.spectrals_path, os.path.basename(request.GET['path'])) f = open(wm_str(path), 'rb') return HttpResponse(f, content_type='image/jpeg') except Exception: return HttpResponseNotFound()
def inner(request, upload_id): try: qobuz_upload = QobuzUpload.objects.get(id=upload_id) path = os.path.join(getattr(qobuz_upload, prop), os.path.basename(request.GET['path'])) s = os.path.getmtime(wm_str(path)) return datetime.datetime.utcfromtimestamp(s) except Exception: return None
def move_files(self): print u'Moving files to new directory...' if 'files' in self.torrent_info['info']: for f in self.get_unicode_torrent_files(): f_path = os.path.join(self.data_path, *f['path']) f_dest_path = os.path.join(self.dest_path, *f['path']) safe_makedirs(os.path.dirname(f_dest_path)) shutil.move(wm_str(f_path), wm_str(f_dest_path)) else: f_path = os.path.join( self.data_path, wm_unicode(self.torrent_info['info']['name'])) f_dest_path = os.path.join( self.dest_path, wm_unicode(self.torrent_info['info']['name'])) safe_makedirs(os.path.dirname(f_dest_path)) shutil.move(wm_str(f_path), wm_str(f_dest_path)) print u'Success!' self.subfolder_move('imported', self.torrent_id)
def download_torrent_file(request, upload_id): qobuz_upload = QobuzUpload.objects.get(id=upload_id) file_name = qobuz_upload.torrent_name + '.torrent' file_path = os.path.join(qobuz_upload.temp_media_path, file_name) f = open(wm_str(file_path), 'rb') response = HttpResponse(f, content_type='application/x-bittorrent') response[ 'Content-Disposition'] = 'attachment; filename="' + file_name + '"' return response
def view_cover(request, upload_id): try: qobuz_upload = QobuzUpload.objects.get(id=upload_id) dest_path = os.path.join(get_temp_dir(qobuz_upload.upload.metadata.id), os.path.basename(request.GET['path'])) f = open(wm_str(dest_path), 'rb') return HttpResponse(f, content_type='image/jpeg') except Exception: return HttpResponseNotFound()
def upload_cover_to_whatimg(request, upload_id): qobuz_upload = QobuzUpload.objects.get(id=upload_id) with open(os.path.join(wm_str(qobuz_upload.temp_media_path), 'folder.jpg'), 'rb') as f: data = f.read() qobuz_upload.what_img_cover = whatimg.upload_image_from_memory( '4460', data) qobuz_upload.save() return redirect(edit_upload, upload_id)
def seed_upload(request, upload_id): qobuz_upload = QobuzUpload.objects.get(id=upload_id) temp_dir = get_temp_dir(qobuz_upload.upload.metadata.id) torrent_path = os.path.join(temp_dir, qobuz_upload.upload.metadata.torrent_name + '.torrent') assert os.path.isfile(wm_str(torrent_path)) info_hash = get_info_hash(torrent_path) what_torrent = WhatTorrent.get_or_create(request, info_hash=info_hash) command = import_external_what_torrent.Command() command.handle(wm_str(temp_dir), wm_str(torrent_path), base_dir=False) try: run_request_transcode(request, what_torrent.id) except Exception: pass qiller = qobuz_upload.upload qiller.state = STATE_DONE qobuz_upload.set_upload(qiller) qobuz_upload.save() return redirect(edit_upload, upload_id)
def inner(request, upload_id): try: qobuz_upload = QobuzUpload.objects.get(id=upload_id) dest_path = os.path.join(get_temp_dir(qobuz_upload.upload.metadata.id), subpath, os.path.basename(request.GET['path'])) s = os.path.getmtime(wm_str(dest_path)) return datetime.datetime.utcfromtimestamp(s) except Exception: raise return None
def fill(self, filename, file_mtime): metadata = mutagen.File(wm_str(filename), easy=True) if hasattr(metadata, 'pictures'): for p in metadata.pictures: p.data = None if hasattr(metadata, 'tags'): if hasattr(metadata.tags, '_EasyID3__id3'): metadata.tags._EasyID3__id3.delall('APIC') self.file_mtime = file_mtime try:
def fill(self, filename, file_mtime): metadata = mutagen.File(wm_str(filename), easy=True) if hasattr(metadata, 'pictures'): for p in metadata.pictures: p.data = None if hasattr(metadata, 'tags'): if hasattr(metadata.tags, '_EasyID3__id3'): metadata.tags._EasyID3__id3.delall('APIC') self.file_mtime = file_mtime self.metadata_pickle = pickle.dumps(metadata) self.artists = self.easy['artist'][:200] self.album = self.easy['album'][:200] self.title = self.easy['title'][:200] self.duration = self.easy['duration']
def handle(self, *args, **options): if not self.check_args(args): print u'Pass the torrent data directory as a first argument, ' \ u'a path to the .torrent file as a second.' return self.data_path, self.torrent_path = [wm_unicode(i) for i in args] with open(wm_str(self.torrent_path), 'rb') as f: self.torrent_info = bencode.bdecode(f.read()) if options['base_dir']: self.data_path = os.path.join(self.data_path, wm_unicode(self.torrent_info['info']['name'])) self.what_torrent = WhatTorrent.get_or_create(self.pseudo_request, info_hash=self.info_hash) if not self.check_files(): return self.move_files() print 'Adding torrent to WM...' manage_torrent.add_torrent(self.pseudo_request, self.trans_instance, self.download_location, self.what_torrent.id) print 'Done!'
def start_seeding(request, upload_id): qobuz_upload = QobuzUpload.objects.get(id=upload_id) dest_upload_dir = DownloadLocation.get_what_preferred().path torrent_file_path = os.path.join(qobuz_upload.temp_media_path, qobuz_upload.torrent_name + u'.torrent') info_hash = get_info_hash(torrent_file_path) what_torrent = WhatTorrent.get_or_create(request, info_hash=info_hash) qobuz_upload.what_torrent = what_torrent qobuz_upload.save() dest_path = os.path.join(dest_upload_dir, str(what_torrent.id)) shutil.rmtree(wm_str(qobuz_upload.spectrals_path)) os.remove(wm_str(torrent_file_path)) try: os.makedirs(wm_str(dest_path)) except OSError: raise Exception('Dest torrent directory already exists.') os.chmod(wm_str(dest_path), 0777) shutil.move(wm_str(qobuz_upload.temp_media_path), wm_str(dest_path)) add_to_wm_transcode(str(what_torrent.id)) return redirect(edit_upload, upload_id)
def safe_makedirs(p): try: os.makedirs(wm_str(p)) except OSError as ex: if ex.errno != errno.EEXIST: raise ex
def get_metadata_batch(cls, what_torrent, trans_torrent, force_update): torrent_path = trans_torrent.path cache_lines = list(what_torrent.whatfilemetadatacache_set.all()) if len(cache_lines) and not force_update: for cache_line in cache_lines: cache_line.path = os.path.join(torrent_path, cache_line.filename) return sorted(cache_lines, key=lambda c: c.path) cache_lines = {c.filename_sha256: c for c in cache_lines} abs_rel_filenames = [] for dirpath, dirnames, filenames in os.walk(wm_str(torrent_path)): unicode_dirpath = wm_unicode(dirpath) unicode_filenames = [wm_unicode(f) for f in filenames] for filename in unicode_filenames: if os.path.splitext(filename)[1].lower() in [ u'.flac', u'.mp3' ]: abs_path = os.path.join(unicode_dirpath, filename) rel_path = os.path.relpath(abs_path, torrent_path) abs_rel_filenames.append((abs_path, rel_path)) abs_rel_filenames.sort(key=lambda f: f[1]) filename_hashes = { f[0]: hashlib.sha256(wm_str(f[1])).hexdigest() for f in abs_rel_filenames } hash_set = set(filename_hashes.values()) old_cache_lines = [] for cache_line in cache_lines.itervalues(): if cache_line.filename_sha256 not in hash_set: old_cache_lines.append(cache_line) dirty_cache_lines = [] result = [] for abs_path, rel_path in abs_rel_filenames: try: file_mtime = os.path.getmtime(wm_str(abs_path)) cache = cache_lines.get(filename_hashes[abs_path]) if cache is None: cache = WhatFileMetadataCache( what_torrent=what_torrent, filename_sha256=filename_hashes[abs_path], filename=rel_path[:400], file_mtime=0) cache.path = abs_path if abs(file_mtime - cache.file_mtime) <= 1: result.append(cache) continue cache.fill(abs_path, file_mtime) dirty_cache_lines.append(cache) result.append(cache) except Exception as ex: print 'Failed:', abs_path, ex if old_cache_lines or dirty_cache_lines: with transaction.atomic(): for cache_line in old_cache_lines: cache_line.delete() for cache_line in dirty_cache_lines: cache_line.save() return result
def view_cover(request, upload_id): qobuz_upload = QobuzUpload.objects.get(id=upload_id) path = os.path.join(qobuz_upload.temp_media_path, os.path.basename(request.GET['path'])) f = open(wm_str(path), 'rb') return HttpResponse(f, content_type='image/jpeg')
def handle(self, *args, **options): if not self.check_args(args): print u'Pass the directory containing your torrent directories from a previous WM' \ u' install. Subfolders of this directory should be named by torrent ID. After' \ u' import, all errored torrent/data sets will be organized into subfolders for' \ u' manual inspection/import.' return self.wm_media = wm_unicode(args[0]) self.error_move = not options['no_move'] for self.torrent_id in next(os.walk(self.wm_media))[1]: try: # Is this actually a directory? if not os.path.isdir(self.base_dir()): print u'"{}" is not a valid directory. Skipping..'.format(self.base_dir()) continue # Get all torrents torrents = [] hashes = [] for p in os.listdir(self.base_dir()): if p.endswith('.torrent') and not p.startswith('._'): try: p = os.path.join(self.base_dir(), wm_unicode(p)) hashes.append(get_info_hash(p)) torrents.append(p) except IOError: print('Warning: Invalid torrent found in {}'.format(self.torrent_id)) continue except BTFailure as e: print('Warning: {}. Invalid torrent found in {}'.format(str(e), self.torrent_id)) continue # Are there any valid torrents? if len(torrents) == 0: if self.torrent_id.isdigit(): print u'Error: No valid torrent files found in "{}".'.format(self.base_dir()) self.subfolder_move('no_torrents', self.torrent_id) continue # Are there multiple unique torrents? if len(set(hashes)) > 1: print u'Error: Multiple unique torrents found' self.subfolder_move('multiple_torrent', self.torrent_id) continue except UnicodeDecodeError as e: print u'UnicodeDecodeError: Please import manually. Skipping..' continue with open(wm_str(torrents[0]), 'rb') as f: try: self.torrent_info = bencode.bdecode(f.read()) self.info_hash = get_info_hash(torrents[0]) except: print u'Error: Invalid torrent file.' self.subfolder_move('invalid_torrent', self.torrent_id) continue self.data_path = os.path.join(self.base_dir(), wm_unicode(self.torrent_info['info']['name'])) print u'Checking to see if torrent is already loaded into WM..' masters = list(ReplicaSet.get_what_master().transinstance_set.all()) try: TransTorrent.objects.get(instance__in=masters, info_hash=self.info_hash) print u'Error: Torrent already added to WM.' self.subfolder_move('already_added', self.torrent_id) continue except TransTorrent.DoesNotExist: pass try: self.what_torrent = WhatTorrent.get_or_create(self.pseudo_request, info_hash=self.info_hash) except RequestException as e: if 'bad hash' in str(e): print u'Error: Bad hash. Torrent may have been trumped/deleted.'.format(str(e)) self.subfolder_move('bad_hash', self.torrent_id) continue else: raise e except OperationalError as e: if 'MySQL' in str(e): print u'Error: {}. Please check {} manually.'.format(str(e), self.torrent_id) self.subfolder_move('mysql_error', self.torrent_id) continue else: raise e if not self.check_files(): print u'Error: File check failed.' try: self.subfolder_move('file_check_fail', self.torrent_id) except UnicodeDecodeError as e: print u'UnicodeDecodeError. Move failed. Please manually check {} Skipping..'.format(self.torrent_id) continue self.move_files() print u'Adding torrent to WM...' self.trans_instance = ReplicaSet.get_what_master().get_preferred_instance() manage_torrent.add_torrent(self.pseudo_request, self.trans_instance, self.download_location, self.what_torrent.id) print u'Done!'
def process(self): what_torrent_id = self.what_torrent['id'] if self.what_torrent_info['group']['categoryName'] != 'Music': print 'Skipping non-Music torrent', what_torrent_id return if self.flac_only and self.what_torrent_info['torrent'][ 'format'] != 'FLAC': print 'Skipping non-FLAC torrent', what_torrent_id return try: status = WhatTorrentMigrationStatus.objects.get( what_torrent_id=what_torrent_id) if status.status == WhatTorrentMigrationStatus.STATUS_COMPLETE: print 'Skipping complete torrent', what_torrent_id return elif status.status == WhatTorrentMigrationStatus.STATUS_DUPLICATE: print 'Skipping duplicate torrent', what_torrent_id return elif status.status == WhatTorrentMigrationStatus.STATUS_SKIPPED: print 'Skipping skipped torrent', what_torrent_id return elif status.status == WhatTorrentMigrationStatus.STATUS_SKIPPED_PERMANENTLY: print 'Skipping permanently skipped torrent', what_torrent_id return elif status.status == WhatTorrentMigrationStatus.STATUS_FAILED_VALIDATION: print 'Skipping failed validation torrent', what_torrent_id return elif status.status == WhatTorrentMigrationStatus.STATUS_RESEEDED: print 'Skipping reseeded torrent', what_torrent_id return else: raise Exception( 'Not sure what to do with status {} on {}'.format( status.status, what_torrent_id)) except WhatTorrentMigrationStatus.DoesNotExist: pass if not self.check_valid(): return self.mktorrent() if not self.find_dupes(): return if not self.new_torrent: self.enhance_torrent_data() self.prepare_payload() self.print_info() self.prepare_payload_files() self.generate_spectrals() raw_input('Will perform upload (CHECK THE SPECTRALS)...') self.perform_upload() self.set_new_location() if self.REAL_RUN: os.makedirs(self.full_new_location) shutil.move(wm_str(self.torrent_dir_path), wm_str(self.full_new_location)) try: recursive_chmod(self.full_new_location, 0777) except OSError: print 'recursive_chmod failed' else: print 'os.makedirs({})'.format(self.full_new_location) print 'shutil.move({}, {})'.format(self.torrent_dir_path, self.full_new_location) print 'recursive_chmod({}, 0777)'.format(self.full_new_location) if self.REAL_RUN: self.add_to_wm() self.migration_status.status = WhatTorrentMigrationStatus.STATUS_COMPLETE self.migration_status.save() self.save_torrent_group_mapping() else: print 'add_to_wm()' print print
def process(self): what_torrent_id = self.what_torrent['id'] if self.what_torrent_info['group']['categoryName'] != 'Music': print 'Skipping non-Music torrent', what_torrent_id return if self.flac_only and self.what_torrent_info['torrent']['format'] != 'FLAC': print 'Skipping non-FLAC torrent', what_torrent_id return try: status = WhatTorrentMigrationStatus.objects.get(what_torrent_id=what_torrent_id) if status.status == WhatTorrentMigrationStatus.STATUS_COMPLETE: print 'Skipping complete torrent', what_torrent_id return elif status.status == WhatTorrentMigrationStatus.STATUS_DUPLICATE: print 'Skipping duplicate torrent', what_torrent_id return elif status.status == WhatTorrentMigrationStatus.STATUS_SKIPPED: print 'Skipping skipped torrent', what_torrent_id return elif status.status == WhatTorrentMigrationStatus.STATUS_SKIPPED_PERMANENTLY: print 'Skipping permanently skipped torrent', what_torrent_id return elif status.status == WhatTorrentMigrationStatus.STATUS_FAILED_VALIDATION: print 'Skipping failed validation torrent', what_torrent_id return elif status.status == WhatTorrentMigrationStatus.STATUS_RESEEDED: print 'Skipping reseeded torrent', what_torrent_id return else: raise Exception('Not sure what to do with status {} on {}'.format( status.status, what_torrent_id)) except WhatTorrentMigrationStatus.DoesNotExist: pass if not self.check_valid(): return self.mktorrent() if not self.find_dupes(): return if not self.new_torrent: self.enhance_torrent_data() self.prepare_payload() self.print_info() self.prepare_payload_files() self.generate_spectrals() raw_input('Will perform upload (CHECK THE SPECTRALS)...') self.perform_upload() self.set_new_location() if self.REAL_RUN: os.makedirs(self.full_new_location) shutil.move(wm_str(self.torrent_dir_path), wm_str(self.full_new_location)) try: recursive_chmod(self.full_new_location, 0777) except OSError: print 'recursive_chmod failed' else: print 'os.makedirs({})'.format(self.full_new_location) print 'shutil.move({}, {})'.format(self.torrent_dir_path, self.full_new_location) print 'recursive_chmod({}, 0777)'.format(self.full_new_location) if self.REAL_RUN: self.add_to_wm() self.migration_status.status = WhatTorrentMigrationStatus.STATUS_COMPLETE self.migration_status.save() self.save_torrent_group_mapping() else: print 'add_to_wm()' print print
def handle(self, *args, **options): if not self.check_args(args): print u'Pass the directory containing your torrent directories from a previous WM' \ u' install. Subfolders of this directory should be named by torrent ID. After' \ u' import, all errored torrent/data sets will be organized into subfolders for' \ u' manual inspection/import.' return self.wm_media = wm_unicode(args[0]) self.error_move = not options['no_move'] for self.torrent_id in next(os.walk(self.wm_media))[1]: try: # Is this actually a directory? if not os.path.isdir(self.base_dir()): print u'"{}" is not a valid directory. Skipping..'.format( self.base_dir()) continue # Get all torrents torrents = [] hashes = [] for p in os.listdir(self.base_dir()): if p.endswith('.torrent') and not p.startswith('._'): try: p = os.path.join(self.base_dir(), wm_unicode(p)) hashes.append(get_info_hash(p)) torrents.append(p) except IOError: print( 'Warning: Invalid torrent found in {}'.format( self.torrent_id)) continue except BTFailure as e: print('Warning: {}. Invalid torrent found in {}'. format(str(e), self.torrent_id)) continue # Are there any valid torrents? if len(torrents) == 0: if self.torrent_id.isdigit(): print u'Error: No valid torrent files found in "{}".'.format( self.base_dir()) self.subfolder_move('no_torrents', self.torrent_id) continue # Are there multiple unique torrents? if len(set(hashes)) > 1: print u'Error: Multiple unique torrents found' self.subfolder_move('multiple_torrent', self.torrent_id) continue except UnicodeDecodeError as e: print u'UnicodeDecodeError: Please import manually. Skipping..' continue with open(wm_str(torrents[0]), 'rb') as f: try: self.torrent_info = bencode.bdecode(f.read()) self.info_hash = get_info_hash(torrents[0]) except: print u'Error: Invalid torrent file.' self.subfolder_move('invalid_torrent', self.torrent_id) continue self.data_path = os.path.join( self.base_dir(), wm_unicode(self.torrent_info['info']['name'])) print u'Checking to see if torrent is already loaded into WM..' masters = list( ReplicaSet.get_what_master().transinstance_set.all()) try: TransTorrent.objects.get(instance__in=masters, info_hash=self.info_hash) print u'Error: Torrent already added to WM.' self.subfolder_move('already_added', self.torrent_id) continue except TransTorrent.DoesNotExist: pass try: self.what_torrent = WhatTorrent.get_or_create( self.pseudo_request, info_hash=self.info_hash) except RequestException as e: if 'bad hash' in str(e): print u'Error: Bad hash. Torrent may have been trumped/deleted.'.format( str(e)) self.subfolder_move('bad_hash', self.torrent_id) continue else: raise e except OperationalError as e: if 'MySQL' in str(e): print u'Error: {}. Please check {} manually.'.format( str(e), self.torrent_id) self.subfolder_move('mysql_error', self.torrent_id) continue else: raise e if not self.check_files(): print u'Error: File check failed.' try: self.subfolder_move('file_check_fail', self.torrent_id) except UnicodeDecodeError as e: print u'UnicodeDecodeError. Move failed. Please manually check {} Skipping..'.format( self.torrent_id) continue self.move_files() print u'Adding torrent to WM...' self.trans_instance = ReplicaSet.get_what_master( ).get_preferred_instance() manage_torrent.add_torrent(self.pseudo_request, self.trans_instance, self.download_location, self.what_torrent.id) print u'Done!'
def check_files(self): print u'Checking for existing files...' if 'files' in self.torrent_info['info']: for f in self.get_unicode_torrent_files(): f_path = os.path.join(self.data_path, *f['path']) print wm_str(u'Checking {0}'.format(f_path)) if not os.path.isfile(wm_str(f_path)): print wm_str(u'{0} does not exist. What are you giving me?'.format(f_path)) return False else: f_path = os.path.join(self.data_path, self.torrent_info['info']['name']) print wm_str(u'Checking {0}'.format(f_path)) if not os.path.isfile(wm_str(f_path)): print wm_str(u'{0} does not exist. What are you giving me?'.format(f_path)) return False print u'Creating destination directory...' self.dest_path = os.path.join(self.download_location.path, unicode(self.what_torrent.id)) os.makedirs(wm_str(self.dest_path)) os.chmod(self.dest_path, 0777) if 'files' in self.torrent_info['info']: self.dest_path = os.path.join(self.dest_path, wm_unicode( self.torrent_info['info']['name'])) os.makedirs(wm_str(self.dest_path)) print u'All torrent data files exist.' return True
def edit_upload(request, upload_id): qobuz_upload = QobuzUpload.objects.get(id=upload_id) if request.method == 'POST': form = EditUploadForm(request.POST, qobuz_upload=qobuz_upload) if form.is_valid(): qobuz_upload.artists = form.cleaned_data['artists'] qobuz_upload.album_name = form.cleaned_data['album_name'] tracks = [] for i, track in enumerate(qobuz_upload.track_data): qobuz_track = qobuz_upload.album_data['tracks']['items'][i] title_field = 'track_{0}_{1}_title'.format( qobuz_track['media_number'], i + 1) artists_field = 'track_{0}_{1}_artists'.format( qobuz_track['media_number'], i + 1) tracks.append({ 'media_number': qobuz_track['media_number'], 'artists': form.cleaned_data[artists_field], 'title': form.cleaned_data[title_field], }) qobuz_upload.track_data_json = ujson.dumps(tracks) qobuz_upload.save() else: form = EditUploadForm(qobuz_upload=qobuz_upload) download_error = '' if qobuz_upload.download_task_id: async_result = AsyncResult(qobuz_upload.download_task_id) if async_result.state == states.PENDING: download_status = 'Waiting to start' elif async_result.state == states.STARTED: download_status = 'Started' elif async_result.state == states.SUCCESS: download_status = 'Completed' elif async_result.state == states.FAILURE: download_status = 'Failed' download_error = '{0}: {1}'.format( type(async_result.result).__name__, async_result.result.message) else: download_status = 'Unknown Status' else: download_status = 'not_started' try: spectral_files = sorted(os.listdir(wm_str( qobuz_upload.spectrals_path))) except OSError: spectral_files = [] try: cover_files = sorted([ f for f in os.listdir(wm_str(qobuz_upload.temp_media_path)) if f.endswith('.jpg') ], reverse=True) except OSError: cover_files = [] data = { 'upload': qobuz_upload, 'form': form, 'download_status': download_status, 'download_error': download_error, 'spectral_files': spectral_files, 'cover_files': cover_files, } return render(request, 'qobuz/edit_upload.html', data)
def get_info_hash(torrent_path): with open(wm_str(torrent_path), 'rb') as torrent_file: return get_info_hash_from_data(torrent_file.read())