Beispiel #1
0
    def sync_files(self):
        if os.path.exists(self.path):
            files = [wm_unicode(f) for f in os.listdir(self.path)]
        else:
            os.mkdir(self.path, 0777)
            os.chmod(self.path, 0777)
            files = []

        files_added = []
        if not any(u'.torrent' in f for f in files):
            files_added.append(u'torrent')
            torrent_path = os.path.join(wm_str(self.path),
                                        wm_str(self.what_torrent.torrent_file_name))
            with open(torrent_path, 'wb') as file:
                file.write(self.what_torrent.torrent_file_binary)
            os.chmod(torrent_path, 0777)
        if not any(u'ReleaseInfo2.txt' == f for f in files):
            files_added.append(u'ReleaseInfo2.txt')
            release_info_path = os.path.join(self.path.encode('utf-8'), u'ReleaseInfo2.txt')
            with open(release_info_path.decode('utf-8'), 'w') as file:
                file.write(self.what_torrent.info)
            os.chmod(os.path.join(release_info_path.decode('utf-8')), 0777)
        if files_added:
            LogEntry.add(None, u'info', u'Added files {0} to {1}'
                         .format(', '.join(files_added), self))
Beispiel #2
0
    def get_metadata_batch(cls, what_torrent, trans_torrent, force_update):
        torrent_path = trans_torrent.path
        cache_lines = list(what_torrent.whatfilemetadatacache_set.all())
        if len(cache_lines) and not force_update:
            for cache_line in cache_lines:
                cache_line.path = os.path.join(torrent_path, cache_line.filename)
            return sorted(cache_lines, key=lambda c: c.path)

        cache_lines = {c.filename_sha256: c for c in cache_lines}

        abs_rel_filenames = []
        for dirpath, dirnames, filenames in os.walk(wm_str(torrent_path)):
            unicode_dirpath = wm_unicode(dirpath)
            unicode_filenames = [wm_unicode(f) for f in filenames]
            for filename in unicode_filenames:
                if os.path.splitext(filename)[1].lower() in [u'.flac', u'.mp3']:
                    abs_path = os.path.join(unicode_dirpath, filename)
                    rel_path = os.path.relpath(abs_path, torrent_path)
                    abs_rel_filenames.append((abs_path, rel_path))
        abs_rel_filenames.sort(key=lambda f: f[1])

        filename_hashes = {f[0]: hashlib.sha256(wm_str(f[1])).hexdigest() for f in
                           abs_rel_filenames}
        hash_set = set(filename_hashes.values())
        old_cache_lines = []
        for cache_line in cache_lines.itervalues():
            if cache_line.filename_sha256 not in hash_set:
                old_cache_lines.append(cache_line)
        dirty_cache_lines = []

        result = []
        for abs_path, rel_path in abs_rel_filenames:
            try:
                file_mtime = os.path.getmtime(wm_str(abs_path))
                cache = cache_lines.get(filename_hashes[abs_path])
                if cache is None:
                    cache = WhatFileMetadataCache(
                        what_torrent=what_torrent,
                        filename_sha256=filename_hashes[abs_path],
                        filename=rel_path[:400],
                        file_mtime=0
                    )
                cache.path = abs_path
                if abs(file_mtime - cache.file_mtime) <= 1:
                    result.append(cache)
                    continue
                cache.fill(abs_path, file_mtime)
                dirty_cache_lines.append(cache)
                result.append(cache)
            except Exception as ex:
                print 'Failed:', abs_path, ex
        if old_cache_lines or dirty_cache_lines:
            with transaction.atomic():
                for cache_line in old_cache_lines:
                    cache_line.delete()
                for cache_line in dirty_cache_lines:
                    cache_line.save()
        return result
Beispiel #3
0
def edit_upload(request, upload_id):
    qobuz_upload = QobuzUpload.objects.get(id=upload_id)
    if request.method == 'POST':
        form = EditUploadForm(request.POST, qobuz_upload=qobuz_upload)
        if form.is_valid():
            qobuz_upload.artists = form.cleaned_data['artists']
            qobuz_upload.album_name = form.cleaned_data['album_name']
            tracks = []
            for i, track in enumerate(qobuz_upload.track_data):
                qobuz_track = qobuz_upload.album_data['tracks']['items'][i]
                title_field = 'track_{0}_{1}_title'.format(qobuz_track['media_number'], i + 1)
                artists_field = 'track_{0}_{1}_artists'.format(qobuz_track['media_number'], i + 1)
                tracks.append({
                    'media_number': qobuz_track['media_number'],
                    'artists': form.cleaned_data[artists_field],
                    'title': form.cleaned_data[title_field],
                })
            qobuz_upload.track_data_json = ujson.dumps(tracks)
            qobuz_upload.save()
    else:
        form = EditUploadForm(qobuz_upload=qobuz_upload)
    download_error = ''
    if qobuz_upload.download_task_id:
        async_result = AsyncResult(qobuz_upload.download_task_id)
        if async_result.state == states.PENDING:
            download_status = 'Waiting to start'
        elif async_result.state == states.STARTED:
            download_status = 'Started'
        elif async_result.state == states.SUCCESS:
            download_status = 'Completed'
        elif async_result.state == states.FAILURE:
            download_status = 'Failed'
            download_error = '{0}: {1}'.format(type(async_result.result).__name__,
                                               async_result.result.message)
        else:
            download_status = 'Unknown Status'
    else:
        download_status = 'not_started'
    try:
        spectral_files = sorted(os.listdir(wm_str(qobuz_upload.spectrals_path)))
    except OSError:
        spectral_files = []
    try:
        cover_files = sorted([f for f in os.listdir(wm_str(qobuz_upload.temp_media_path))
                              if f.endswith('.jpg')], reverse=True)
    except OSError:
        cover_files = []
    data = {
        'upload': qobuz_upload,
        'form': form,
        'download_status': download_status,
        'download_error': download_error,
        'spectral_files': spectral_files,
        'cover_files': cover_files,
    }
    return render(request, 'qobuz/edit_upload.html', data)
 def subfolder_move(self, subfolder, torrent_id):
     if not self.error_move:
         print u'Skipping {}..'.format(torrent_id)
         return
     print u'Moving {} to {} subfolder..'.format(torrent_id, subfolder)
     s = wm_str(os.path.join(self.wm_media, torrent_id)) # source
     d = wm_str(os.path.join(self.wm_media, subfolder, torrent_id)) # dest
     safe_makedirs(os.path.dirname(d))
     try:
         shutil.move(s, d)
     except Exception as e:
         shutil.rmtree(d)
         raise e
 def move_files(self):
     print u'Moving files to new directory...'
     if 'files' in self.torrent_info['info']:
         for f in self.get_unicode_torrent_files():
             f_path = os.path.join(self.data_path, *f['path'])
             f_dest_path = os.path.join(self.dest_path, *f['path'])
             safe_makedirs(os.path.dirname(f_dest_path))
             shutil.move(wm_str(f_path), wm_str(f_dest_path))
     else:
         f_path = os.path.join(self.data_path, wm_unicode(self.torrent_info['info']['name']))
         f_dest_path = os.path.join(self.dest_path, wm_unicode(
             self.torrent_info['info']['name']))
         safe_makedirs(os.path.dirname(f_dest_path))
         shutil.move(wm_str(f_path), wm_str(f_dest_path))
    def find_existing_torrent_group(self):
        if self.existing_new_group is not None:
            return

        existing_group_id = None

        group_id = self.what_torrent_info['group']['id']
        try:
            mapping = TorrentGroupMapping.objects.get(what_group_id=group_id)
            mapping_group = self.what.request('torrentgroup', id=mapping.pth_group_id)['response']
            if mapping_group['group']['id'] != mapping.pth_group_id:
                raise Exception('NOOOOO THIS CANNOT HAPPEN {} {}!'.format(
                    mapping_group['group']['id'],
                    mapping.pth_group_id,
                ))
            existing_group_id = mapping.pth_group_id
            print 'Found torrent group mapping with {}'.format(existing_group_id)
        except TorrentGroupMapping.DoesNotExist:
            pass
        except BadIdException:
            print 'Mapping has gone bad, deleting...'
            mapping.delete()

        if existing_group_id is None:
            group_year = self.what_torrent_info['group']['year']
            group_name = html_parser.unescape(self.what_torrent_info['group']['name']).lower()
            search_str = '{} {}'.format(wm_str(group_name), wm_str(str(group_year)))
            results = self.what.request('browse', searchstr=search_str)['response']['results']
            for result in results:
                if html_parser.unescape(result['groupName']).lower() == group_name and \
                                result['groupYear'] == group_year:
                    if not existing_group_id:
                        existing_group_id = result['groupId']
                        print 'Found existing group', existing_group_id
                    else:
                        print 'Multiple matching existing groups!!!!!!!!!!'
                        existing_group_id = None
                        break
        if existing_group_id is None:
            existing_group_id = raw_input(u'Enter existing group id (empty if non-existent): ')
            if existing_group_id:
                TorrentGroupMapping.objects.get_or_create(
                    what_group_id=self.what_torrent_info['group']['id'],
                    pth_group_id=existing_group_id
                )
        if existing_group_id:
            self.existing_new_group = self.existing_new_group = self.what.request(
                'torrentgroup', id=existing_group_id)['response']
    def __init__(self, what, location_mapping, data, flac_only):
        self.flac_only = flac_only
        self.what = what
        self.location_mapping = location_mapping
        self.data = data
        self.what_torrent = self.data['what_torrent']
        self.what_torrent_info = ujson.loads(self.what_torrent['info'])
        self.full_location = os.path.join(
            wm_str(self.data['location']['path']),
            str(self.what_torrent['id']),
        )
        self.torrent_dict = bencode.bdecode(
            b64decode(self.what_torrent['torrent_file']))
        self.torrent_name = self.torrent_dict['info']['name']
        self.torrent_new_name = self.torrent_name
        self.torrent_dir_path = os.path.join(
            self.full_location.encode('utf-8'), self.torrent_name)

        self.new_torrent = None
        self.log_files = set()
        self.log_files_full_paths = []
        self.torrent_file_new_data = None
        self.torrent_new_infohash = None
        self.payload = None
        self.payload_files = None
        self.existing_new_group = None
        self.full_new_location = None
    def __init__(self, what, location_mapping, data, flac_only):
        self.flac_only = flac_only
        self.what = what
        self.location_mapping = location_mapping
        self.data = data
        self.what_torrent = self.data['what_torrent']
        self.what_torrent_info = ujson.loads(self.what_torrent['info'])
        self.full_location = os.path.join(
            wm_str(self.data['location']['path']),
            str(self.what_torrent['id']),
        )
        self.torrent_dict = bencode.bdecode(b64decode(self.what_torrent['torrent_file']))
        self.torrent_name = self.torrent_dict['info']['name']
        self.torrent_new_name = self.torrent_name
        self.torrent_dir_path = os.path.join(self.full_location.encode('utf-8'), self.torrent_name)

        self.new_torrent = None
        self.log_files = set()
        self.log_files_full_paths = []
        self.torrent_file_new_data = None
        self.torrent_new_infohash = None
        self.payload = None
        self.payload_files = None
        self.existing_new_group = None
        self.full_new_location = None
 def handle(self, *args, **options):
     if not self.check_args(args):
         print u'Pass the torrent data directory as a first argument, ' \
               u'a path to the .torrent file as a second.'
         return
     self.data_path, self.torrent_path = [wm_unicode(i) for i in args]
     with open(wm_str(self.torrent_path), 'rb') as f:
         self.torrent_info = bencode.bdecode(f.read())
     if options['base_dir']:
         self.data_path = os.path.join(self.data_path,
                                       wm_unicode(self.torrent_info['info']['name']))
     print u'Checking to see if torrent is already loaded into WM..'
     masters = list(ReplicaSet.get_what_master().transinstance_set.all())
     try:
         TransTorrent.objects.get(instance__in=masters, info_hash=self.info_hash)
         print u'Torrent already added to WM. Skipping...'
         return False
     except TransTorrent.DoesNotExist:
         pass
     self.what_torrent = WhatTorrent.get_or_create(self.pseudo_request, info_hash=self.info_hash)
     if not self.check_files():
         return
     self.move_files()
     print 'Adding torrent to WM...'
     manage_torrent.add_torrent(self.pseudo_request, self.trans_instance,
                                self.download_location, self.what_torrent.id)
     print 'Done!'
Beispiel #10
0
def upload_cover_to_whatimg(request, upload_id):
    qobuz_upload = QobuzUpload.objects.get(id=upload_id)
    with open(os.path.join(wm_str(qobuz_upload.temp_media_path), 'folder.jpg'), 'rb') as f:
        data = f.read()
    qobuz_upload.what_img_cover = whatimg.upload_image_from_memory('4460', data)
    qobuz_upload.save()
    return redirect(edit_upload, upload_id)
Beispiel #11
0
def download_torrent_file(request, upload_id):
    qobuz_upload = QobuzUpload.objects.get(id=upload_id)
    file_name = qobuz_upload.torrent_name + '.torrent'
    file_path = os.path.join(qobuz_upload.temp_media_path, file_name)
    f = open(wm_str(file_path), 'rb')
    response = HttpResponse(f, content_type='application/x-bittorrent')
    response['Content-Disposition'] = 'attachment; filename="' + file_name + '"'
    return response
Beispiel #12
0
def view_spectral(request, upload_id):
    try:
        qobuz_upload = QobuzUpload.objects.get(id=upload_id)
        path = os.path.join(qobuz_upload.spectrals_path, os.path.basename(request.GET['path']))
        f = open(wm_str(path), 'rb')
        return HttpResponse(f, content_type='image/jpeg')
    except Exception:
        return HttpResponseNotFound()
Beispiel #13
0
 def inner(request, upload_id):
     try:
         qobuz_upload = QobuzUpload.objects.get(id=upload_id)
         path = os.path.join(getattr(qobuz_upload, prop), os.path.basename(request.GET['path']))
         s = os.path.getmtime(wm_str(path))
         return datetime.datetime.utcfromtimestamp(s)
     except Exception:
         return None
 def move_files(self):
     print u'Moving files to new directory...'
     if 'files' in self.torrent_info['info']:
         for f in self.get_unicode_torrent_files():
             f_path = os.path.join(self.data_path, *f['path'])
             f_dest_path = os.path.join(self.dest_path, *f['path'])
             safe_makedirs(os.path.dirname(f_dest_path))
             shutil.move(wm_str(f_path), wm_str(f_dest_path))
     else:
         f_path = os.path.join(
             self.data_path, wm_unicode(self.torrent_info['info']['name']))
         f_dest_path = os.path.join(
             self.dest_path, wm_unicode(self.torrent_info['info']['name']))
         safe_makedirs(os.path.dirname(f_dest_path))
         shutil.move(wm_str(f_path), wm_str(f_dest_path))
     print u'Success!'
     self.subfolder_move('imported', self.torrent_id)
Beispiel #15
0
def download_torrent_file(request, upload_id):
    qobuz_upload = QobuzUpload.objects.get(id=upload_id)
    file_name = qobuz_upload.torrent_name + '.torrent'
    file_path = os.path.join(qobuz_upload.temp_media_path, file_name)
    f = open(wm_str(file_path), 'rb')
    response = HttpResponse(f, content_type='application/x-bittorrent')
    response[
        'Content-Disposition'] = 'attachment; filename="' + file_name + '"'
    return response
Beispiel #16
0
def view_cover(request, upload_id):
    try:
        qobuz_upload = QobuzUpload.objects.get(id=upload_id)
        dest_path = os.path.join(get_temp_dir(qobuz_upload.upload.metadata.id),
                                 os.path.basename(request.GET['path']))
        f = open(wm_str(dest_path), 'rb')
        return HttpResponse(f, content_type='image/jpeg')
    except Exception:
        return HttpResponseNotFound()
Beispiel #17
0
 def inner(request, upload_id):
     try:
         qobuz_upload = QobuzUpload.objects.get(id=upload_id)
         path = os.path.join(getattr(qobuz_upload, prop),
                             os.path.basename(request.GET['path']))
         s = os.path.getmtime(wm_str(path))
         return datetime.datetime.utcfromtimestamp(s)
     except Exception:
         return None
Beispiel #18
0
def upload_cover_to_whatimg(request, upload_id):
    qobuz_upload = QobuzUpload.objects.get(id=upload_id)
    with open(os.path.join(wm_str(qobuz_upload.temp_media_path), 'folder.jpg'),
              'rb') as f:
        data = f.read()
    qobuz_upload.what_img_cover = whatimg.upload_image_from_memory(
        '4460', data)
    qobuz_upload.save()
    return redirect(edit_upload, upload_id)
Beispiel #19
0
def seed_upload(request, upload_id):
    qobuz_upload = QobuzUpload.objects.get(id=upload_id)
    temp_dir = get_temp_dir(qobuz_upload.upload.metadata.id)
    torrent_path = os.path.join(temp_dir, qobuz_upload.upload.metadata.torrent_name + '.torrent')
    assert os.path.isfile(wm_str(torrent_path))
    info_hash = get_info_hash(torrent_path)
    what_torrent = WhatTorrent.get_or_create(request, info_hash=info_hash)
    command = import_external_what_torrent.Command()
    command.handle(wm_str(temp_dir), wm_str(torrent_path), base_dir=False)
    try:
        run_request_transcode(request, what_torrent.id)
    except Exception:
        pass
    qiller = qobuz_upload.upload
    qiller.state = STATE_DONE
    qobuz_upload.set_upload(qiller)
    qobuz_upload.save()
    return redirect(edit_upload, upload_id)
Beispiel #20
0
def view_spectral(request, upload_id):
    try:
        qobuz_upload = QobuzUpload.objects.get(id=upload_id)
        path = os.path.join(qobuz_upload.spectrals_path,
                            os.path.basename(request.GET['path']))
        f = open(wm_str(path), 'rb')
        return HttpResponse(f, content_type='image/jpeg')
    except Exception:
        return HttpResponseNotFound()
Beispiel #21
0
 def inner(request, upload_id):
     try:
         qobuz_upload = QobuzUpload.objects.get(id=upload_id)
         dest_path = os.path.join(get_temp_dir(qobuz_upload.upload.metadata.id), subpath,
                                  os.path.basename(request.GET['path']))
         s = os.path.getmtime(wm_str(dest_path))
         return datetime.datetime.utcfromtimestamp(s)
     except Exception:
         raise
         return None
Beispiel #22
0
 def fill(self, filename, file_mtime):
     metadata = mutagen.File(wm_str(filename), easy=True)
     if hasattr(metadata, 'pictures'):
         for p in metadata.pictures:
             p.data = None
     if hasattr(metadata, 'tags'):
         if hasattr(metadata.tags, '_EasyID3__id3'):
             metadata.tags._EasyID3__id3.delall('APIC')
     self.file_mtime = file_mtime
             try:
Beispiel #23
0
 def fill(self, filename, file_mtime):
     metadata = mutagen.File(wm_str(filename), easy=True)
     if hasattr(metadata, 'pictures'):
         for p in metadata.pictures:
             p.data = None
     if hasattr(metadata, 'tags'):
         if hasattr(metadata.tags, '_EasyID3__id3'):
             metadata.tags._EasyID3__id3.delall('APIC')
     self.file_mtime = file_mtime
     self.metadata_pickle = pickle.dumps(metadata)
     self.artists = self.easy['artist'][:200]
     self.album = self.easy['album'][:200]
     self.title = self.easy['title'][:200]
     self.duration = self.easy['duration']
Beispiel #24
0
 def fill(self, filename, file_mtime):
     metadata = mutagen.File(wm_str(filename), easy=True)
     if hasattr(metadata, 'pictures'):
         for p in metadata.pictures:
             p.data = None
     if hasattr(metadata, 'tags'):
         if hasattr(metadata.tags, '_EasyID3__id3'):
             metadata.tags._EasyID3__id3.delall('APIC')
     self.file_mtime = file_mtime
     self.metadata_pickle = pickle.dumps(metadata)
     self.artists = self.easy['artist'][:200]
     self.album = self.easy['album'][:200]
     self.title = self.easy['title'][:200]
     self.duration = self.easy['duration']
 def handle(self, *args, **options):
     if not self.check_args(args):
         print u'Pass the torrent data directory as a first argument, ' \
               u'a path to the .torrent file as a second.'
         return
     self.data_path, self.torrent_path = [wm_unicode(i) for i in args]
     with open(wm_str(self.torrent_path), 'rb') as f:
         self.torrent_info = bencode.bdecode(f.read())
     if options['base_dir']:
         self.data_path = os.path.join(self.data_path,
                                       wm_unicode(self.torrent_info['info']['name']))
     self.what_torrent = WhatTorrent.get_or_create(self.pseudo_request, info_hash=self.info_hash)
     if not self.check_files():
         return
     self.move_files()
     print 'Adding torrent to WM...'
     manage_torrent.add_torrent(self.pseudo_request, self.trans_instance,
                                self.download_location, self.what_torrent.id)
     print 'Done!'
Beispiel #26
0
def start_seeding(request, upload_id):
    qobuz_upload = QobuzUpload.objects.get(id=upload_id)
    dest_upload_dir = DownloadLocation.get_what_preferred().path
    torrent_file_path = os.path.join(qobuz_upload.temp_media_path,
                                     qobuz_upload.torrent_name + u'.torrent')
    info_hash = get_info_hash(torrent_file_path)
    what_torrent = WhatTorrent.get_or_create(request, info_hash=info_hash)
    qobuz_upload.what_torrent = what_torrent
    qobuz_upload.save()
    dest_path = os.path.join(dest_upload_dir, str(what_torrent.id))
    shutil.rmtree(wm_str(qobuz_upload.spectrals_path))
    os.remove(wm_str(torrent_file_path))
    try:
        os.makedirs(wm_str(dest_path))
    except OSError:
        raise Exception('Dest torrent directory already exists.')
    os.chmod(wm_str(dest_path), 0777)
    shutil.move(wm_str(qobuz_upload.temp_media_path), wm_str(dest_path))
    add_to_wm_transcode(str(what_torrent.id))
    return redirect(edit_upload, upload_id)
Beispiel #27
0
def start_seeding(request, upload_id):
    qobuz_upload = QobuzUpload.objects.get(id=upload_id)
    dest_upload_dir = DownloadLocation.get_what_preferred().path
    torrent_file_path = os.path.join(qobuz_upload.temp_media_path,
                                     qobuz_upload.torrent_name + u'.torrent')
    info_hash = get_info_hash(torrent_file_path)
    what_torrent = WhatTorrent.get_or_create(request, info_hash=info_hash)
    qobuz_upload.what_torrent = what_torrent
    qobuz_upload.save()
    dest_path = os.path.join(dest_upload_dir, str(what_torrent.id))
    shutil.rmtree(wm_str(qobuz_upload.spectrals_path))
    os.remove(wm_str(torrent_file_path))
    try:
        os.makedirs(wm_str(dest_path))
    except OSError:
        raise Exception('Dest torrent directory already exists.')
    os.chmod(wm_str(dest_path), 0777)
    shutil.move(wm_str(qobuz_upload.temp_media_path), wm_str(dest_path))
    add_to_wm_transcode(str(what_torrent.id))
    return redirect(edit_upload, upload_id)
def safe_makedirs(p):
    try:
        os.makedirs(wm_str(p))
    except OSError as ex:
        if ex.errno != errno.EEXIST:
            raise ex
Beispiel #29
0
    def get_metadata_batch(cls, what_torrent, trans_torrent, force_update):
        torrent_path = trans_torrent.path
        cache_lines = list(what_torrent.whatfilemetadatacache_set.all())
        if len(cache_lines) and not force_update:
            for cache_line in cache_lines:
                cache_line.path = os.path.join(torrent_path,
                                               cache_line.filename)
            return sorted(cache_lines, key=lambda c: c.path)

        cache_lines = {c.filename_sha256: c for c in cache_lines}

        abs_rel_filenames = []
        for dirpath, dirnames, filenames in os.walk(wm_str(torrent_path)):
            unicode_dirpath = wm_unicode(dirpath)
            unicode_filenames = [wm_unicode(f) for f in filenames]
            for filename in unicode_filenames:
                if os.path.splitext(filename)[1].lower() in [
                        u'.flac', u'.mp3'
                ]:
                    abs_path = os.path.join(unicode_dirpath, filename)
                    rel_path = os.path.relpath(abs_path, torrent_path)
                    abs_rel_filenames.append((abs_path, rel_path))
        abs_rel_filenames.sort(key=lambda f: f[1])

        filename_hashes = {
            f[0]: hashlib.sha256(wm_str(f[1])).hexdigest()
            for f in abs_rel_filenames
        }
        hash_set = set(filename_hashes.values())
        old_cache_lines = []
        for cache_line in cache_lines.itervalues():
            if cache_line.filename_sha256 not in hash_set:
                old_cache_lines.append(cache_line)
        dirty_cache_lines = []

        result = []
        for abs_path, rel_path in abs_rel_filenames:
            try:
                file_mtime = os.path.getmtime(wm_str(abs_path))
                cache = cache_lines.get(filename_hashes[abs_path])
                if cache is None:
                    cache = WhatFileMetadataCache(
                        what_torrent=what_torrent,
                        filename_sha256=filename_hashes[abs_path],
                        filename=rel_path[:400],
                        file_mtime=0)
                cache.path = abs_path
                if abs(file_mtime - cache.file_mtime) <= 1:
                    result.append(cache)
                    continue
                cache.fill(abs_path, file_mtime)
                dirty_cache_lines.append(cache)
                result.append(cache)
            except Exception as ex:
                print 'Failed:', abs_path, ex
        if old_cache_lines or dirty_cache_lines:
            with transaction.atomic():
                for cache_line in old_cache_lines:
                    cache_line.delete()
                for cache_line in dirty_cache_lines:
                    cache_line.save()
        return result
Beispiel #30
0
def view_cover(request, upload_id):
    qobuz_upload = QobuzUpload.objects.get(id=upload_id)
    path = os.path.join(qobuz_upload.temp_media_path, os.path.basename(request.GET['path']))
    f = open(wm_str(path), 'rb')
    return HttpResponse(f, content_type='image/jpeg')
    def handle(self, *args, **options):
        if not self.check_args(args):
            print u'Pass the directory containing your torrent directories from a previous WM' \
                  u' install. Subfolders of this directory should be named by torrent ID. After' \
                  u' import, all errored torrent/data sets will be organized into subfolders for' \
                  u' manual inspection/import.'
            return

        self.wm_media = wm_unicode(args[0])
        self.error_move = not options['no_move']

        for self.torrent_id in next(os.walk(self.wm_media))[1]:
            try:
                # Is this actually a directory?
                if not os.path.isdir(self.base_dir()):
                    print u'"{}" is not a valid directory. Skipping..'.format(self.base_dir())
                    continue

                # Get all torrents
                torrents = []
                hashes = []
                for p in os.listdir(self.base_dir()):
                    if p.endswith('.torrent') and not p.startswith('._'):
                        try:
                            p = os.path.join(self.base_dir(), wm_unicode(p))
                            hashes.append(get_info_hash(p))
                            torrents.append(p)
                        except IOError:
                            print('Warning: Invalid torrent found in {}'.format(self.torrent_id))
                            continue
                        except  BTFailure as e:
                            print('Warning: {}. Invalid torrent found in {}'.format(str(e), self.torrent_id))
                            continue
    
                # Are there any valid torrents?
                if len(torrents) == 0:
                    if self.torrent_id.isdigit():
                        print u'Error: No valid torrent files found in "{}".'.format(self.base_dir())
                        self.subfolder_move('no_torrents', self.torrent_id)
                    continue

                # Are there multiple unique torrents?
                if len(set(hashes)) > 1:
                    print u'Error: Multiple unique torrents found'
                    self.subfolder_move('multiple_torrent', self.torrent_id)
                    continue

            except UnicodeDecodeError as e:
                print u'UnicodeDecodeError: Please import manually. Skipping..'
                continue

            with open(wm_str(torrents[0]), 'rb') as f:
                try:
                    self.torrent_info = bencode.bdecode(f.read())
                    self.info_hash = get_info_hash(torrents[0])
                except:
                    print u'Error: Invalid torrent file.'
                    self.subfolder_move('invalid_torrent', self.torrent_id)
                    continue
                self.data_path = os.path.join(self.base_dir(), wm_unicode(self.torrent_info['info']['name']))
            print u'Checking to see if torrent is already loaded into WM..'
            masters = list(ReplicaSet.get_what_master().transinstance_set.all())
            try:
                TransTorrent.objects.get(instance__in=masters, info_hash=self.info_hash)
                print u'Error: Torrent already added to WM.'
                self.subfolder_move('already_added', self.torrent_id)
                continue
            except TransTorrent.DoesNotExist:
                pass
            try:
                self.what_torrent = WhatTorrent.get_or_create(self.pseudo_request, info_hash=self.info_hash)
            except RequestException as e:
                if 'bad hash' in str(e):
                    print u'Error: Bad hash. Torrent may have been trumped/deleted.'.format(str(e))
                    self.subfolder_move('bad_hash', self.torrent_id)
                    continue
                else:
                    raise e
            except OperationalError as e:
                if 'MySQL' in str(e):
                    print u'Error: {}. Please check {} manually.'.format(str(e), self.torrent_id)
                    self.subfolder_move('mysql_error', self.torrent_id)
                    continue
                else:
                    raise e
            if not self.check_files():
                print u'Error: File check failed.'
                try:
                    self.subfolder_move('file_check_fail', self.torrent_id)
                except UnicodeDecodeError as e:
                    print u'UnicodeDecodeError. Move failed. Please manually check {} Skipping..'.format(self.torrent_id)
                continue
            self.move_files()
            print u'Adding torrent to WM...'
            self.trans_instance = ReplicaSet.get_what_master().get_preferred_instance()
            manage_torrent.add_torrent(self.pseudo_request, self.trans_instance,
                                    self.download_location, self.what_torrent.id)
            print u'Done!'
    def process(self):
        what_torrent_id = self.what_torrent['id']

        if self.what_torrent_info['group']['categoryName'] != 'Music':
            print 'Skipping non-Music torrent', what_torrent_id
            return

        if self.flac_only and self.what_torrent_info['torrent'][
                'format'] != 'FLAC':
            print 'Skipping non-FLAC torrent', what_torrent_id
            return

        try:
            status = WhatTorrentMigrationStatus.objects.get(
                what_torrent_id=what_torrent_id)
            if status.status == WhatTorrentMigrationStatus.STATUS_COMPLETE:
                print 'Skipping complete torrent', what_torrent_id
                return
            elif status.status == WhatTorrentMigrationStatus.STATUS_DUPLICATE:
                print 'Skipping duplicate torrent', what_torrent_id
                return
            elif status.status == WhatTorrentMigrationStatus.STATUS_SKIPPED:
                print 'Skipping skipped torrent', what_torrent_id
                return
            elif status.status == WhatTorrentMigrationStatus.STATUS_SKIPPED_PERMANENTLY:
                print 'Skipping permanently skipped torrent', what_torrent_id
                return
            elif status.status == WhatTorrentMigrationStatus.STATUS_FAILED_VALIDATION:
                print 'Skipping failed validation torrent', what_torrent_id
                return
            elif status.status == WhatTorrentMigrationStatus.STATUS_RESEEDED:
                print 'Skipping reseeded torrent', what_torrent_id
                return
            else:
                raise Exception(
                    'Not sure what to do with status {} on {}'.format(
                        status.status, what_torrent_id))
        except WhatTorrentMigrationStatus.DoesNotExist:
            pass
        if not self.check_valid():
            return
        self.mktorrent()
        if not self.find_dupes():
            return
        if not self.new_torrent:
            self.enhance_torrent_data()
            self.prepare_payload()
            self.print_info()
            self.prepare_payload_files()
            self.generate_spectrals()
            raw_input('Will perform upload (CHECK THE SPECTRALS)...')
            self.perform_upload()
        self.set_new_location()
        if self.REAL_RUN:
            os.makedirs(self.full_new_location)
            shutil.move(wm_str(self.torrent_dir_path),
                        wm_str(self.full_new_location))
            try:
                recursive_chmod(self.full_new_location, 0777)
            except OSError:
                print 'recursive_chmod failed'
        else:
            print 'os.makedirs({})'.format(self.full_new_location)
            print 'shutil.move({}, {})'.format(self.torrent_dir_path,
                                               self.full_new_location)
            print 'recursive_chmod({}, 0777)'.format(self.full_new_location)
        if self.REAL_RUN:
            self.add_to_wm()
            self.migration_status.status = WhatTorrentMigrationStatus.STATUS_COMPLETE
            self.migration_status.save()
            self.save_torrent_group_mapping()
        else:
            print 'add_to_wm()'
        print
        print
    def process(self):
        what_torrent_id = self.what_torrent['id']

        if self.what_torrent_info['group']['categoryName'] != 'Music':
            print 'Skipping non-Music torrent', what_torrent_id
            return

        if self.flac_only and self.what_torrent_info['torrent']['format'] != 'FLAC':
            print 'Skipping non-FLAC torrent', what_torrent_id
            return

        try:
            status = WhatTorrentMigrationStatus.objects.get(what_torrent_id=what_torrent_id)
            if status.status == WhatTorrentMigrationStatus.STATUS_COMPLETE:
                print 'Skipping complete torrent', what_torrent_id
                return
            elif status.status == WhatTorrentMigrationStatus.STATUS_DUPLICATE:
                print 'Skipping duplicate torrent', what_torrent_id
                return
            elif status.status == WhatTorrentMigrationStatus.STATUS_SKIPPED:
                print 'Skipping skipped torrent', what_torrent_id
                return
            elif status.status == WhatTorrentMigrationStatus.STATUS_SKIPPED_PERMANENTLY:
                print 'Skipping permanently skipped torrent', what_torrent_id
                return
            elif status.status == WhatTorrentMigrationStatus.STATUS_FAILED_VALIDATION:
                print 'Skipping failed validation torrent', what_torrent_id
                return
            elif status.status == WhatTorrentMigrationStatus.STATUS_RESEEDED:
                print 'Skipping reseeded torrent', what_torrent_id
                return
            else:
                raise Exception('Not sure what to do with status {} on {}'.format(
                    status.status, what_torrent_id))
        except WhatTorrentMigrationStatus.DoesNotExist:
            pass
        if not self.check_valid():
            return
        self.mktorrent()
        if not self.find_dupes():
            return
        if not self.new_torrent:
            self.enhance_torrent_data()
            self.prepare_payload()
            self.print_info()
            self.prepare_payload_files()
            self.generate_spectrals()
            raw_input('Will perform upload (CHECK THE SPECTRALS)...')
            self.perform_upload()
        self.set_new_location()
        if self.REAL_RUN:
            os.makedirs(self.full_new_location)
            shutil.move(wm_str(self.torrent_dir_path), wm_str(self.full_new_location))
            try:
                recursive_chmod(self.full_new_location, 0777)
            except OSError:
                print 'recursive_chmod failed'
        else:
            print 'os.makedirs({})'.format(self.full_new_location)
            print 'shutil.move({}, {})'.format(self.torrent_dir_path, self.full_new_location)
            print 'recursive_chmod({}, 0777)'.format(self.full_new_location)
        if self.REAL_RUN:
            self.add_to_wm()
            self.migration_status.status = WhatTorrentMigrationStatus.STATUS_COMPLETE
            self.migration_status.save()
            self.save_torrent_group_mapping()
        else:
            print 'add_to_wm()'
        print
        print
    def handle(self, *args, **options):
        if not self.check_args(args):
            print u'Pass the directory containing your torrent directories from a previous WM' \
                  u' install. Subfolders of this directory should be named by torrent ID. After' \
                  u' import, all errored torrent/data sets will be organized into subfolders for' \
                  u' manual inspection/import.'
            return

        self.wm_media = wm_unicode(args[0])
        self.error_move = not options['no_move']

        for self.torrent_id in next(os.walk(self.wm_media))[1]:
            try:
                # Is this actually a directory?
                if not os.path.isdir(self.base_dir()):
                    print u'"{}" is not a valid directory. Skipping..'.format(
                        self.base_dir())
                    continue

                # Get all torrents
                torrents = []
                hashes = []
                for p in os.listdir(self.base_dir()):
                    if p.endswith('.torrent') and not p.startswith('._'):
                        try:
                            p = os.path.join(self.base_dir(), wm_unicode(p))
                            hashes.append(get_info_hash(p))
                            torrents.append(p)
                        except IOError:
                            print(
                                'Warning: Invalid torrent found in {}'.format(
                                    self.torrent_id))
                            continue
                        except BTFailure as e:
                            print('Warning: {}. Invalid torrent found in {}'.
                                  format(str(e), self.torrent_id))
                            continue

                # Are there any valid torrents?
                if len(torrents) == 0:
                    if self.torrent_id.isdigit():
                        print u'Error: No valid torrent files found in "{}".'.format(
                            self.base_dir())
                        self.subfolder_move('no_torrents', self.torrent_id)
                    continue

                # Are there multiple unique torrents?
                if len(set(hashes)) > 1:
                    print u'Error: Multiple unique torrents found'
                    self.subfolder_move('multiple_torrent', self.torrent_id)
                    continue

            except UnicodeDecodeError as e:
                print u'UnicodeDecodeError: Please import manually. Skipping..'
                continue

            with open(wm_str(torrents[0]), 'rb') as f:
                try:
                    self.torrent_info = bencode.bdecode(f.read())
                    self.info_hash = get_info_hash(torrents[0])
                except:
                    print u'Error: Invalid torrent file.'
                    self.subfolder_move('invalid_torrent', self.torrent_id)
                    continue
                self.data_path = os.path.join(
                    self.base_dir(),
                    wm_unicode(self.torrent_info['info']['name']))
            print u'Checking to see if torrent is already loaded into WM..'
            masters = list(
                ReplicaSet.get_what_master().transinstance_set.all())
            try:
                TransTorrent.objects.get(instance__in=masters,
                                         info_hash=self.info_hash)
                print u'Error: Torrent already added to WM.'
                self.subfolder_move('already_added', self.torrent_id)
                continue
            except TransTorrent.DoesNotExist:
                pass
            try:
                self.what_torrent = WhatTorrent.get_or_create(
                    self.pseudo_request, info_hash=self.info_hash)
            except RequestException as e:
                if 'bad hash' in str(e):
                    print u'Error: Bad hash. Torrent may have been trumped/deleted.'.format(
                        str(e))
                    self.subfolder_move('bad_hash', self.torrent_id)
                    continue
                else:
                    raise e
            except OperationalError as e:
                if 'MySQL' in str(e):
                    print u'Error: {}. Please check {} manually.'.format(
                        str(e), self.torrent_id)
                    self.subfolder_move('mysql_error', self.torrent_id)
                    continue
                else:
                    raise e
            if not self.check_files():
                print u'Error: File check failed.'
                try:
                    self.subfolder_move('file_check_fail', self.torrent_id)
                except UnicodeDecodeError as e:
                    print u'UnicodeDecodeError. Move failed. Please manually check {} Skipping..'.format(
                        self.torrent_id)
                continue
            self.move_files()
            print u'Adding torrent to WM...'
            self.trans_instance = ReplicaSet.get_what_master(
            ).get_preferred_instance()
            manage_torrent.add_torrent(self.pseudo_request,
                                       self.trans_instance,
                                       self.download_location,
                                       self.what_torrent.id)
            print u'Done!'
 def check_files(self):
     print u'Checking for existing files...'
     if 'files' in self.torrent_info['info']:
         for f in self.get_unicode_torrent_files():
             f_path = os.path.join(self.data_path, *f['path'])
             print wm_str(u'Checking {0}'.format(f_path))
             if not os.path.isfile(wm_str(f_path)):
                 print wm_str(u'{0} does not exist. What are you giving me?'.format(f_path))
                 return False
     else:
         f_path = os.path.join(self.data_path, self.torrent_info['info']['name'])
         print wm_str(u'Checking {0}'.format(f_path))
         if not os.path.isfile(wm_str(f_path)):
             print wm_str(u'{0} does not exist. What are you giving me?'.format(f_path))
             return False
     print u'Creating destination directory...'
     self.dest_path = os.path.join(self.download_location.path, unicode(self.what_torrent.id))
     os.makedirs(wm_str(self.dest_path))
     os.chmod(self.dest_path, 0777)
     if 'files' in self.torrent_info['info']:
         self.dest_path = os.path.join(self.dest_path, wm_unicode(
             self.torrent_info['info']['name']))
         os.makedirs(wm_str(self.dest_path))
     print u'All torrent data files exist.'
     return True
Beispiel #36
0
def view_cover(request, upload_id):
    qobuz_upload = QobuzUpload.objects.get(id=upload_id)
    path = os.path.join(qobuz_upload.temp_media_path,
                        os.path.basename(request.GET['path']))
    f = open(wm_str(path), 'rb')
    return HttpResponse(f, content_type='image/jpeg')
Beispiel #37
0
def edit_upload(request, upload_id):
    qobuz_upload = QobuzUpload.objects.get(id=upload_id)
    if request.method == 'POST':
        form = EditUploadForm(request.POST, qobuz_upload=qobuz_upload)
        if form.is_valid():
            qobuz_upload.artists = form.cleaned_data['artists']
            qobuz_upload.album_name = form.cleaned_data['album_name']
            tracks = []
            for i, track in enumerate(qobuz_upload.track_data):
                qobuz_track = qobuz_upload.album_data['tracks']['items'][i]
                title_field = 'track_{0}_{1}_title'.format(
                    qobuz_track['media_number'], i + 1)
                artists_field = 'track_{0}_{1}_artists'.format(
                    qobuz_track['media_number'], i + 1)
                tracks.append({
                    'media_number': qobuz_track['media_number'],
                    'artists': form.cleaned_data[artists_field],
                    'title': form.cleaned_data[title_field],
                })
            qobuz_upload.track_data_json = ujson.dumps(tracks)
            qobuz_upload.save()
    else:
        form = EditUploadForm(qobuz_upload=qobuz_upload)
    download_error = ''
    if qobuz_upload.download_task_id:
        async_result = AsyncResult(qobuz_upload.download_task_id)
        if async_result.state == states.PENDING:
            download_status = 'Waiting to start'
        elif async_result.state == states.STARTED:
            download_status = 'Started'
        elif async_result.state == states.SUCCESS:
            download_status = 'Completed'
        elif async_result.state == states.FAILURE:
            download_status = 'Failed'
            download_error = '{0}: {1}'.format(
                type(async_result.result).__name__,
                async_result.result.message)
        else:
            download_status = 'Unknown Status'
    else:
        download_status = 'not_started'
    try:
        spectral_files = sorted(os.listdir(wm_str(
            qobuz_upload.spectrals_path)))
    except OSError:
        spectral_files = []
    try:
        cover_files = sorted([
            f for f in os.listdir(wm_str(qobuz_upload.temp_media_path))
            if f.endswith('.jpg')
        ],
                             reverse=True)
    except OSError:
        cover_files = []
    data = {
        'upload': qobuz_upload,
        'form': form,
        'download_status': download_status,
        'download_error': download_error,
        'spectral_files': spectral_files,
        'cover_files': cover_files,
    }
    return render(request, 'qobuz/edit_upload.html', data)
Beispiel #38
0
def get_info_hash(torrent_path):
    with open(wm_str(torrent_path), 'rb') as torrent_file:
        return get_info_hash_from_data(torrent_file.read())
Beispiel #39
0
def get_info_hash(torrent_path):
    with open(wm_str(torrent_path), 'rb') as torrent_file:
        return get_info_hash_from_data(torrent_file.read())