def handle(self, *args, **options):
if not self.check_args(args):
print u'Pass the torrent data directory as a first argument, ' \
u'a path to the .torrent file as a second.'
return
self.data_path, self.torrent_path = [wm_unicode(i) for i in args]
with open(wm_str(self.torrent_path), 'rb') as f:
self.torrent_info = bencode.bdecode(f.read())
if options['base_dir']:
self.data_path = os.path.join(self.data_path,
wm_unicode(self.torrent_info['info']['name']))
print u'Checking to see if torrent is already loaded into WM..'
masters = list(ReplicaSet.get_what_master().transinstance_set.all())
try:
TransTorrent.objects.get(instance__in=masters, info_hash=self.info_hash)
print u'Torrent already added to WM. Skipping...'
return False
except TransTorrent.DoesNotExist:
pass
self.what_torrent = WhatTorrent.get_or_create(self.pseudo_request, info_hash=self.info_hash)
if not self.check_files():
return
self.move_files()
print 'Adding torrent to WM...'
manage_torrent.add_torrent(self.pseudo_request, self.trans_instance,
self.download_location, self.what_torrent.id)
print 'Done!'
def get_metadata_batch(cls, what_torrent, trans_torrent, force_update):
torrent_path = trans_torrent.path
cache_lines = list(what_torrent.whatfilemetadatacache_set.all())
if len(cache_lines) and not force_update:
for cache_line in cache_lines:
cache_line.path = os.path.join(torrent_path, cache_line.filename)
return sorted(cache_lines, key=lambda c: c.path)
cache_lines = {c.filename_sha256: c for c in cache_lines}
abs_rel_filenames = []
for dirpath, dirnames, filenames in os.walk(wm_str(torrent_path)):
unicode_dirpath = wm_unicode(dirpath)
unicode_filenames = [wm_unicode(f) for f in filenames]
for filename in unicode_filenames:
if os.path.splitext(filename)[1].lower() in [u'.flac', u'.mp3']:
abs_path = os.path.join(unicode_dirpath, filename)
rel_path = os.path.relpath(abs_path, torrent_path)
abs_rel_filenames.append((abs_path, rel_path))
abs_rel_filenames.sort(key=lambda f: f[1])
filename_hashes = {f[0]: hashlib.sha256(wm_str(f[1])).hexdigest() for f in
abs_rel_filenames}
hash_set = set(filename_hashes.values())
old_cache_lines = []
for cache_line in cache_lines.itervalues():
if cache_line.filename_sha256 not in hash_set:
old_cache_lines.append(cache_line)
dirty_cache_lines = []
result = []
for abs_path, rel_path in abs_rel_filenames:
try:
file_mtime = os.path.getmtime(wm_str(abs_path))
cache = cache_lines.get(filename_hashes[abs_path])
if cache is None:
cache = WhatFileMetadataCache(
what_torrent=what_torrent,
filename_sha256=filename_hashes[abs_path],
filename=rel_path[:400],
file_mtime=0
)
cache.path = abs_path
if abs(file_mtime - cache.file_mtime) <= 1:
result.append(cache)
continue
cache.fill(abs_path, file_mtime)
dirty_cache_lines.append(cache)
result.append(cache)
except Exception as ex:
print 'Failed:', abs_path, ex
if old_cache_lines or dirty_cache_lines:
with transaction.atomic():
for cache_line in old_cache_lines:
cache_line.delete()
for cache_line in dirty_cache_lines:
cache_line.save()
return result
def move_files(self):
print u'Moving files to new directory...'
if 'files' in self.torrent_info['info']:
for f in self.get_unicode_torrent_files():
f_path = os.path.join(self.data_path, *f['path'])
f_dest_path = os.path.join(self.dest_path, *f['path'])
safe_makedirs(os.path.dirname(f_dest_path))
shutil.move(wm_str(f_path), wm_str(f_dest_path))
else:
f_path = os.path.join(self.data_path, wm_unicode(self.torrent_info['info']['name']))
f_dest_path = os.path.join(self.dest_path, wm_unicode(
self.torrent_info['info']['name']))
safe_makedirs(os.path.dirname(f_dest_path))
shutil.move(wm_str(f_path), wm_str(f_dest_path))
def check_files(self):
print u'Checking for existing files...'
if 'files' in self.torrent_info['info']:
for f in self.get_unicode_torrent_files():
f_path = os.path.join(self.data_path, *f['path'])
print wm_str(u'Checking {0}'.format(f_path))
if not os.path.isfile(wm_str(f_path)):
print wm_str(u'{0} does not exist. What are you giving me?'.format(f_path))
return False
else:
f_path = os.path.join(self.data_path, self.torrent_info['info']['name'])
print wm_str(u'Checking {0}'.format(f_path))
if not os.path.isfile(wm_str(f_path)):
print wm_str(u'{0} does not exist. What are you giving me?'.format(f_path))
return False
print u'Creating destination directory...'
self.dest_path = os.path.join(self.download_location.path, unicode(self.what_torrent.id))
os.makedirs(wm_str(self.dest_path))
os.chmod(self.dest_path, 0777)
if 'files' in self.torrent_info['info']:
self.dest_path = os.path.join(self.dest_path, wm_unicode(
self.torrent_info['info']['name']))
os.makedirs(wm_str(self.dest_path))
print u'All torrent data files exist.'
return True
def get_unicode_torrent_files(self):
files = []
for f in self.torrent_info['info']['files']:
nf = dict(f)
nf['path'] = [wm_unicode(i) for i in nf['path']]
files.append(nf)
return files
def sync_files(self):
if os.path.exists(self.path):
files = [wm_unicode(f) for f in os.listdir(self.path)]
else:
os.mkdir(self.path, 0777)
os.chmod(self.path, 0777)
files = []
files_added = []
if not any(u'.torrent' in f for f in files):
files_added.append(u'torrent')
torrent_path = os.path.join(
wm_str(self.path), wm_str(self.what_torrent.torrent_file_name))
with open(torrent_path, 'wb') as file:
file.write(self.what_torrent.torrent_file_binary)
os.chmod(torrent_path, 0777)
if not any(u'ReleaseInfo2.txt' == f for f in files):
files_added.append(u'ReleaseInfo2.txt')
release_info_path = os.path.join(self.path.encode('utf-8'),
u'ReleaseInfo2.txt')
with open(release_info_path.decode('utf-8'), 'w') as file:
file.write(self.what_torrent.info)
os.chmod(os.path.join(release_info_path.decode('utf-8')), 0777)
if files_added:
LogEntry.add(
None, u'info',
u'Added files {0} to {1}'.format(', '.join(files_added), self))
def sync_files(self):
if os.path.exists(self.path):
files = [wm_unicode(f) for f in os.listdir(self.path)]
else:
os.mkdir(self.path, 0777)
os.chmod(self.path, 0777)
files = []
files_added = []
if not any(u'.torrent' in f for f in files):
files_added.append(u'torrent')
torrent_path = os.path.join(wm_str(self.path),
wm_str(self.what_torrent.torrent_file_name))
with open(torrent_path, 'wb') as file:
file.write(self.what_torrent.torrent_file_binary)
os.chmod(torrent_path, 0777)
if not any(u'ReleaseInfo2.txt' == f for f in files):
files_added.append(u'ReleaseInfo2.txt')
release_info_path = os.path.join(self.path.encode('utf-8'), u'ReleaseInfo2.txt')
with open(release_info_path.decode('utf-8'), 'w') as file:
file.write(self.what_torrent.info)
os.chmod(os.path.join(release_info_path.decode('utf-8')), 0777)
if files_added:
LogEntry.add(None, u'info', u'Added files {0} to {1}'
.format(', '.join(files_added), self))
def handle(self, *args, **options):
if not self.check_args(args):
print u'Pass the torrent data directory as a first argument, ' \
u'a path to the .torrent file as a second.'
return
self.data_path, self.torrent_path = [wm_unicode(i) for i in args]
with open(wm_str(self.torrent_path), 'rb') as f:
self.torrent_info = bencode.bdecode(f.read())
if options['base_dir']:
self.data_path = os.path.join(self.data_path,
wm_unicode(self.torrent_info['info']['name']))
self.what_torrent = WhatTorrent.get_or_create(self.pseudo_request, info_hash=self.info_hash)
if not self.check_files():
return
self.move_files()
print 'Adding torrent to WM...'
manage_torrent.add_torrent(self.pseudo_request, self.trans_instance,
self.download_location, self.what_torrent.id)
print 'Done!'
def handle(self, *args, **options):
if len(args) != 1:
print u'Pass only the source directory.'
return 1
source_dir = wm_unicode(args[0])
if source_dir.endswith('/'):
source_dir = source_dir[:-1]
what = get_what_client(lambda: None)
job = TranscodeSingleJob(what, None, report_progress, None, None, source_dir)
job.create_torrent()
raw_input('Please upload the torrent and press enter...')
job.move_torrent_to_dest()
add_to_wm_transcode(job.new_torrent['torrent']['id'])
def handle(self, *args, **options):
if len(args) != 1:
print u'Pass only the source directory.'
return 1
source_dir = wm_unicode(args[0])
if source_dir.endswith('/'):
source_dir = source_dir[:-1]
what = get_what_client(lambda: None)
job = TranscodeSingleJob(what, None, report_progress, None, None,
source_dir)
job.create_torrent()
raw_input('Please upload the torrent and press enter...')
job.move_torrent_to_dest()
add_to_wm_transcode(job.new_torrent['torrent']['id'])
def get_metadata_batch(cls, what_torrent, trans_torrent, force_update):
torrent_path = trans_torrent.path
cache_lines = list(what_torrent.whatfilemetadatacache_set.all())
if len(cache_lines) and not force_update:
for cache_line in cache_lines:
cache_line.path = os.path.join(torrent_path,
cache_line.filename)
return sorted(cache_lines, key=lambda c: c.path)
cache_lines = {c.filename_sha256: c for c in cache_lines}
abs_rel_filenames = []
for dirpath, dirnames, filenames in os.walk(wm_str(torrent_path)):
unicode_dirpath = wm_unicode(dirpath)
unicode_filenames = [wm_unicode(f) for f in filenames]
for filename in unicode_filenames:
if os.path.splitext(filename)[1].lower() in [
u'.flac', u'.mp3'
]:
abs_path = os.path.join(unicode_dirpath, filename)
rel_path = os.path.relpath(abs_path, torrent_path)
abs_rel_filenames.append((abs_path, rel_path))
abs_rel_filenames.sort(key=lambda f: f[1])
filename_hashes = {
f[0]: hashlib.sha256(wm_str(f[1])).hexdigest()
for f in abs_rel_filenames
}
hash_set = set(filename_hashes.values())
old_cache_lines = []
for cache_line in cache_lines.itervalues():
if cache_line.filename_sha256 not in hash_set:
old_cache_lines.append(cache_line)
dirty_cache_lines = []
result = []
for abs_path, rel_path in abs_rel_filenames:
try:
file_mtime = os.path.getmtime(wm_str(abs_path))
cache = cache_lines.get(filename_hashes[abs_path])
if cache is None:
cache = WhatFileMetadataCache(
what_torrent=what_torrent,
filename_sha256=filename_hashes[abs_path],
filename=rel_path[:400],
file_mtime=0)
cache.path = abs_path
if abs(file_mtime - cache.file_mtime) <= 1:
result.append(cache)
continue
cache.fill(abs_path, file_mtime)
dirty_cache_lines.append(cache)
result.append(cache)
except Exception as ex:
print 'Failed:', abs_path, ex
if old_cache_lines or dirty_cache_lines:
with transaction.atomic():
for cache_line in old_cache_lines:
cache_line.delete()
for cache_line in dirty_cache_lines:
cache_line.save()
return result
def handle(self, *args, **options):
if not self.check_args(args):
print u'Pass the directory containing your torrent directories from a previous WM' \
u' install. Subfolders of this directory should be named by torrent ID. After' \
u' import, all errored torrent/data sets will be organized into subfolders for' \
u' manual inspection/import.'
return
self.wm_media = wm_unicode(args[0])
self.error_move = not options['no_move']
for self.torrent_id in next(os.walk(self.wm_media))[1]:
try:
# Is this actually a directory?
if not os.path.isdir(self.base_dir()):
print u'"{}" is not a valid directory. Skipping..'.format(self.base_dir())
continue
# Get all torrents
torrents = []
hashes = []
for p in os.listdir(self.base_dir()):
if p.endswith('.torrent') and not p.startswith('._'):
try:
p = os.path.join(self.base_dir(), wm_unicode(p))
hashes.append(get_info_hash(p))
torrents.append(p)
except IOError:
print('Warning: Invalid torrent found in {}'.format(self.torrent_id))
continue
except BTFailure as e:
print('Warning: {}. Invalid torrent found in {}'.format(str(e), self.torrent_id))
continue
# Are there any valid torrents?
if len(torrents) == 0:
if self.torrent_id.isdigit():
print u'Error: No valid torrent files found in "{}".'.format(self.base_dir())
self.subfolder_move('no_torrents', self.torrent_id)
continue
# Are there multiple unique torrents?
if len(set(hashes)) > 1:
print u'Error: Multiple unique torrents found'
self.subfolder_move('multiple_torrent', self.torrent_id)
continue
except UnicodeDecodeError as e:
print u'UnicodeDecodeError: Please import manually. Skipping..'
continue
with open(wm_str(torrents[0]), 'rb') as f:
try:
self.torrent_info = bencode.bdecode(f.read())
self.info_hash = get_info_hash(torrents[0])
except:
print u'Error: Invalid torrent file.'
self.subfolder_move('invalid_torrent', self.torrent_id)
continue
self.data_path = os.path.join(self.base_dir(), wm_unicode(self.torrent_info['info']['name']))
print u'Checking to see if torrent is already loaded into WM..'
masters = list(ReplicaSet.get_what_master().transinstance_set.all())
try:
TransTorrent.objects.get(instance__in=masters, info_hash=self.info_hash)
print u'Error: Torrent already added to WM.'
self.subfolder_move('already_added', self.torrent_id)
continue
except TransTorrent.DoesNotExist:
pass
try:
self.what_torrent = WhatTorrent.get_or_create(self.pseudo_request, info_hash=self.info_hash)
except RequestException as e:
if 'bad hash' in str(e):
print u'Error: Bad hash. Torrent may have been trumped/deleted.'.format(str(e))
self.subfolder_move('bad_hash', self.torrent_id)
continue
else:
raise e
except OperationalError as e:
if 'MySQL' in str(e):
print u'Error: {}. Please check {} manually.'.format(str(e), self.torrent_id)
self.subfolder_move('mysql_error', self.torrent_id)
continue
else:
raise e
if not self.check_files():
print u'Error: File check failed.'
try:
self.subfolder_move('file_check_fail', self.torrent_id)
except UnicodeDecodeError as e:
print u'UnicodeDecodeError. Move failed. Please manually check {} Skipping..'.format(self.torrent_id)
continue
self.move_files()
print u'Adding torrent to WM...'
self.trans_instance = ReplicaSet.get_what_master().get_preferred_instance()
manage_torrent.add_torrent(self.pseudo_request, self.trans_instance,
self.download_location, self.what_torrent.id)
print u'Done!'
def handle(self, *args, **options):
if not self.check_args(args):
print u'Pass the directory containing your torrent directories from a previous WM' \
u' install. Subfolders of this directory should be named by torrent ID. After' \
u' import, all errored torrent/data sets will be organized into subfolders for' \
u' manual inspection/import.'
return
self.wm_media = wm_unicode(args[0])
self.error_move = not options['no_move']
for self.torrent_id in next(os.walk(self.wm_media))[1]:
try:
# Is this actually a directory?
if not os.path.isdir(self.base_dir()):
print u'"{}" is not a valid directory. Skipping..'.format(
self.base_dir())
continue
# Get all torrents
torrents = []
hashes = []
for p in os.listdir(self.base_dir()):
if p.endswith('.torrent') and not p.startswith('._'):
try:
p = os.path.join(self.base_dir(), wm_unicode(p))
hashes.append(get_info_hash(p))
torrents.append(p)
except IOError:
print(
'Warning: Invalid torrent found in {}'.format(
self.torrent_id))
continue
except BTFailure as e:
print('Warning: {}. Invalid torrent found in {}'.
format(str(e), self.torrent_id))
continue
# Are there any valid torrents?
if len(torrents) == 0:
if self.torrent_id.isdigit():
print u'Error: No valid torrent files found in "{}".'.format(
self.base_dir())
self.subfolder_move('no_torrents', self.torrent_id)
continue
# Are there multiple unique torrents?
if len(set(hashes)) > 1:
print u'Error: Multiple unique torrents found'
self.subfolder_move('multiple_torrent', self.torrent_id)
continue
except UnicodeDecodeError as e:
print u'UnicodeDecodeError: Please import manually. Skipping..'
continue
with open(wm_str(torrents[0]), 'rb') as f:
try:
self.torrent_info = bencode.bdecode(f.read())
self.info_hash = get_info_hash(torrents[0])
except:
print u'Error: Invalid torrent file.'
self.subfolder_move('invalid_torrent', self.torrent_id)
continue
self.data_path = os.path.join(
self.base_dir(),
wm_unicode(self.torrent_info['info']['name']))
print u'Checking to see if torrent is already loaded into WM..'
masters = list(
ReplicaSet.get_what_master().transinstance_set.all())
try:
TransTorrent.objects.get(instance__in=masters,
info_hash=self.info_hash)
print u'Error: Torrent already added to WM.'
self.subfolder_move('already_added', self.torrent_id)
continue
except TransTorrent.DoesNotExist:
pass
try:
self.what_torrent = WhatTorrent.get_or_create(
self.pseudo_request, info_hash=self.info_hash)
except RequestException as e:
if 'bad hash' in str(e):
print u'Error: Bad hash. Torrent may have been trumped/deleted.'.format(
str(e))
self.subfolder_move('bad_hash', self.torrent_id)
continue
else:
raise e
except OperationalError as e:
if 'MySQL' in str(e):
print u'Error: {}. Please check {} manually.'.format(
str(e), self.torrent_id)
self.subfolder_move('mysql_error', self.torrent_id)
continue
else:
raise e
if not self.check_files():
print u'Error: File check failed.'
try:
self.subfolder_move('file_check_fail', self.torrent_id)
except UnicodeDecodeError as e:
print u'UnicodeDecodeError. Move failed. Please manually check {} Skipping..'.format(
self.torrent_id)
continue
self.move_files()
print u'Adding torrent to WM...'
self.trans_instance = ReplicaSet.get_what_master(
).get_preferred_instance()
manage_torrent.add_torrent(self.pseudo_request,
self.trans_instance,
self.download_location,
self.what_torrent.id)
print u'Done!'