def download():
if not os.path.exists(os.path.join(settings.config_path, 'release.json')):
return True
release = get_latest_release()
if release:
ox.makedirs(settings.updates_path)
os.chdir(os.path.dirname(settings.base_dir))
current_files = {'release.json'}
for module in release['modules']:
if release['modules'][module]['version'] > current_version(module):
module_tar = os.path.join(settings.updates_path, release['modules'][module]['name'])
base_url = settings.server.get('release_url').rsplit('/', 1)[0]
url = '/'.join([base_url, release['modules'][module]['name']])
if not os.path.exists(module_tar):
logger.debug('download', os.path.basename(module_tar))
get(url, module_tar)
if ox.sha1sum(module_tar) != release['modules'][module]['sha1']:
logger.debug('invalid checksum', os.path.basename(module_tar))
os.unlink(module_tar)
return False
current_files.add(os.path.basename(module_tar))
for f in set(next(os.walk(settings.updates_path))[2])-current_files:
os.unlink(os.path.join(settings.updates_path, f))
return True
return True
def update_icon(self):
frames = []
if not self.poster_frames:
items = self.get_items(self.user).filter(rendered=True)
if items.count():
poster_frames = []
for i in range(0, items.count(), max(1,
int(items.count() / 4))):
poster_frames.append({
'item': items[int(i)].itemId,
'position': items[int(i)].poster_frame
})
self.poster_frames = tuple(poster_frames)
self.save()
for i in self.poster_frames:
from item.models import Item
qs = Item.objects.filter(itemId=i['item'])
if qs.count() > 0:
frame = qs[0].frame(i['position'])
if frame:
frames.append(frame)
self.icon.name = self.path('icon.jpg')
icon = self.icon.path
if frames:
while len(frames) < 4:
frames += frames
folder = os.path.dirname(icon)
ox.makedirs(folder)
for f in glob("%s/icon*.jpg" % folder):
os.unlink(f)
cmd = [settings.LIST_ICON, '-f', ','.join(frames), '-o', icon]
p = subprocess.Popen(cmd)
p.wait()
self.save()
def save_files(context, request, callback):
listname = request.arguments.get('list', None)
if listname:
listname = listname[0]
if isinstance(listname, bytes):
listname = listname.decode('utf-8')
with context():
prefs = settings.preferences
ids = []
for upload in request.files.get('files', []):
filename = upload.filename
id = get_id(data=upload.body)
ids.append(id)
file = File.get(id)
if not file:
prefix_books = os.path.join(os.path.expanduser(prefs['libraryPath']), 'Books/')
prefix_imported = os.path.join(prefix_books, 'Imported/')
ox.makedirs(prefix_imported)
import_name = os.path.join(prefix_imported, filename)
n = 1
while os.path.exists(import_name):
n += 1
name, extension = filename.rsplit('.', 1)
import_name = os.path.join(prefix_imported, '%s [%d].%s' % (name, n, extension))
with open(import_name, 'wb') as fd:
fd.write(upload.body)
file = add_file(id, import_name, prefix_books)
if listname and ids:
l = List.get(settings.USER_ID, listname)
if l:
l.add_items(ids)
response = json_response({'ids': ids})
callback(response)
def frame(video, frame, position, height=128, redo=False):
'''
params:
video input
frame output
position as float in seconds
height of frame
redo boolean to extract file even if it exists
'''
if exists(video):
folder = os.path.dirname(frame)
if redo or not exists(frame):
ox.makedirs(folder)
if video.endswith('.mp4'):
cmd = [
AVCONV, '-y',
'-ss', str(position),
'-i', video,
'-an', '-vframes', '1',
'-vf', 'scale=-1:%s' % height
]
if not frame.endswith('.png'):
cmd += ['-f', 'mjpeg']
cmd += [frame]
else:
cmd = ['oxframe', '-i', video, '-o', frame,
'-p', str(position), '-y', str(height)]
run_command(cmd)
def config(self, args):
print "Current Config:\n User %s\n URL:%s\n" %(self._config['username'], self._config['url'])
print "Leave empty to keep current value\n"
username = raw_input('Username: '******'username'] = username
password = getpass.getpass('Password: '******'password'] = password
url = raw_input('Pan.do/ra URL(i.e. http://pad.ma/api/): ')
if url:
self._config['url'] = url
self.save_config()
print "\nconfiguration updated."
#install required programs
if sys.platform == 'darwin':
osname = 'macosx'
elif sys.platform == 'win32':
osname = 'exe'
else:
osname = 'linux'
bindir = os.path.expanduser('~/.ox/bin')
ox.makedirs(bindir)
for p in ('ffmpeg', 'ffmpeg2theora'):
path = os.path.join(bindir, p)
if sys.platform == 'win32':
p += '.exe'
if not os.path.exists(path):
print "installing %s in %s" % (p, bindir)
ox.net.save_url('http://firefogg.org/bin/%s.%s' % (p, osname), path)
os.chmod(path, 0755)
def move(self):
def format_underscores(string):
return re.sub('^\.|\.$|:|/|\?|<|>', '_', string)
prefs = settings.preferences
prefix = os.path.join(os.path.expanduser(prefs['libraryPath']), 'Books/')
j = self.item.json()
current_path = self.fullpath()
if not os.path.exists(current_path):
logger.debug('file is missing. %s', current_path)
return
author = '; '.join([get_sort_name(a) for a in j.get('author', [])])
if not author:
author = 'Unknown Author'
title = j.get('title', 'Untitled')
extension = j['extension']
if len(title) > 100:
title = title[:100]
title = format_underscores(title)
author = format_underscores(author)
publisher = j.get('publisher')
if publisher:
extra = ', '.join(publisher)
else:
extra = ''
date = j.get('date')
if date and len(date) >= 4:
extra += ' ' + date[:4]
if extra:
title = '%s (%s)' % (title, extra.strip())
filename = '%s.%s' % (title, extension)
first = unicodedata.normalize('NFD', author[0].upper())[0].upper()
new_path = os.path.join(first, author, filename)
new_path = new_path.replace('\x00', '')
new_path = ox.decode_html(new_path)
if self.path == new_path:
return
h = ''
while os.path.exists(os.path.join(prefix, new_path)):
h = self.sha1[:len(h)+1]
filename = '%s.%s.%s' % (title, h, extension)
first = unicodedata.normalize('NFD', author[0].upper())[0].upper()
new_path = os.path.join(first, author, filename)
if current_path == os.path.join(prefix, new_path):
break
if self.path != new_path:
path = os.path.join(prefix, new_path)
ox.makedirs(os.path.dirname(path))
shutil.move(current_path, path)
self.path = new_path
self.save()
def save_chunk(self, chunk, chunk_id=-1, done=False):
if self.uploading:
if not self.file:
self.file.name = self.path('data.pdf')
ox.makedirs(os.path.dirname(self.file.path))
with open(self.file.path, 'w') as f:
f.write(chunk.read())
self.save()
else:
with open(self.file.path, 'a') as f:
f.write(chunk.read())
if done:
self.uploading = False
self.save()
return True
return False
def render_PUT(self, request):
if request.path.startswith('/upload'):
parts = request.path.split('/')
oshash = parts[-1]
if len(oshash) == 16:
path = self.media_path(oshash)
ox.makedirs(os.path.dirname(path))
with open(path, 'wb') as f:
shutil.copyfileobj(request.content, f)
self.update_status(oshash, 'done')
self.upload.put(oshash)
return self.render_json(request, {
'path': path
})
request.setResponseCode(404)
return '404 unkown location'
def save_chunk(self, chunk, chunk_id=-1, done=False):
if self.uploading:
if not self.file:
self.file.name = self.path('data.pdf')
ox.makedirs(os.path.dirname(self.file.path))
with open(self.file.path, 'w') as f:
f.write(chunk.read())
self.save()
else:
with open(self.file.path, 'a') as f:
f.write(chunk.read())
if done:
self.uploading = False
self.save()
return True
return False
def get_icon(self, size=16):
path = self.path('icon%d.jpg' % size)
path = os.path.join(settings.MEDIA_ROOT, path)
if not os.path.exists(path):
folder = os.path.dirname(path)
ox.makedirs(folder)
if self.icon and os.path.exists(self.icon.path):
source = self.icon.path
max_size = min(self.icon.width, self.icon.height)
else:
source = os.path.join(settings.STATIC_ROOT, 'jpg/list256.jpg')
max_size = 256
if size < max_size:
extract.resize_image(source, path, size=size)
else:
path = source
return path
def get_icon(self, size=16):
path = self.path('icon%d.jpg' % size)
path = os.path.join(settings.MEDIA_ROOT, path)
if not os.path.exists(path):
folder = os.path.dirname(path)
ox.makedirs(folder)
if self.icon and os.path.exists(self.icon.path):
source = self.icon.path
max_size = min(self.icon.width, self.icon.height)
else:
source = os.path.join(settings.STATIC_ROOT, 'jpg/list256.jpg')
max_size = 256
if size < max_size:
extract.resize_image(source, path, size=size)
else:
path = source
return path
def save_chunk(self, chunk, chunk_id=-1, done=False):
if not self.available:
if not self.data:
name = 'data.%s' % self.info.get('extension', 'avi')
self.data.name = self.get_path(name)
ox.makedirs(os.path.dirname(self.data.path))
with open(self.data.path, 'w') as f:
f.write(chunk.read())
self.save()
else:
with open(self.data.path, 'a') as f:
f.write(chunk.read())
if done:
self.info.update(ox.avinfo(self.data.path))
self.parse_info()
self.save()
return True
return False
def save_chunk(self, chunk, chunk_id=-1, done=False):
if not self.available:
if not self.data:
name = 'data.%s' % self.info.get('extension', 'avi')
self.data.name = self.get_path(name)
ox.makedirs(os.path.dirname(self.data.path))
with open(self.data.path, 'w') as f:
f.write(chunk.read())
self.save()
else:
with open(self.data.path, 'a') as f:
f.write(chunk.read())
if done:
self.info.update(ox.avinfo(self.data.path))
self.parse_info()
self.save()
return True
return False
def save_chunk(self, chunk, chunk_id=-1, done=False):
if self.uploading:
if not self.file:
name = 'data.%s' % self.extension
self.file.name = self.path(name)
ox.makedirs(os.path.dirname(self.file.path))
with open(self.file.path, 'w') as f:
f.write(chunk.read())
self.save()
else:
with open(self.file.path, 'a') as f:
f.write(chunk.read())
if done:
self.uploading = False
self.get_ratio()
self.oshash = ox.oshash(self.file.path)
self.save()
return True
return False
def install():
if not os.path.exists(os.path.join(settings.updates_path, 'release.json')):
return True
if not os.path.exists(os.path.join(settings.config_path, 'release.json')):
return True
with open(os.path.join(settings.updates_path, 'release.json')) as fd:
release = json.load(fd)
old_version = current_version('openmedialibrary')
new_version = release['modules']['openmedialibrary']['version']
if verify(release) and old_version < new_version:
os.chdir(os.path.dirname(settings.base_dir))
for module in release['modules']:
if release['modules'][module]['version'] > current_version(module):
module_tar = os.path.join(settings.updates_path, release['modules'][module]['name'])
if os.path.exists(module_tar) and ox.sha1sum(module_tar) == release['modules'][module]['sha1']:
#tar fails if old platform is moved before extract
new = '%s_new' % module
ox.makedirs(new)
os.chdir(new)
tar = tarfile.open(module_tar)
tar.extractall()
tar.close()
os.chdir(os.path.dirname(settings.base_dir))
module_old = '%s_old' % module
if os.path.exists(module):
shutil.move(module, module_old)
shutil.move(os.path.join(new, module), module)
if os.path.exists(module_old):
shutil.rmtree(module_old)
shutil.rmtree(new)
else:
os.unlink(module_tar)
return False
shutil.copy(os.path.join(settings.updates_path, 'release.json'), os.path.join(settings.config_path, 'release.json'))
for cmd in [
['./ctl', 'stop'],
['./ctl', 'setup'],
['./ctl', 'postupdate', '-o', old_version, '-n', new_version]
]:
subprocess.call(cmd)
upgrade_app()
return True
return True
def save_chunk(self, chunk, chunk_id=-1, done=False):
if self.uploading:
if not self.file:
name = 'data.%s' % self.extension
self.file.name = self.path(name)
ox.makedirs(os.path.dirname(self.file.path))
with open(self.file.path, 'w') as f:
f.write(chunk.read())
self.save()
else:
with open(self.file.path, 'a') as f:
f.write(chunk.read())
if done:
self.uploading = False
self.get_info()
self.get_ratio()
self.oshash = ox.oshash(self.file.path)
self.save()
return True
return False
def save_file(self, content):
u = state.user()
f = File.get(self.id)
content_id = media.get_id(data=content)
if content_id != self.id:
logger.debug('INVALID CONTENT %s vs %s', self.id, content_id)
return False
if not f:
path = 'Downloads/%s.%s' % (self.id, self.info['extension'])
info = self.info.copy()
for key in ('mediastate', 'coverRatio', 'previewRatio'):
if key in info:
del info[key]
f = File.get_or_create(self.id, info, path=path)
path = self.get_path()
if not os.path.exists(path):
ox.makedirs(os.path.dirname(path))
with open(path, 'wb') as fd:
fd.write(content)
if u not in self.users:
self.add_user(u)
t = Transfer.get_or_create(self.id)
t.progress = 1
t.save()
self.added = datetime.utcnow()
Changelog.record(u, 'additem', self.id, f.info)
self.update()
f.move()
self.update_icons()
self.save()
trigger_event('transfer', {
'id': self.id, 'progress': 1
})
return True
else:
logger.debug('TRIED TO SAVE EXISTING FILE!!!')
t = Transfer.get_or_create(self.id)
t.progress = 1
t.save()
self.update()
return False
def save_chunk_stream(self, chunk, chunk_id=-1, done=False):
if not self.available:
config = settings.CONFIG['video']
stream, created = Stream.objects.get_or_create(
file=self,
resolution=max(config['resolutions']),
format=config['formats'][0])
if created:
stream.media.name = stream.path(stream.name())
ox.makedirs(os.path.dirname(stream.media.path))
with open(stream.media.path, 'w') as f:
f.write(chunk.read())
stream.save()
else:
with open(stream.media.path, 'a') as f:
#FIXME: should check that chunk_id/offset is right
f.write(chunk.read())
if done:
stream.available = True
stream.info = {}
stream.save()
return True
return False
def save_chunk_stream(self, chunk, chunk_id=-1, done=False):
if not self.available:
config = settings.CONFIG['video']
stream, created = Stream.objects.get_or_create(
file=self,
resolution=max(config['resolutions']),
format=config['formats'][0])
if created:
stream.media.name = stream.path(stream.name())
ox.makedirs(os.path.dirname(stream.media.path))
with open(stream.media.path, 'w') as f:
f.write(chunk.read())
stream.save()
else:
with open(stream.media.path, 'a') as f:
#FIXME: should check that chunk_id/offset is right
f.write(chunk.read())
if done:
stream.available = True
stream.info = {}
stream.save()
return True
return False
def update_icon(self):
frames = []
if not self.poster_frames:
items = self.get_items(self.user).filter(rendered=True)
if items.count():
poster_frames = []
for i in range(0, items.count(), max(1, int(items.count()/4))):
poster_frames.append({
'item': items[int(i)].itemId,
'position': items[int(i)].poster_frame
})
self.poster_frames = tuple(poster_frames)
self.save()
for i in self.poster_frames:
from item.models import Item
qs = Item.objects.filter(itemId=i['item'])
if qs.count() > 0:
frame = qs[0].frame(i['position'])
if frame:
frames.append(frame)
self.icon.name = self.path('icon.jpg')
icon = self.icon.path
if frames:
while len(frames) < 4:
frames += frames
folder = os.path.dirname(icon)
ox.makedirs(folder)
for f in glob("%s/icon*.jpg" % folder):
os.unlink(f)
cmd = [
settings.LIST_ICON,
'-f', ','.join(frames),
'-o', icon
]
p = subprocess.Popen(cmd)
p.wait()
self.save()
def frame(video, frame, position, height=128, redo=False):
'''
params:
video input
frame output
position as float in seconds
height of frame
redo boolean to extract file even if it exists
'''
if exists(video):
folder = os.path.dirname(frame)
if redo or not exists(frame):
ox.makedirs(folder)
#wafaa commented
'''if video.endswith('.mp4'):
cmd = [
AVCONV, '-y',
'-ss', str(position),
'-i', video,
'-an', '-vframes', '1',
'-vf', 'scale=-1:%s' % height
]
if not frame.endswith('.png'):
cmd += ['-f', 'mjpeg']
cmd += [frame]
else:
cmd = ['oxframe', '-i', video, '-o', frame,
'-p', str(position), '-y', str(height)]
'''
#wafaa
if video.endswith('.png'):
#wafaa
#x = 'x'
#frame_height = x + str(height)
cmd = [convert, video, '-thumbnail', 'x128', frame]
run_command(cmd)
def run_import(options=None):
options = options or {}
logger.debug('run_import')
prefs = settings.preferences
prefix = os.path.expanduser(options.get('path', prefs['importPath']))
if os.path.islink(prefix):
prefix = os.path.realpath(prefix)
if not prefix[-1] == os.sep:
prefix += os.sep
prefix_books = os.path.join(os.path.expanduser(prefs['libraryPath']),
'Books' + os.sep)
prefix_imported = os.path.join(prefix_books, 'Imported' + os.sep)
if prefix_books.startswith(prefix) or prefix.startswith(prefix_books):
error = 'invalid path'
elif not os.path.exists(prefix):
error = 'path not found'
elif not os.path.isdir(prefix):
error = 'path must be a folder'
else:
error = None
if error:
trigger_event(
'activity', {
'activity': 'import',
'progress': [0, 0],
'status': {
'code': 404,
'text': error
}
})
state.activity = {}
return
listname = options.get('list')
if listname:
listitems = []
assert isinstance(prefix, str)
books = []
count = 0
for root, folders, files in os.walk(prefix):
for f in files:
if not state.tasks.connected:
return
#if f.startswith('._') or f == '.DS_Store':
if f.startswith('.'):
continue
f = os.path.join(root, f)
ext = f.split('.')[-1]
if ext in extensions:
books.append(f)
count += 1
if state.activity.get('cancel'):
state.activity = {}
return
if count % 1000 == 0:
state.activity = {
'activity': 'import',
'path': prefix,
'progress': [0, count],
}
trigger_event('activity', state.activity)
state.activity = {
'activity': 'import',
'path': prefix,
'progress': [0, len(books)],
}
trigger_event('activity', state.activity)
position = 0
added = 0
last = 0
for f in ox.sorted_strings(books):
position += 1
if not os.path.exists(f):
continue
with db.session():
id = media.get_id(f)
file = File.get(id)
if not file:
f_import = f
f = f.replace(prefix, prefix_imported)
ox.makedirs(os.path.dirname(f))
if options.get('mode') == 'move':
shutil.move(f_import, f)
else:
shutil.copy(f_import, f)
file = add_file(id, f, prefix_books, f_import)
file.move()
added += 1
if listname:
listitems.append(file.item.id)
if time.time() - last > 5:
last = time.time()
state.activity = {
'activity': 'import',
'progress': [position, len(books)],
'path': prefix,
'added': added,
}
trigger_event('activity', state.activity)
if state.activity.get('cancel'):
state.activity = {}
return
with db.session():
if listname and listitems:
l = List.get(settings.USER_ID, listname)
if l:
l.add_items(listitems)
trigger_event(
'activity', {
'activity': 'import',
'progress': [position, len(books)],
'path': prefix,
'status': {
'code': 200,
'text': ''
},
'added': added,
})
state.activity = {}
remove_empty_folders(prefix_books)
if options.get('mode') == 'move':
remove_empty_folders(prefix)
def stream(video, target, profile, info):
if not os.path.exists(target):
ox.makedirs(os.path.dirname(target))
'''
WebM look into
lag
mb_static_threshold
qmax/qmin
rc_buf_aggressivity=0.95
token_partitions=4
level / speedlevel
bt?
H264, should bitrates be a bit lower? other stuff possible?
'''
profile, format = profile.split('.')
bpp = 0.17
if profile == '1080p':
height = 1080
audiorate = 48000
audioquality = 6
audiobitrate = None
audiochannels = None
if profile == '720p':
height = 720
audiorate = 48000
audioquality = 5
audiobitrate = None
audiochannels = None
if profile == '480p':
height = 480
audiorate = 44100
audioquality = 3
audiobitrate = None
audiochannels = 2
elif profile == '432p':
height = 432
audiorate = 44100
audioquality = 3
audiobitrate = None
audiochannels = 2
elif profile == '360p':
height = 360
audiorate = 44100
audioquality = 1
audiobitrate = None
audiochannels = 1
elif profile == '288p':
height = 288
audiorate = 44100
audioquality = 0
audiobitrate = None
audiochannels = 1
elif profile == '240p':
height = 240
audiorate = 44100
audioquality = 0
audiobitrate = None
audiochannels = 1
elif profile == '144p':
height = 144
audiorate = 22050
audioquality = -1
audiobitrate = '22k'
audiochannels = 1
else:
height = 96
audiorate = 22050
audioquality = -1
audiobitrate = '22k'
audiochannels = 1
if info['video'] and 'display_aspect_ratio' in info['video'][0]:
fps = AspectRatio(info['video'][0]['framerate'])
fps = min(30, float(fps))
dar = AspectRatio(info['video'][0]['display_aspect_ratio'])
width = int(dar * height)
width += width % 2
bitrate = height*width*fps*bpp/1000
aspect = dar.ratio
#use 1:1 pixel aspect ratio if dar is close to that
if abs(width/height - dar) < 0.02:
aspect = '%s:%s' % (width, height)
video_settings = [
'-vb', '%dk'%bitrate,
'-aspect', aspect,
#'-vf', 'yadif',
'-vf', 'hqdn3d,scale=%s:%s'%(width, height),
'-g', '%d' % int(fps*5),
]
if format == 'webm':
video_settings += [
'-deadline', 'good',
'-cpu-used', '0',
'-lag-in-frames', '16',
'-auto-alt-ref', '1',
]
if format == 'mp4':
#quicktime does not support bpyramid
'''
video_settings += [
'-vcodec', 'libx264',
'-flags', '+loop+mv4',
'-cmp', '256',
'-partitions', '+parti4x4+parti8x8+partp4x4+partp8x8+partb8x8',
'-me_method', 'hex',
'-subq', '7',
'-trellis', '1',
'-refs', '5',
'-bf', '3',
'-flags2', '+bpyramid+wpred+mixed_refs+dct8x8',
'-coder', '1',
'-me_range', '16',
'-keyint_min', '25', #FIXME: should this be related to fps?
'-sc_threshold','40',
'-i_qfactor', '0.71',
'-qmin', '10', '-qmax', '51',
'-qdiff', '4'
]
'''
video_settings += [
'-vcodec', 'libx264',
'-flags', '+loop+mv4',
'-cmp', '256',
'-partitions', '+parti4x4+parti8x8+partp4x4+partp8x8+partb8x8',
'-me_method', 'hex',
'-subq', '7',
'-trellis', '1',
'-refs', '5',
'-bf', '0',
'-flags2', '+mixed_refs',
'-coder', '0',
'-me_range', '16',
'-sc_threshold', '40',
'-i_qfactor', '0.71',
'-qmin', '10', '-qmax', '51',
'-qdiff', '4'
]
else:
video_settings = ['-vn']
if info['audio']:
audio_settings = ['-ar', str(audiorate), '-aq', str(audioquality)]
if audiochannels and 'channels' in info['audio'][0] \
and info['audio'][0]['channels'] > audiochannels:
audio_settings += ['-ac', str(audiochannels)]
if audiobitrate:
audio_settings += ['-ab', audiobitrate]
if format == 'mp4':
audio_settings += ['-acodec', 'libvo_aacenc']
else:
audio_settings += ['-acodec', 'libvorbis']
else:
audio_settings = ['-an']
cmd = [AVCONV, '-y', '-i', video, '-threads', '4'] \
+ audio_settings \
+ video_settings
if format == 'webm':
cmd += ['-f', 'webm', target]
elif format == 'mp4':
#mp4 needs postprocessing(qt-faststart), write to temp file
cmd += ["%s.mp4" % target]
else:
cmd += [target]
#print cmd
p = subprocess.Popen(cmd, stdin=subprocess.PIPE,
stdout=open('/dev/null', 'w'),
stderr=subprocess.STDOUT)
p.communicate()
if p.returncode != 0:
t = "%s.mp4" % target if format == 'mp4' else target
if os.path.exists(t):
os.unlink(t)
return False
if format == 'mp4':
cmd = ['qt-faststart', "%s.mp4" % target, target]
#print cmd
p = subprocess.Popen(cmd, stdin=subprocess.PIPE,
stdout=open('/dev/null', 'w'),
stderr=subprocess.STDOUT)
p.communicate()
os.unlink("%s.mp4" % target)
return True
def image_cmd(video, target, profile, info):
if not os.path.exists(target):
ox.makedirs(os.path.dirname(target))
'''
look into
lag
mb_static_threshold
qmax/qmin
rc_buf_aggressivity=0.95
token_partitions=4
level / speedlevel
bt?
'''
profile, format = profile.split('.')
bpp = 0.17
if profile == '1080p':
height = 1080
audiorate = 48000
audioquality = 6
audiobitrate = None
audiochannels = None
if profile == '720p':
height = 720
audiorate = 48000
audioquality = 5
audiobitrate = None
audiochannels = None
if profile == '480p':
height = 480
audiorate = 44100
audioquality = 3
audiobitrate = None
audiochannels = 2
elif profile == '432p':
height = 432
audiorate = 44100
audioquality = 2
audiobitrate = None
audiochannels = 2
elif profile == '360p':
height = 360
audiorate = 44100
audioquality = 1
audiobitrate = None
audiochannels = 1
elif profile == '288p':
height = 288
audiorate = 44100
audioquality = 0
audiobitrate = None
audiochannels = 1
elif profile == '240p':
height = 240
audiorate = 44100
audioquality = 0
audiobitrate = None
audiochannels = 1
elif profile == '144p':
height = 144
audiorate = 22050
audioquality = -1
audiobitrate = '22k'
audiochannels = 1
else:
height = 96
audiorate = 22050
audioquality = -1
audiobitrate = '22k'
audiochannels = 1
cmd = [command('convert'), video, '-scale', 'x720', target]
#cmd = [command('convert'), video, '-scale', 'x1080', target]
#x = 'x'
#res_val = x + str(profile)
#cmd = [command('convert'), video, '-resize', res_val, target]
print "cmd print %s" % cmd
return cmd
def video_cmd(video, target, profile, info):
if not os.path.exists(target):
ox.makedirs(os.path.dirname(target))
'''
look into
lag
mb_static_threshold
qmax/qmin
rc_buf_aggressivity=0.95
token_partitions=4
level / speedlevel
bt?
'''
profile, format = profile.split('.')
bpp = 0.17
if profile == '1080p':
height = 1080
audiorate = 48000
audioquality = 6
audiobitrate = None
audiochannels = None
if profile == '720p':
height = 720
audiorate = 48000
audioquality = 5
audiobitrate = None
audiochannels = None
if profile == '480p':
height = 480
audiorate = 44100
audioquality = 3
audiobitrate = None
audiochannels = 2
elif profile == '432p':
height = 432
audiorate = 44100
audioquality = 2
audiobitrate = None
audiochannels = 2
elif profile == '360p':
height = 360
audiorate = 44100
audioquality = 1
audiobitrate = None
audiochannels = 1
elif profile == '288p':
height = 288
audiorate = 44100
audioquality = 0
audiobitrate = None
audiochannels = 1
elif profile == '240p':
height = 240
audiorate = 44100
audioquality = 0
audiobitrate = None
audiochannels = 1
elif profile == '144p':
height = 144
audiorate = 22050
audioquality = -1
audiobitrate = '22k'
audiochannels = 1
else:
height = 96
audiorate = 22050
audioquality = -1
audiobitrate = '22k'
audiochannels = 1
if info['video'] and 'display_aspect_ratio' in info['video'][0]:
dar = AspectRatio(info['video'][0]['display_aspect_ratio'])
fps = AspectRatio(info['video'][0]['framerate'])
width = int(dar * height)
width += width % 2
extra = []
if fps == 50:
fps = 25
extra += ['-r', '25']
if fps == 60:
fps = 30
extra += ['-r', '30']
fps = min(float(fps), 30)
bitrate = height*width*fps*bpp/1000
aspect = dar.ratio
#use 1:1 pixel aspect ratio if dar is close to that
if abs(width/height - dar) < 0.02:
aspect = '%s:%s' % (width, height)
video_settings = [
'-vb', '%dk'%bitrate,
'-aspect', aspect,
'-g', '%d' % int(fps*5),
'-vf', 'yadif,hqdn3d,scale=%s:%s'%(width, height),
] + extra
if format == 'webm':
video_settings += [
'-deadline', 'good',
'-cpu-used', '0',
'-lag-in-frames', '16',
'-auto-alt-ref', '1',
]
if format == 'mp4':
#quicktime does not support bpyramid
'''
video_settings += [
'-vcodec', 'libx264',
'-flags', '+loop+mv4',
'-cmp', '256',
'-partitions', '+parti4x4+parti8x8+partp4x4+partp8x8+partb8x8',
'-me_method', 'hex',
'-subq', '7',
'-trellis', '1',
'-refs', '5',
'-bf', '3',
'-flags2', '+bpyramid+wpred+mixed_refs+dct8x8',
'-coder', '1',
'-me_range', '16',
'-keyint_min', '25', #FIXME: should this be related to fps?
'-sc_threshold','40',
'-i_qfactor', '0.71',
'-qmin', '10', '-qmax', '51',
'-qdiff', '4'
]
'''
video_settings += [
'-vcodec', 'libx264',
'-flags', '+loop+mv4',
'-cmp', '256',
'-partitions', '+parti4x4+parti8x8+partp4x4+partp8x8+partb8x8',
'-me_method', 'hex',
'-subq', '7',
'-trellis', '1',
'-refs', '5',
'-bf', '0',
'-flags2', '+mixed_refs',
'-coder', '0',
'-me_range', '16',
'-sc_threshold', '40',
'-i_qfactor', '0.71',
'-qmin', '10', '-qmax', '51',
'-qdiff', '4'
]
video_settings += ['-map', '0:%s,0:0'%info['video'][0]['id']]
else:
video_settings = ['-vn']
if info['audio']:
if video_settings == ['-vn'] or not info['video']:
n = 0
else:
n = 1
video_settings += ['-map', '0:%s,0:%s' % (info['audio'][0]['id'], n)]
audio_settings = ['-ar', str(audiorate), '-aq', str(audioquality)]
ac = info['audio'][0].get('channels', audiochannels)
if ac:
ac = min(ac, audiochannels)
else:
ac = audiochannels
audio_settings += ['-ac', str(ac)]
if audiobitrate:
audio_settings += ['-ab', audiobitrate]
if format == 'mp4':
audio_settings += ['-acodec', 'libvo_aacenc']
else:
audio_settings += ['-acodec', 'libvorbis']
else:
audio_settings = ['-an']
'''wafaa commented the following command related to video
cmd = [command('ffmpeg'), '-y', '-i', video, '-threads', '4'] \
+ audio_settings \
+ video_settings
'''
'''wafaa added the following command instead of the previous commented part
to convert any image format to jpg with resolution 480p for height and added
the target to the cmd instead of appending it in next lines. Also, all conditions
that are checking file extensions are commented
We need to add cmd to keep orig image so can use it later if needed
if we needed and metadata cuz jpg is loosly format'''
#cmd = [command('convert'), video, '-resize', 'x480', target]
'''
#Wrong cuz the second command will override the 1st cmd
cmd = [command('cp'), video, target]
cmd += [command('|')]
cmd += [command('convert'), video, '-resize', 'x480', '-define', 'webp:lossless=true', target]
'''
cmd = [command('convert'), video, '-resize', 'x480', target]
#cmd = [command('convert'), video, '-resize', 'x480', '-define', 'webp:lossless=true', target]
'''The following cmd cwebp is to convert images to the WebP format'''
#cmd = [command('cwebp'), video, '-resize', 'x480', '-define', 'webp:lossless=true', '-o', target]
#cmd = [command('cwebp'), video, '-o', target]
'''The following cmd dwebp is to convert images from the WebP format'''
#cmd = [command('dwebp'), video, '-resize', 'x480', '-define', 'webp:lossless=true', '-o', target]
#cmd2 = [command('cp'), video, target]
print "cmd print %s" % cmd
''' wafaa commented the following lines
if format == 'webm':
cmd += ['-f', 'webm', target]
elif format == 'mp4':
#mp4 needs postprocessing(qt-faststart), write to temp file
cmd += ["%s.mp4" % target]
elif format == 'jpg':
cmd += ["%s.jpg" % target]
else:
cmd += [target]
print "cmd print %cmd" % cmd
'''
return cmd
def run_import(options=None):
options = options or {}
logger.debug("run_import")
prefs = settings.preferences
prefix = os.path.expanduser(options.get("path", prefs["importPath"]))
if os.path.islink(prefix):
prefix = os.path.realpath(prefix)
if not prefix[-1] == os.sep:
prefix += os.sep
prefix_books = os.path.join(os.path.expanduser(prefs["libraryPath"]), "Books" + os.sep)
prefix_imported = os.path.join(prefix_books, "Imported" + os.sep)
if prefix_books.startswith(prefix) or prefix.startswith(prefix_books):
error = "invalid path"
elif not os.path.exists(prefix):
error = "path not found"
elif not os.path.isdir(prefix):
error = "path must be a folder"
else:
error = None
if error:
trigger_event("activity", {"activity": "import", "progress": [0, 0], "status": {"code": 404, "text": error}})
state.activity = {}
return
listname = options.get("list")
if listname:
listitems = []
assert isinstance(prefix, str)
books = []
count = 0
for root, folders, files in os.walk(prefix):
for f in files:
if not state.tasks.connected:
return
# if f.startswith('._') or f == '.DS_Store':
if f.startswith("."):
continue
f = os.path.join(root, f)
ext = f.split(".")[-1]
if ext in extensions:
books.append(f)
count += 1
if state.activity.get("cancel"):
state.activity = {}
return
if count % 1000 == 0:
state.activity = {"activity": "import", "path": prefix, "progress": [0, count]}
trigger_event("activity", state.activity)
state.activity = {"activity": "import", "path": prefix, "progress": [0, len(books)]}
trigger_event("activity", state.activity)
position = 0
added = 0
last = 0
for f in ox.sorted_strings(books):
position += 1
if not os.path.exists(f):
continue
with db.session():
id = media.get_id(f)
file = File.get(id)
if not file:
f_import = f
f = f.replace(prefix, prefix_imported)
ox.makedirs(os.path.dirname(f))
if options.get("mode") == "move":
shutil.move(f_import, f)
else:
shutil.copy(f_import, f)
file = add_file(id, f, prefix_books, f_import)
file.move()
added += 1
if listname:
listitems.append(file.item.id)
if time.time() - last > 5:
last = time.time()
state.activity = {"activity": "import", "progress": [position, len(books)], "path": prefix, "added": added}
trigger_event("activity", state.activity)
if state.activity.get("cancel"):
state.activity = {}
return
with db.session():
if listname and listitems:
l = List.get(settings.USER_ID, listname)
if l:
l.add_items(listitems)
trigger_event(
"activity",
{
"activity": "import",
"progress": [position, len(books)],
"path": prefix,
"status": {"code": 200, "text": ""},
"added": added,
},
)
state.activity = {}
remove_empty_folders(prefix_books)
if options.get("mode") == "move":
remove_empty_folders(prefix)
def stream(video, target, profile, info, avconv=None):
if not os.path.exists(target):
ox.makedirs(os.path.dirname(target))
'''
WebM look into
lag
mb_static_threshold
qmax/qmin
rc_buf_aggressivity=0.95
token_partitions=4
level / speedlevel
bt?
H264, should bitrates be a bit lower? other stuff possible?
'''
profile, format = profile.split('.')
bpp = 0.17
if profile == '1080p':
height = 1080
audiorate = 48000
audioquality = 6
audiobitrate = None
audiochannels = None
if profile == '720p':
height = 720
audiorate = 48000
audioquality = 5
audiobitrate = None
audiochannels = None
if profile == '480p':
height = 480
audiorate = 44100
audioquality = 3
audiobitrate = None
audiochannels = 2
elif profile == '432p':
height = 432
audiorate = 44100
audioquality = 3
audiobitrate = None
audiochannels = 2
elif profile == '360p':
height = 360
audiorate = 44100
audioquality = 1
audiobitrate = None
audiochannels = 1
elif profile == '288p':
height = 288
audiorate = 44100
audioquality = 0
audiobitrate = None
audiochannels = 1
elif profile == '240p':
height = 240
audiorate = 44100
audioquality = 0
audiobitrate = None
audiochannels = 1
elif profile == '144p':
height = 144
audiorate = 22050
audioquality = -1
audiobitrate = '22k'
audiochannels = 1
else:
height = 96
audiorate = 22050
audioquality = -1
audiobitrate = '22k'
audiochannels = 1
#wafaa
x = 'x'
res_val = x + str(height)
cmd = [convert, video, '-resize', res_val, target]
#cmd = [convert, video, target]
'''
if not avconv:
avconv = AVCONV
cmd = [avconv, '-y', '-i', video, '-threads', '4'] \
+ audio_settings \
+ video_settings
if format == 'webm':
cmd += ['-f', 'webm', target]
elif format == 'mp4':
#mp4 needs postprocessing(qt-faststart), write to temp file
cmd += ["%s.mp4" % target]
else:
cmd += [target]
'''
#print cmd
p = subprocess.Popen(cmd,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
stdout, stderr = p.communicate()
if p.returncode != 0:
t = "%s.mp4" % target if format == 'mp4' else target
if os.path.exists(t):
os.unlink(t)
return False, stdout
if format == 'mp4':
cmd = ['qt-faststart', "%s.mp4" % target, target]
#print cmd
p = subprocess.Popen(cmd,
stdin=subprocess.PIPE,
stdout=open('/dev/null', 'w'),
stderr=subprocess.STDOUT)
p.communicate()
os.unlink("%s.mp4" % target)
return True, None
