def encode_video(input_file_path, output_file_name):
"""
We first obtain the video via http, encode it, and then upload it back
"""
server_url = get_config_option("Download Settings", "server_url")
root_dir = get_config_option("FTP Settings" , "root_dir")
file_path, input_file_name = os.path.split(input_file_path)
url = urljoin(server_url, input_file_path.lstrip("\\"))
logging.info("Downloading {0}".format(url))
try:
urllib.urlretrieve(url, input_file_name)
logging.info("Finished downloading {0}".format(url))
except IOError:
delay = random.randrange(40)
logging.info("Seems like server is overloaded. Retrying in {0} seconds.".format(delay))
time.sleep(delay)
urllib.urlretrieve(url, input_file_name)
logging.info("Finished downloading {0}".format(url))
ffmpeg_args = ["ffmpeg", "-y", "-i", input_file_name, "-an","-c:v", "libx264", "-b:v", "1024k", output_file_name]
if sys.platform.startswith('linux'):
logging.info("Linux detected. Modifying command")
ffmpeg_args[0] = 'avconv'
ffmpeg_args = " ".join(ffmpeg_args)
if not check_call(ffmpeg_args, shell=True):
ftp = FTP(host=get_config_option("FTP Settings" , "host"), user=get_config_option("FTP Settings", "username") , passwd=get_config_option("FTP Settings" , "password"))
ftp.cwd(root_dir)
full_path = re.split("/", file_path)
for d in full_path:
try:
ftp.cwd(d)
except error_perm:
ftp.mkd(d)
ftp.cwd(d)
f = open(output_file_name, 'rb')
logging.info("Starting upload of {0} as {1}".format(output_file_name, input_file_name))
ftp.storbinary('STOR ' + input_file_name, f)
logging.info("Finished upload of {0} as {1}".format(output_file_name, input_file_name))
ftp.quit()
f.close()
#Remove files after encoding them
try:
os.remove(input_file_name)
except OSError:
logging.info("Failed to remove {0}".format(input_file_name))
else:
logging.info("Removed {0}".format(input_file_name))
try:
os.remove(output_file_name)
except OSError:
logging.info("Failed to remove {0}".format(output_file_name))
else:
logging.info("Removed {0}".format(output_file_name))
def __init__( self, path ):
self._steps = []
config = ConfigParser.SafeConfigParser()
config.read( path )
self._path = path
self._platforms = []
found = True
stepNum = 0
while found:
sec = "Step{0}".format( stepNum )
if not config.has_section( sec ):
found = False
continue
stepNum += 1
installscript = utils.makeAbsolutePath( utils.get_config_option( config, None, "installscript", None, sec ), os.path.dirname( path ) )
checkerTestDir = utils.get_config_option( config, None, "checkerTestDir", None, sec )
checkerTestDir = utils.makeAbsolutePath( checkerTestDir, os.path.dirname( path ) ) if checkerTestDir else ""
timeout = int( utils.get_config_option( config, None, "timeout", 60 * 60, sec ) )
self._steps.append( Step( installscript, checkerTestDir, timeout ) )
self._name = utils.get_config_option( config, None, "name", utils.basename( path ) )
self._targetDirectory = utils.get_config_option( config, None, "targetDirectory", "" )
self._maintenanceToolLocation = utils.get_config_option( config, None, "maintenanceToolLocation", "" )
platforms = utils.get_config_option( config, None, "platforms" )
if platforms != None:
self._platforms = platforms.split( ',' )
def __init__( self, path ):
self._steps = []
config = ConfigParser.SafeConfigParser()
config.read( path )
self._path = path
self._platforms = []
found = True
stepNum = 0
while found:
sec = "Step{0}".format( stepNum )
if not config.has_section( sec ):
found = False
continue
stepNum += 1
installscript = utils.makeAbsolutePath( utils.get_config_option( config, None, "installscript", None, sec ), os.path.dirname( path ) )
checkerTestDir = utils.get_config_option( config, None, "checkerTestDir", None, sec )
checkerTestDir = utils.makeAbsolutePath( checkerTestDir, os.path.dirname( path ) ) if checkerTestDir else ""
timeout = int( utils.get_config_option( config, None, "timeout", 60 * 60, sec ) )
self._steps.append( Step( installscript, checkerTestDir, timeout ) )
self._name = utils.get_config_option( config, None, "name", utils.basename( path ) )
self._targetDirectory = utils.get_config_option( config, None, "targetDirectory", "" )
self._maintenanceToolLocation = utils.get_config_option( config, None, "maintenanceToolLocation", "" )
platforms = utils.get_config_option( config, None, "platforms" )
if platforms != None:
self._platforms = platforms.split( ',' )
def generate_jobs(abs_dir):
ftp = get_ftp_connection()
if abs_dir is not None:
root = abs_dir
num_jobs = 1
#TODO: Actually implement the date feature
for root, dirs, files in os.walk(root):
if len(files):
file_list = []
try:
ftp.cwd("/" + get_config_option("FTP Settings" , "root_dir"))
formatted_dir = os.path.splitdrive(root)[1]
formatted_dir = formatted_dir.replace("\\", "/")
formatted_dir = formatted_dir[1:] #We need to cut off the leading slash
ftp.cwd(formatted_dir)
file_list = ftp.nlst()
except error_perm:
# Occurs if there is no directory of the name we are looking for
print (formatted_dir + "NOT FOUND")
pass
except error_temp:
print "Seems like FTP connection was lost. Reconnecting"
ftp = get_ftp_connection()
for f in files:
if f.endswith('.avi'):
the_file = os.path.join(root, f)
the_path = os.path.splitdrive((the_file))
final_path = the_path[1].replace("\\", "/")
if not clean_old and num_jobs % 10 == 0:
print "Sleeping for 10 minutes"
time.sleep(600)
if f not in file_list:
if not clean_old:
print("{0} -> Adding {1} to list" .format(num_jobs, final_path))
encode_video.delay(final_path, "OUTPUT_" + f)
num_jobs += 1 #Dont increment unless we actually added a job
else:
print("{0} ALREADY EXISTS REMOTELY. REMOVING LOCALLY." .format(final_path))
os.remove(os.path.join(root, f))
elif not os.listdir(root):
print("{0} is empty. Removing directory").format(root)
os.rmdir(root)
ftp.quit()
try:
options.vmx = args[0]
except IndexError:
options.vmx = None
options.ostype = None
options.guestTempDir = None
config = ConfigParser.SafeConfigParser()
if options.config:
print("** Reading config: " + options.config )
config.read( options.config )
else:
print("** No config given")
options.username = utils.get_config_option( config, options.username, "username", "nokia" )
options.password = utils.get_config_option( config, options.password, "password", "nokia" )
options.script = utils.get_config_option( config, options.script, "script", "guest.py" )
options.python = utils.get_config_option( config, options.python, "python", "c:/python26/python.exe" )
options.snapshot = utils.get_config_option( config, options.snapshot, "snapshot", "base" )
options.vmrun = utils.get_config_option( config, options.vmrun, "vmrun" )
options.vmx = utils.get_config_option( config, options.vmx, "vmx" )
options.guestTempDir = utils.get_config_option( config, options.guestTempDir, "tempDir", "c:\\windows\\temp" )
options.ostype = utils.get_config_option( config, options.ostype, "os", "windows" )
check_option( options.vmx, "vmx-file" )
# Search the PATH and the a few extra locations for 'vmrun'
if not options.vmrun:
options.vmrun = utils.findVMRun()
def get_ftp_connection():
ftp = FTP(host=get_config_option("FTP Settings" , "host"), user=get_config_option("FTP Settings", "username") , passwd=get_config_option("FTP Settings" , "password"))
return ftp
def fromVMRunAndPath(vmrun, path):
config = ConfigParser.SafeConfigParser()
config.read(path)
hostType = utils.get_config_option(config, None, "type", "", "Host")
hostLocation = utils.get_config_option(config, None, "location", "",
"Host")
hostUsername = utils.get_config_option(config, None, "username", "",
"Host")
hostPassword = utils.get_config_option(config, None, "password", "",
"Host")
vmxVal = utils.get_config_option(config, None, "vmx")
if not len(hostType) == 0:
vmx = utils.makeAbsolutePath(vmxVal, os.path.dirname(path))
else:
vmx = vmxVal
username = utils.get_config_option(config, None, "username", "nokia")
password = utils.get_config_option(config, None, "password", "nokia")
tempDir = utils.get_config_option(config, None, "tempDir",
"c:\\windows\\temp")
ostype = utils.get_config_option(config, None, "os", "windows")
vm = VirtualMachine(vmrun, vmx, username, password, tempDir, ostype)
vm._name = utils.get_config_option(config, None, "name",
utils.basename(path))
vm._snapshot = utils.get_config_option(config, None, "snapshot", "base")
vm._python = utils.get_config_option(config, None, "python",
"c:/python26/python.exe")
vm._hostType = hostType
vm._hostLocation = hostLocation
vm._hostUsername = hostUsername
vm._hostPassword = hostPassword
return vm
try:
options.vmx = args[0]
except IndexError:
options.vmx = None
options.ostype = None
options.guestTempDir = None
config = ConfigParser.SafeConfigParser()
if options.config:
print("** Reading config: " + options.config )
config.read( options.config )
else:
print("** No config given")
options.username = utils.get_config_option( config, options.username, "username", "nokia" )
options.password = utils.get_config_option( config, options.password, "password", "nokia" )
options.script = utils.get_config_option( config, options.script, "script", "guest.py" )
options.python = utils.get_config_option( config, options.python, "python", "c:/python26/python.exe" )
options.snapshot = utils.get_config_option( config, options.snapshot, "snapshot", "base" )
options.vmrun = utils.get_config_option( config, options.vmrun, "vmrun" )
options.vmx = utils.get_config_option( config, options.vmx, "vmx" )
options.guestTempDir = utils.get_config_option( config, options.guestTempDir, "tempDir", "c:\\windows\\temp" )
options.ostype = utils.get_config_option( config, options.ostype, "os", "windows" )
check_option( options.vmx, "vmx-file" )
# Search the PATH and the a few extra locations for 'vmrun'
if not options.vmrun:
options.vmrun = utils.findVMRun()
optionParser.add_option("-c", "--checkerInstallation", dest="checkerInstallation", help="checker installation to use for post-installation checks", metavar="CHECKERINSTALLATION" )
(options, args) = optionParser.parse_args()
try:
configpath = utils.makeAbsolutePath( args[0], os.getcwd() )
except IndexError:
optionParser.print_usage( sys.stderr )
sys.exit( 1 )
config = ConfigParser.SafeConfigParser()
config.read( configpath )
#make an unary functor to create absolute paths
abspath = partial( utils.makeAbsolutePath, relativeTo=os.path.dirname( configpath ) )
vmrun = utils.get_config_option( config, options.vmrun, "vmrun", utils.findVMRun() )
useGui = utils.get_config_option( config, None, "gui", "true" ).lower() == "true"
createErrorSnapshots = utils.get_config_option( config, None, "createErrorSnapshots", "true" ).lower() == "true"
hostType = utils.get_config_option( config, None, "type", "", "Host" )
hostLocation = utils.get_config_option( config, None, "location", "", "Host" )
hostUsername = utils.get_config_option( config, None, "username", "", "Host" )
hostPassword = utils.get_config_option( config, None, "password", "", "Host" )
cdashHost = utils.get_config_option( config, None, "host", "", "CDash" )
cdashLocation = utils.get_config_option( config, None, "location", "", "CDash" )
cdashProject = utils.get_config_option( config, None, "project", "", "CDash" )
if vmrun == None:
die( "Could not find vmrun executable. Please specify it in the config file (vmrun=...) or via the --vmrun option" )
checkerInstallation = utils.get_config_option( config, options.checkerInstallation, "checkerInstallation" )
def fromVMRunAndPath( vmrun, path ):
config = ConfigParser.SafeConfigParser()
config.read( path )
hostType = utils.get_config_option( config, None, "type", "", "Host" )
hostLocation = utils.get_config_option( config, None, "location", "", "Host" )
hostUsername = utils.get_config_option( config, None, "username", "", "Host" )
hostPassword = utils.get_config_option( config, None, "password", "", "Host" )
vmxVal = utils.get_config_option( config, None, "vmx" )
if not len( hostType ) == 0:
vmx = utils.makeAbsolutePath( vmxVal, os.path.dirname( path ) )
else:
vmx = vmxVal
username = utils.get_config_option( config, None, "username", "nokia" )
password = utils.get_config_option( config, None, "password", "nokia" )
tempDir = utils.get_config_option( config, None, "tempDir", "c:\\windows\\temp" )
ostype = utils.get_config_option( config, None, "os", "windows" )
vm = VirtualMachine( vmrun, vmx, username, password, tempDir, ostype )
vm._name = utils.get_config_option( config, None, "name", utils.basename( path ) )
vm._snapshot = utils.get_config_option( config, None, "snapshot", "base" )
vm._python = utils.get_config_option( config, None, "python", "c:/python26/python.exe" )
vm._hostType = hostType
vm._hostLocation = hostLocation
vm._hostUsername = hostUsername
vm._hostPassword = hostPassword
return vm
from subprocess import check_call
from ftplib import FTP,error_perm
from urlparse import urljoin
from celery import Celery
import os
import sys
import urllib
import re
import logging
import random
import time
from utils import get_config_option
logging.basicConfig(format= '%(asctime)s ' + "Encoder " + '%(message)s', level=logging.DEBUG)
app = Celery('tasks', broker=get_config_option('General', 'broker_url'))
@app.task
def encode_video(input_file_path, output_file_name):
"""
We first obtain the video via http, encode it, and then upload it back
"""
server_url = get_config_option("Download Settings", "server_url")
root_dir = get_config_option("FTP Settings" , "root_dir")
file_path, input_file_name = os.path.split(input_file_path)
url = urljoin(server_url, input_file_path.lstrip("\\"))
logging.info("Downloading {0}".format(url))
try:
urllib.urlretrieve(url, input_file_name)
logging.info("Finished downloading {0}".format(url))
except IOError:
