def encode_video(input_file_path, output_file_name): """ We first obtain the video via http, encode it, and then upload it back """ server_url = get_config_option("Download Settings", "server_url") root_dir = get_config_option("FTP Settings" , "root_dir") file_path, input_file_name = os.path.split(input_file_path) url = urljoin(server_url, input_file_path.lstrip("\\")) logging.info("Downloading {0}".format(url)) try: urllib.urlretrieve(url, input_file_name) logging.info("Finished downloading {0}".format(url)) except IOError: delay = random.randrange(40) logging.info("Seems like server is overloaded. Retrying in {0} seconds.".format(delay)) time.sleep(delay) urllib.urlretrieve(url, input_file_name) logging.info("Finished downloading {0}".format(url)) ffmpeg_args = ["ffmpeg", "-y", "-i", input_file_name, "-an","-c:v", "libx264", "-b:v", "1024k", output_file_name] if sys.platform.startswith('linux'): logging.info("Linux detected. Modifying command") ffmpeg_args[0] = 'avconv' ffmpeg_args = " ".join(ffmpeg_args) if not check_call(ffmpeg_args, shell=True): ftp = FTP(host=get_config_option("FTP Settings" , "host"), user=get_config_option("FTP Settings", "username") , passwd=get_config_option("FTP Settings" , "password")) ftp.cwd(root_dir) full_path = re.split("/", file_path) for d in full_path: try: ftp.cwd(d) except error_perm: ftp.mkd(d) ftp.cwd(d) f = open(output_file_name, 'rb') logging.info("Starting upload of {0} as {1}".format(output_file_name, input_file_name)) ftp.storbinary('STOR ' + input_file_name, f) logging.info("Finished upload of {0} as {1}".format(output_file_name, input_file_name)) ftp.quit() f.close() #Remove files after encoding them try: os.remove(input_file_name) except OSError: logging.info("Failed to remove {0}".format(input_file_name)) else: logging.info("Removed {0}".format(input_file_name)) try: os.remove(output_file_name) except OSError: logging.info("Failed to remove {0}".format(output_file_name)) else: logging.info("Removed {0}".format(output_file_name))
def __init__( self, path ): self._steps = [] config = ConfigParser.SafeConfigParser() config.read( path ) self._path = path self._platforms = [] found = True stepNum = 0 while found: sec = "Step{0}".format( stepNum ) if not config.has_section( sec ): found = False continue stepNum += 1 installscript = utils.makeAbsolutePath( utils.get_config_option( config, None, "installscript", None, sec ), os.path.dirname( path ) ) checkerTestDir = utils.get_config_option( config, None, "checkerTestDir", None, sec ) checkerTestDir = utils.makeAbsolutePath( checkerTestDir, os.path.dirname( path ) ) if checkerTestDir else "" timeout = int( utils.get_config_option( config, None, "timeout", 60 * 60, sec ) ) self._steps.append( Step( installscript, checkerTestDir, timeout ) ) self._name = utils.get_config_option( config, None, "name", utils.basename( path ) ) self._targetDirectory = utils.get_config_option( config, None, "targetDirectory", "" ) self._maintenanceToolLocation = utils.get_config_option( config, None, "maintenanceToolLocation", "" ) platforms = utils.get_config_option( config, None, "platforms" ) if platforms != None: self._platforms = platforms.split( ',' )
def generate_jobs(abs_dir): ftp = get_ftp_connection() if abs_dir is not None: root = abs_dir num_jobs = 1 #TODO: Actually implement the date feature for root, dirs, files in os.walk(root): if len(files): file_list = [] try: ftp.cwd("/" + get_config_option("FTP Settings" , "root_dir")) formatted_dir = os.path.splitdrive(root)[1] formatted_dir = formatted_dir.replace("\\", "/") formatted_dir = formatted_dir[1:] #We need to cut off the leading slash ftp.cwd(formatted_dir) file_list = ftp.nlst() except error_perm: # Occurs if there is no directory of the name we are looking for print (formatted_dir + "NOT FOUND") pass except error_temp: print "Seems like FTP connection was lost. Reconnecting" ftp = get_ftp_connection() for f in files: if f.endswith('.avi'): the_file = os.path.join(root, f) the_path = os.path.splitdrive((the_file)) final_path = the_path[1].replace("\\", "/") if not clean_old and num_jobs % 10 == 0: print "Sleeping for 10 minutes" time.sleep(600) if f not in file_list: if not clean_old: print("{0} -> Adding {1} to list" .format(num_jobs, final_path)) encode_video.delay(final_path, "OUTPUT_" + f) num_jobs += 1 #Dont increment unless we actually added a job else: print("{0} ALREADY EXISTS REMOTELY. REMOVING LOCALLY." .format(final_path)) os.remove(os.path.join(root, f)) elif not os.listdir(root): print("{0} is empty. Removing directory").format(root) os.rmdir(root) ftp.quit()
try: options.vmx = args[0] except IndexError: options.vmx = None options.ostype = None options.guestTempDir = None config = ConfigParser.SafeConfigParser() if options.config: print("** Reading config: " + options.config ) config.read( options.config ) else: print("** No config given") options.username = utils.get_config_option( config, options.username, "username", "nokia" ) options.password = utils.get_config_option( config, options.password, "password", "nokia" ) options.script = utils.get_config_option( config, options.script, "script", "guest.py" ) options.python = utils.get_config_option( config, options.python, "python", "c:/python26/python.exe" ) options.snapshot = utils.get_config_option( config, options.snapshot, "snapshot", "base" ) options.vmrun = utils.get_config_option( config, options.vmrun, "vmrun" ) options.vmx = utils.get_config_option( config, options.vmx, "vmx" ) options.guestTempDir = utils.get_config_option( config, options.guestTempDir, "tempDir", "c:\\windows\\temp" ) options.ostype = utils.get_config_option( config, options.ostype, "os", "windows" ) check_option( options.vmx, "vmx-file" ) # Search the PATH and the a few extra locations for 'vmrun' if not options.vmrun: options.vmrun = utils.findVMRun()
def get_ftp_connection(): ftp = FTP(host=get_config_option("FTP Settings" , "host"), user=get_config_option("FTP Settings", "username") , passwd=get_config_option("FTP Settings" , "password")) return ftp
def fromVMRunAndPath(vmrun, path): config = ConfigParser.SafeConfigParser() config.read(path) hostType = utils.get_config_option(config, None, "type", "", "Host") hostLocation = utils.get_config_option(config, None, "location", "", "Host") hostUsername = utils.get_config_option(config, None, "username", "", "Host") hostPassword = utils.get_config_option(config, None, "password", "", "Host") vmxVal = utils.get_config_option(config, None, "vmx") if not len(hostType) == 0: vmx = utils.makeAbsolutePath(vmxVal, os.path.dirname(path)) else: vmx = vmxVal username = utils.get_config_option(config, None, "username", "nokia") password = utils.get_config_option(config, None, "password", "nokia") tempDir = utils.get_config_option(config, None, "tempDir", "c:\\windows\\temp") ostype = utils.get_config_option(config, None, "os", "windows") vm = VirtualMachine(vmrun, vmx, username, password, tempDir, ostype) vm._name = utils.get_config_option(config, None, "name", utils.basename(path)) vm._snapshot = utils.get_config_option(config, None, "snapshot", "base") vm._python = utils.get_config_option(config, None, "python", "c:/python26/python.exe") vm._hostType = hostType vm._hostLocation = hostLocation vm._hostUsername = hostUsername vm._hostPassword = hostPassword return vm
optionParser.add_option("-c", "--checkerInstallation", dest="checkerInstallation", help="checker installation to use for post-installation checks", metavar="CHECKERINSTALLATION" ) (options, args) = optionParser.parse_args() try: configpath = utils.makeAbsolutePath( args[0], os.getcwd() ) except IndexError: optionParser.print_usage( sys.stderr ) sys.exit( 1 ) config = ConfigParser.SafeConfigParser() config.read( configpath ) #make an unary functor to create absolute paths abspath = partial( utils.makeAbsolutePath, relativeTo=os.path.dirname( configpath ) ) vmrun = utils.get_config_option( config, options.vmrun, "vmrun", utils.findVMRun() ) useGui = utils.get_config_option( config, None, "gui", "true" ).lower() == "true" createErrorSnapshots = utils.get_config_option( config, None, "createErrorSnapshots", "true" ).lower() == "true" hostType = utils.get_config_option( config, None, "type", "", "Host" ) hostLocation = utils.get_config_option( config, None, "location", "", "Host" ) hostUsername = utils.get_config_option( config, None, "username", "", "Host" ) hostPassword = utils.get_config_option( config, None, "password", "", "Host" ) cdashHost = utils.get_config_option( config, None, "host", "", "CDash" ) cdashLocation = utils.get_config_option( config, None, "location", "", "CDash" ) cdashProject = utils.get_config_option( config, None, "project", "", "CDash" ) if vmrun == None: die( "Could not find vmrun executable. Please specify it in the config file (vmrun=...) or via the --vmrun option" ) checkerInstallation = utils.get_config_option( config, options.checkerInstallation, "checkerInstallation" )
def fromVMRunAndPath( vmrun, path ): config = ConfigParser.SafeConfigParser() config.read( path ) hostType = utils.get_config_option( config, None, "type", "", "Host" ) hostLocation = utils.get_config_option( config, None, "location", "", "Host" ) hostUsername = utils.get_config_option( config, None, "username", "", "Host" ) hostPassword = utils.get_config_option( config, None, "password", "", "Host" ) vmxVal = utils.get_config_option( config, None, "vmx" ) if not len( hostType ) == 0: vmx = utils.makeAbsolutePath( vmxVal, os.path.dirname( path ) ) else: vmx = vmxVal username = utils.get_config_option( config, None, "username", "nokia" ) password = utils.get_config_option( config, None, "password", "nokia" ) tempDir = utils.get_config_option( config, None, "tempDir", "c:\\windows\\temp" ) ostype = utils.get_config_option( config, None, "os", "windows" ) vm = VirtualMachine( vmrun, vmx, username, password, tempDir, ostype ) vm._name = utils.get_config_option( config, None, "name", utils.basename( path ) ) vm._snapshot = utils.get_config_option( config, None, "snapshot", "base" ) vm._python = utils.get_config_option( config, None, "python", "c:/python26/python.exe" ) vm._hostType = hostType vm._hostLocation = hostLocation vm._hostUsername = hostUsername vm._hostPassword = hostPassword return vm
from subprocess import check_call from ftplib import FTP,error_perm from urlparse import urljoin from celery import Celery import os import sys import urllib import re import logging import random import time from utils import get_config_option logging.basicConfig(format= '%(asctime)s ' + "Encoder " + '%(message)s', level=logging.DEBUG) app = Celery('tasks', broker=get_config_option('General', 'broker_url')) @app.task def encode_video(input_file_path, output_file_name): """ We first obtain the video via http, encode it, and then upload it back """ server_url = get_config_option("Download Settings", "server_url") root_dir = get_config_option("FTP Settings" , "root_dir") file_path, input_file_name = os.path.split(input_file_path) url = urljoin(server_url, input_file_path.lstrip("\\")) logging.info("Downloading {0}".format(url)) try: urllib.urlretrieve(url, input_file_name) logging.info("Finished downloading {0}".format(url)) except IOError: