def run(self): '''Creates required directories, base_dir at ~/cluster_programs/project_name copies source and encoder to temporary job directory, encodes the source uploads the resulting files to the file server, and deletes all temporary files. Sends status updates on encode to the message server every second''' homedir = os.path.expanduser('~') projectdir = os.path.join(homedir, 'cluster_programs', self._project_name) jobdir=os.path.join(homedir, 'cluster_programs', self._project_name, self._job_name) sourcedir=os.path.join('\\\\', self._server_ip,self._source_path) outputdir=os.path.join('\\\\', self._server_ip, self._destination_path) os.chdir(projectdir) try: os.mkdir(self._job_name) except OSError: logging.debug('Directory' + self._job_name +' exists, continuing using existing dir') shutil.copy(self._exe_name, self._job_name) os.chdir(jobdir) logging.debug('Connecting with message server') status_updates=MessageWriter(server=self._message_server, vhost=self._vhost, userid=self._userid, password=self._password, exchange='direct_transfer', exchange_durable=True, \ exchange_auto_delete=False, exchange_type='direct', routing_key=self._server_hostname, queue_durable=False, queue_auto_delete=True) logging.debug('Connection with message server established') logging.debug('Copying Source') status_updates.send_message(self._encode_message('Copying Source')) if os.path.exists('source'): shutil.rmtree('source') shutil.copytree(sourcedir, 'source') status_updates.send_message(self._encode_message('Finished Copying Source')) logging.debug('Finished Copying Source') logging.debug('Starting encode process '+self._exe_name) status_updates.send_message(self._encode_message('Starting Process '+self._exe_name)) proc = subprocess.Popen(self._command, bufsize=1, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) loglock=threading.Lock() logfile='encodinglog.txt' t = ProcessMonitor(proc, logfile, loglock) t.start() while proc.poll() is None: time.sleep(30) loglock.acquire() log=open(logfile, 'rU') current_progress=tail(log, 1) log.close() loglock.release() logging.debug(current_progress) status_updates.send_message(self._encode_message(current_progress)) logging.debug('Finished Encoding') status_updates.send_message(self._encode_message('Finished Encoding')) logging.debug('Copying Output to Server') status_updates.send_message(self._encode_message('Copying Output to Server')) shutil.copy(self._output_file, outputdir) logging.debug('Finished Copying Output to Server') status_updates.send_message(self._encode_message('Finished Copying Output to Server')) os.chdir(homedir) logging.debug('Deleting temporary files and directories') shutil.rmtree(path=jobdir, ignore_errors=True) logging.debug('Closing connection with message server') status_updates.close()
def run(self): '''Main body of the code.''' logging.debug('ProcessDVD called with argument ' + self.path) (base_dir,file_name)=os.path.split(self.path) (root,ext)=os.path.splitext(file_name) logging.debug('Changing directory to ' + base_dir) os.chdir(base_dir) if ext == '.ISO' or ext == '.iso': logging.debug('ISO file detected, proceding to mount') logging.debug('Making directory ' + root) try: os.mkdir(root) except OSError: logging.debug('Directory exists, using existing') if len(os.listdir(root)) != 0: logging.debug('It appears something is mounted at '+root) logging.debug('Attempting unmount') try: subprocess.check_call(['sudo','umount',root]) except: logging.debug('Unmount failed') sys.exit('Mountpoint exists and has contents') logging.debug('Mounting iso '+file_name+' to '+root) try: subprocess.check_call(['sudo','mount','-o','loop',file_name,root]) except: sys.exit('An unhandled exception occurred while executing a mount command. \ Check that you have passwordless sudo mount enabled') self.cur_disk_path=os.path.join(base_dir,root) elif ext == '.mp4' or ext == '.MP4' or ext == '.mkv' or ext == '.MKV' \ or ext == '.avi' or ext == '.AVI' or ext == '.mts' or ext == '.MTS': logging.debug('Non-ISO video file detected') self.cur_disk_path=self.path else: logging.error('Only ISOs, MP4s, MKVs, MTSs, and AVIs are handled') sys.exit('Only ISOs, MP4s, MKVs, MTSs, and AVIs are handled') logging.debug('Scanning video') parsed_DVD=parseDVD(self.cur_disk_path) logging.debug('Generating encoding commands') logging.debug('self.compression_type is ' + str(self.compression_type)) f=globals()[str(self.compression_type)] commands=jobGenerator(parsed_DVD,f()) if ext == '.ISO' or ext == '.iso': logging.debug('Unmounting ISO') try: subprocess.check_call(['sudo','umount',root]) except: logging.error('Unmounting failed. Ensure that passworless sudo umount\ is enabled') logging.debug('Removing temporary mount directory') try: os.rmdir(root) except: logging.error('Unable to remove temporary mount directory '+root) logging.debug('Establishing connection with message server') writer=MessageWriter(server=MESSAGE_SERVER, vhost=VHOST, \ userid=MESSAGE_USERID, password=MESSAGE_PWD, \ exchange=EXCHANGE, exchange_durable=True, \ exchange_auto_delete=False, exchange_type='direct',\ routing_key=JOB_QUEUE, queue_durable=True,\ queue_auto_delete=False) for i,command in enumerate(commands): job_mountpoint='' if ext == '.ISO' or ext == '.iso': #Must mount an ISO, will leave mounted until job complete message job_mountpoint=root+'_job_'+str(i) logging.debug('Making job subdirectory ' + job_mountpoint) try: os.mkdir(job_mountpoint) except OSError: logging.debug('Directory already exists, assuming previous mount') if len(os.listdir(job_mountpoint)) != 0: try: subprocess.check_call(['sudo','umount',job_mountpoint]) except: logging.error('Unable to unmount ' + job_mountpoint +'. Skipping') continue logging.debug('Mounting ISO at ' + job_mountpoint) try: subprocess.check_call(['sudo','mount','-o','loop',file_name,job_mountpoint]) except: logging.error('Unhandled exception while mounting ISO for job.\ Check that passworless sudo mount is available.') logging.error('Skipping') continue else: job_mountpoint=file_name logging.debug('Appending output file to command') command.append('-i') command.append(job_mountpoint) logging.debug('Sending complete command to message server') myip=socket.gethostbyname(socket.getfqdn()) ftp_location='ftp://' + str(myip)+ ':' + str(FTP_PORT) + '/jobs/' + job_mountpoint writer.send_message(pickle.dumps([job_mountpoint,ftp_location,command])) logging.debug([job_mountpoint,ftp_location,command]) writer.close()
class ProjectThread(threading.Thread): '''Needs to listen on vhost/project-name/queue, vhost/project-name/hostname, and establish a write connection on vhost/project-name/server So, two messagereaders and one messagewriter Also must make the project directory, copy the encoding exe to this directory''' def __init__(self, project_name, project_exe, project_exe_path, \ server_ip='localhost', message_server_ip='localhost',\ message_server_hostname='localhost', vhost='/', userid='guest',\ password='******'): threading.Thread.__init__(self) self._project_name=project_name self._project_exe=project_exe self._project_exe_path=project_exe_path self._server_ip=server_ip self._message_server_ip=message_server_ip self._message_server_hostname=message_server_hostname self._vhost=vhost self._userid=userid self._password=password self._running=True def run(self): homedir = os.path.expanduser('~') basedir = os.path.join(homedir,'cluster_programs') projectdir = os.path.join(homedir, 'cluster_programs', self._project_name) logging.debug('Making base directory for project '+self._project_name) os.chdir(basedir) try: os.mkdir(self._project_name) except OSError: logging.debug('Directory' + self._project_name +' exists, continuing using existing dir') os.chdir(projectdir) logging.debug('Base directory created at '+ projectdir) logging.debug('Copying encoding executable for '+self._project_name) exe_path=os.path.join('\\\\', self._server_ip,self._project_exe_path, self._project_exe) shutil.copy(exe_path,self._project_exe) logging.debug('Finished copying encoding executable') #Establishes the direct-connect message connection with server #Sends messages to vhost/direct-connect/message-server-hostname #Messages will have form [hostname, project, message] self._server_direct_connect=MessageWriter(server=self._message_server_ip,\ vhost=self._vhost,userid=self._userid,password=self._password,\ exchange=self._project_name,exchange_type='direct',exchange_durable=True,\ exchange_auto_delete=False,\ routing_key=self._message_server_hostname) #Establishes a listener that will accept new jobs and spin off #client job threads in response self._project_queue_listener=MessageReader(server=self._message_server_ip,\ vhost=self._vhost,userid=self._userid,password=self._password,\ exchange=self._project_name,routing_key='queue',exchange_type='direct',\ callback=self._job_creation) #Establishes a listener that will accept direct messages from server #and respond appropriately self._project_server_listener=MessageReader(server=self._message_server_ip,\ vhost=self._vhost,userid=self._userid,password=self._password,\ exchange=self._project_name,routing_key=socket.gethostname(),\ exchange_type='direct',callback=self._process_message) self._project_queue_listener.start() self._project_server_listener.start() while self._running==True: pass #Still need to work out message acknowledgment. Will probably do so with #a queue. Currently, acknowledgment is turned off, which is not ideal self._project_queue_listener.stop() self._project_server_listener.stop() self._server_direct_connect.close() self._project_queue_listener.join() self._project_server_listener.join() # Needs a listener on vhost/project_name # Messages will be of form [hostname, server ip, source, destination, command] def _process_message(self,message): pass #Currently all communication is one directional def _job_creation(self,message): '''Note: does not keep track of threads as I currently don't have a way of stopping a job thread''' cmd=pickle.loads(str=message) #[job_name, source_path, destination_path, [command], output_file] job=JobThread(project_name=self._project_name, job_name=cmd[0], exe_name=self._project_exe,\ source_path=cmd[1], destination_path=cmd[2], command=cmd[3],\ output_file=cmd[4],\ message_server=self._message_server_ip,\ vhost=self._vhost, userid=self._userid, password=self._password,\ server_hostname=self._message_server_hostname,\ server_ip=self._server_ip) job.start() def stop(self): self._running=False