def run(self):
homedir = os.path.expanduser('~')
basedir = os.path.join(homedir,'cluster_programs')
projectdir = os.path.join(homedir, 'cluster_programs', self._project_name)
logging.debug('Making base directory for project '+self._project_name)
os.chdir(basedir)
try:
os.mkdir(self._project_name)
except OSError:
logging.debug('Directory' + self._project_name +' exists, continuing using existing dir')
os.chdir(projectdir)
logging.debug('Base directory created at '+ projectdir)
logging.debug('Copying encoding executable for '+self._project_name)
exe_path=os.path.join('\\\\', self._server_ip,self._project_exe_path,
self._project_exe)
shutil.copy(exe_path,self._project_exe)
logging.debug('Finished copying encoding executable')
#Establishes the direct-connect message connection with server
#Sends messages to vhost/direct-connect/message-server-hostname
#Messages will have form [hostname, project, message]
self._server_direct_connect=MessageWriter(server=self._message_server_ip,\
vhost=self._vhost,userid=self._userid,password=self._password,\
exchange=self._project_name,exchange_type='direct',exchange_durable=True,\
exchange_auto_delete=False,\
routing_key=self._message_server_hostname)
#Establishes a listener that will accept new jobs and spin off
#client job threads in response
self._project_queue_listener=MessageReader(server=self._message_server_ip,\
vhost=self._vhost,userid=self._userid,password=self._password,\
exchange=self._project_name,routing_key='queue',exchange_type='direct',\
callback=self._job_creation)
#Establishes a listener that will accept direct messages from server
#and respond appropriately
self._project_server_listener=MessageReader(server=self._message_server_ip,\
vhost=self._vhost,userid=self._userid,password=self._password,\
exchange=self._project_name,routing_key=socket.gethostname(),\
exchange_type='direct',callback=self._process_message)
self._project_queue_listener.start()
self._project_server_listener.start()
while self._running==True:
pass
#Still need to work out message acknowledgment. Will probably do so with
#a queue. Currently, acknowledgment is turned off, which is not ideal
self._project_queue_listener.stop()
self._project_server_listener.stop()
self._server_direct_connect.close()
self._project_queue_listener.join()
self._project_server_listener.join()
def run(self):
'''Main body of the code.'''
logging.debug('ProcessDVD called with argument ' + self.path)
(base_dir,file_name)=os.path.split(self.path)
(root,ext)=os.path.splitext(file_name)
logging.debug('Changing directory to ' + base_dir)
os.chdir(base_dir)
if ext == '.ISO' or ext == '.iso':
logging.debug('ISO file detected, proceding to mount')
logging.debug('Making directory ' + root)
try:
os.mkdir(root)
except OSError:
logging.debug('Directory exists, using existing')
if len(os.listdir(root)) != 0:
logging.debug('It appears something is mounted at '+root)
logging.debug('Attempting unmount')
try:
subprocess.check_call(['sudo','umount',root])
except:
logging.debug('Unmount failed')
sys.exit('Mountpoint exists and has contents')
logging.debug('Mounting iso '+file_name+' to '+root)
try:
subprocess.check_call(['sudo','mount','-o','loop',file_name,root])
except:
sys.exit('An unhandled exception occurred while executing a mount command. \
Check that you have passwordless sudo mount enabled')
self.cur_disk_path=os.path.join(base_dir,root)
elif ext == '.mp4' or ext == '.MP4' or ext == '.mkv' or ext == '.MKV' \
or ext == '.avi' or ext == '.AVI' or ext == '.mts' or ext == '.MTS':
logging.debug('Non-ISO video file detected')
self.cur_disk_path=self.path
else:
logging.error('Only ISOs, MP4s, MKVs, MTSs, and AVIs are handled')
sys.exit('Only ISOs, MP4s, MKVs, MTSs, and AVIs are handled')
logging.debug('Scanning video')
parsed_DVD=parseDVD(self.cur_disk_path)
logging.debug('Generating encoding commands')
logging.debug('self.compression_type is ' + str(self.compression_type))
f=globals()[str(self.compression_type)]
commands=jobGenerator(parsed_DVD,f())
if ext == '.ISO' or ext == '.iso':
logging.debug('Unmounting ISO')
try:
subprocess.check_call(['sudo','umount',root])
except:
logging.error('Unmounting failed. Ensure that passworless sudo umount\
is enabled')
logging.debug('Removing temporary mount directory')
try:
os.rmdir(root)
except:
logging.error('Unable to remove temporary mount directory '+root)
logging.debug('Establishing connection with message server')
writer=MessageWriter(server=MESSAGE_SERVER, vhost=VHOST, \
userid=MESSAGE_USERID, password=MESSAGE_PWD, \
exchange=EXCHANGE, exchange_durable=True, \
exchange_auto_delete=False, exchange_type='direct',\
routing_key=JOB_QUEUE, queue_durable=True,\
queue_auto_delete=False)
for i,command in enumerate(commands):
job_mountpoint=''
if ext == '.ISO' or ext == '.iso':
#Must mount an ISO, will leave mounted until job complete message
job_mountpoint=root+'_job_'+str(i)
logging.debug('Making job subdirectory ' + job_mountpoint)
try:
os.mkdir(job_mountpoint)
except OSError:
logging.debug('Directory already exists, assuming previous mount')
if len(os.listdir(job_mountpoint)) != 0:
try:
subprocess.check_call(['sudo','umount',job_mountpoint])
except:
logging.error('Unable to unmount ' + job_mountpoint +'. Skipping')
continue
logging.debug('Mounting ISO at ' + job_mountpoint)
try:
subprocess.check_call(['sudo','mount','-o','loop',file_name,job_mountpoint])
except:
logging.error('Unhandled exception while mounting ISO for job.\
Check that passworless sudo mount is available.')
logging.error('Skipping')
continue
else:
job_mountpoint=file_name
logging.debug('Appending output file to command')
command.append('-i')
command.append(job_mountpoint)
logging.debug('Sending complete command to message server')
myip=socket.gethostbyname(socket.getfqdn())
ftp_location='ftp://' + str(myip)+ ':' + str(FTP_PORT) + '/jobs/' + job_mountpoint
writer.send_message(pickle.dumps([job_mountpoint,ftp_location,command]))
logging.debug([job_mountpoint,ftp_location,command])
writer.close()
def run(self):
'''Creates required directories, base_dir at ~/cluster_programs/project_name
copies source and encoder to temporary job directory, encodes the source
uploads the resulting files to the file server, and deletes all temporary
files.
Sends status updates on encode to the message server every second'''
homedir = os.path.expanduser('~')
projectdir = os.path.join(homedir, 'cluster_programs', self._project_name)
jobdir=os.path.join(homedir, 'cluster_programs', self._project_name, self._job_name)
sourcedir=os.path.join('\\\\', self._server_ip,self._source_path)
outputdir=os.path.join('\\\\', self._server_ip, self._destination_path)
os.chdir(projectdir)
try:
os.mkdir(self._job_name)
except OSError:
logging.debug('Directory' + self._job_name +' exists, continuing using existing dir')
shutil.copy(self._exe_name, self._job_name)
os.chdir(jobdir)
logging.debug('Connecting with message server')
status_updates=MessageWriter(server=self._message_server, vhost=self._vhost, userid=self._userid, password=self._password, exchange='direct_transfer', exchange_durable=True, \
exchange_auto_delete=False, exchange_type='direct', routing_key=self._server_hostname, queue_durable=False, queue_auto_delete=True)
logging.debug('Connection with message server established')
logging.debug('Copying Source')
status_updates.send_message(self._encode_message('Copying Source'))
if os.path.exists('source'):
shutil.rmtree('source')
shutil.copytree(sourcedir, 'source')
status_updates.send_message(self._encode_message('Finished Copying Source'))
logging.debug('Finished Copying Source')
logging.debug('Starting encode process '+self._exe_name)
status_updates.send_message(self._encode_message('Starting Process '+self._exe_name))
proc = subprocess.Popen(self._command, bufsize=1, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True)
loglock=threading.Lock()
logfile='encodinglog.txt'
t = ProcessMonitor(proc, logfile, loglock)
t.start()
while proc.poll() is None:
time.sleep(30)
loglock.acquire()
log=open(logfile, 'rU')
current_progress=tail(log, 1)
log.close()
loglock.release()
logging.debug(current_progress)
status_updates.send_message(self._encode_message(current_progress))
logging.debug('Finished Encoding')
status_updates.send_message(self._encode_message('Finished Encoding'))
logging.debug('Copying Output to Server')
status_updates.send_message(self._encode_message('Copying Output to Server'))
shutil.copy(self._output_file, outputdir)
logging.debug('Finished Copying Output to Server')
status_updates.send_message(self._encode_message('Finished Copying Output to Server'))
os.chdir(homedir)
logging.debug('Deleting temporary files and directories')
shutil.rmtree(path=jobdir, ignore_errors=True)
logging.debug('Closing connection with message server')
status_updates.close()
class ProjectThread(threading.Thread):
'''Needs to listen on vhost/project-name/queue, vhost/project-name/hostname,
and establish a write connection on vhost/project-name/server
So, two messagereaders and one messagewriter
Also must make the project directory, copy the encoding exe to this directory'''
def __init__(self, project_name, project_exe, project_exe_path, \
server_ip='localhost', message_server_ip='localhost',\
message_server_hostname='localhost', vhost='/', userid='guest',\
password='******'):
threading.Thread.__init__(self)
self._project_name=project_name
self._project_exe=project_exe
self._project_exe_path=project_exe_path
self._server_ip=server_ip
self._message_server_ip=message_server_ip
self._message_server_hostname=message_server_hostname
self._vhost=vhost
self._userid=userid
self._password=password
self._running=True
def run(self):
homedir = os.path.expanduser('~')
basedir = os.path.join(homedir,'cluster_programs')
projectdir = os.path.join(homedir, 'cluster_programs', self._project_name)
logging.debug('Making base directory for project '+self._project_name)
os.chdir(basedir)
try:
os.mkdir(self._project_name)
except OSError:
logging.debug('Directory' + self._project_name +' exists, continuing using existing dir')
os.chdir(projectdir)
logging.debug('Base directory created at '+ projectdir)
logging.debug('Copying encoding executable for '+self._project_name)
exe_path=os.path.join('\\\\', self._server_ip,self._project_exe_path,
self._project_exe)
shutil.copy(exe_path,self._project_exe)
logging.debug('Finished copying encoding executable')
#Establishes the direct-connect message connection with server
#Sends messages to vhost/direct-connect/message-server-hostname
#Messages will have form [hostname, project, message]
self._server_direct_connect=MessageWriter(server=self._message_server_ip,\
vhost=self._vhost,userid=self._userid,password=self._password,\
exchange=self._project_name,exchange_type='direct',exchange_durable=True,\
exchange_auto_delete=False,\
routing_key=self._message_server_hostname)
#Establishes a listener that will accept new jobs and spin off
#client job threads in response
self._project_queue_listener=MessageReader(server=self._message_server_ip,\
vhost=self._vhost,userid=self._userid,password=self._password,\
exchange=self._project_name,routing_key='queue',exchange_type='direct',\
callback=self._job_creation)
#Establishes a listener that will accept direct messages from server
#and respond appropriately
self._project_server_listener=MessageReader(server=self._message_server_ip,\
vhost=self._vhost,userid=self._userid,password=self._password,\
exchange=self._project_name,routing_key=socket.gethostname(),\
exchange_type='direct',callback=self._process_message)
self._project_queue_listener.start()
self._project_server_listener.start()
while self._running==True:
pass
#Still need to work out message acknowledgment. Will probably do so with
#a queue. Currently, acknowledgment is turned off, which is not ideal
self._project_queue_listener.stop()
self._project_server_listener.stop()
self._server_direct_connect.close()
self._project_queue_listener.join()
self._project_server_listener.join()
# Needs a listener on vhost/project_name
# Messages will be of form [hostname, server ip, source, destination, command]
def _process_message(self,message):
pass
#Currently all communication is one directional
def _job_creation(self,message):
'''Note: does not keep track of threads as I currently don't have a way
of stopping a job thread'''
cmd=pickle.loads(str=message)
#[job_name, source_path, destination_path, [command], output_file]
job=JobThread(project_name=self._project_name, job_name=cmd[0], exe_name=self._project_exe,\
source_path=cmd[1], destination_path=cmd[2], command=cmd[3],\
output_file=cmd[4],\
message_server=self._message_server_ip,\
vhost=self._vhost, userid=self._userid, password=self._password,\
server_hostname=self._message_server_hostname,\
server_ip=self._server_ip)
job.start()
def stop(self):
self._running=False
