def control_loop(): '''Main loop, retrieving the schedule. ''' set_service_status(Service.SCHEDULE, ServiceStatus.BUSY) notify.notify('READY=1') while not terminate(): notify.notify('WATCHDOG=1') # Try getting an updated schedule get_schedule() session = get_session() next_event = session.query(UpcomingEvent)\ .filter(UpcomingEvent.end > timestamp())\ .order_by(UpcomingEvent.start)\ .first() if next_event: logger.info('Next scheduled recording: %s', datetime.fromtimestamp(next_event.start)) notify.notify('STATUS=Next scheduled recording: %s' % datetime.fromtimestamp(next_event.start)) else: logger.info('No scheduled recording') notify.notify('STATUS=No scheduled recording') session.close() next_update = timestamp() + config()['agent']['update_frequency'] while not terminate() and timestamp() < next_update: time.sleep(0.1) logger.info('Shutting down schedule service') set_service_status(Service.SCHEDULE, ServiceStatus.STOPPED)
def ingest(event): '''Ingest a finished recording to the Opencast server. ''' # Update status set_service_status(Service.INGEST, ServiceStatus.BUSY) recording_state(event.uid, 'uploading') update_event_status(event, Status.UPLOADING) # Select ingest service # The ingest service to use is selected at random from the available # ingest services to ensure that not every capture agent uses the same # service at the same time service = config('service-ingest') service = service[randrange(0, len(service))] logger.info('Selecting ingest service to use: ' + service) # create mediapackage logger.info('Creating new mediapackage') mediapackage = http_request(service + '/createMediaPackage') # extract workflow_def, workflow_config and add DC catalogs prop = 'org.opencastproject.capture.agent.properties' dcns = 'http://www.opencastproject.org/xsd/1.0/dublincore/' for attachment in event.get_data().get('attach'): data = attachment.get('data') if attachment.get('x-apple-filename') == prop: workflow_def, workflow_config = get_config_params(data) # Check for dublincore catalogs elif attachment.get('fmttype') == 'application/xml' and dcns in data: name = attachment.get('x-apple-filename', '').rsplit('.', 1)[0] logger.info('Adding %s DC catalog' % name) fields = [('mediaPackage', mediapackage), ('flavor', 'dublincore/%s' % name), ('dublinCore', data.encode('utf-8'))] mediapackage = http_request(service + '/addDCCatalog', fields) # add track for (flavor, track) in event.get_tracks(): logger.info('Adding track ({0} -> {1})'.format(flavor, track)) track = track.encode('ascii', 'ignore') fields = [('mediaPackage', mediapackage), ('flavor', flavor), ('BODY1', (pycurl.FORM_FILE, track))] mediapackage = http_request(service + '/addTrack', fields) # ingest logger.info('Ingest recording') fields = [('mediaPackage', mediapackage)] if workflow_def: fields.append(('workflowDefinitionId', workflow_def)) if event.uid: fields.append(('workflowInstanceId', event.uid.encode('ascii', 'ignore'))) fields += workflow_config mediapackage = http_request(service + '/ingest', fields) # Update status recording_state(event.uid, 'upload_finished') update_event_status(event, Status.FINISHED_UPLOADING) set_service_status_immediate(Service.INGEST, ServiceStatus.IDLE)
def safe_start_ingest(event): '''Start a capture process but make sure to catch any errors during this process, log them but otherwise ignore them. ''' try: return start_ingest(event) except Exception: logger.error('Start ingest failed') logger.error(traceback.format_exc()) set_service_status(Service.INGEST, ServiceStatus.IDLE) return False
def control_loop(): '''Main loop, updating the capture agent state. ''' set_service_status(Service.AGENTSTATE, ServiceStatus.BUSY) while not terminate(): update_agent_state() next_update = timestamp() + config()['agent']['update_frequency'] while not terminate() and timestamp() < next_update: time.sleep(0.1) logger.info('Shutting down agentstate service') set_service_status(Service.AGENTSTATE, ServiceStatus.STOPPED)
def control_loop(): '''Main loop of the capture agent, retrieving and checking the schedule as well as starting the capture process if necessry. ''' set_service_status(Service.INGEST, ServiceStatus.IDLE) while not terminate(): # Get next recording event = get_session().query(RecordedEvent)\ .filter(RecordedEvent.status == Status.FINISHED_RECORDING).first() if event: safe_start_ingest(event) time.sleep(1.0) logger.info('Shutting down ingest service') set_service_status(Service.INGEST, ServiceStatus.STOPPED)
def control_loop(): '''Main loop of the capture agent, retrieving and checking the schedule as well as starting the capture process if necessry. ''' set_service_status(Service.INGEST, ServiceStatus.IDLE) while not terminate(): # Get next recording events = get_session().query(RecordedEvent)\ .filter(RecordedEvent.status == Status.FINISHED_RECORDING) if events.count(): safe_start_ingest(events[0]) time.sleep(1.0) logger.info('Shutting down ingest service') set_service_status(Service.INGEST, ServiceStatus.STOPPED)
def control_loop(): '''Main loop of the capture agent, retrieving and checking the schedule as well as starting the capture process if necessry. ''' set_service_status(Service.CAPTURE, ServiceStatus.IDLE) while not terminate(): # Get next recording event = get_session().query(UpcomingEvent)\ .filter(UpcomingEvent.start <= timestamp())\ .filter(UpcomingEvent.end > timestamp())\ .first() if event: safe_start_capture(event) time.sleep(1.0) logger.info('Shutting down capture service') set_service_status(Service.CAPTURE, ServiceStatus.STOPPED)
def control_loop(): '''Main loop, updating the capture agent state. ''' set_service_status(Service.AGENTSTATE, ServiceStatus.BUSY) notify.notify('READY=1') notify.notify('STATUS=Running') while not terminate(): notify.notify('WATCHDOG=1') next_update = timestamp() + config('agent', 'update_frequency') while not terminate() and timestamp() < next_update: time.sleep(0.1) if not terminate(): update_agent_state() logger.info('Shutting down agentstate service') set_service_status(Service.AGENTSTATE, ServiceStatus.STOPPED)
def control_loop(): '''Main loop, retrieving the schedule. ''' set_service_status(Service.SCHEDULE, ServiceStatus.BUSY) while not terminate(): # Try getting an updated schedule get_schedule() q = get_session().query(UpcomingEvent)\ .filter(UpcomingEvent.end > timestamp()) if q.count(): logger.info('Next scheduled recording: %s', datetime.fromtimestamp(q[0].start)) else: logger.info('No scheduled recording') next_update = timestamp() + config()['agent']['update_frequency'] while not terminate() and timestamp() < next_update: time.sleep(0.1) logger.info('Shutting down schedule service') set_service_status(Service.SCHEDULE, ServiceStatus.STOPPED)
def control_loop(): '''Main loop of the capture agent, retrieving and checking the schedule as well as starting the capture process if necessry. ''' set_service_status_immediate(Service.CAPTURE, ServiceStatus.IDLE) notify.notify('READY=1') notify.notify('STATUS=Waiting') while not terminate(): notify.notify('WATCHDOG=1') # Get next recording session = get_session() event = session.query(UpcomingEvent)\ .filter(UpcomingEvent.start <= timestamp())\ .filter(UpcomingEvent.end > timestamp())\ .first() if event: safe_start_capture(event) session.close() time.sleep(1.0) logger.info('Shutting down capture service') set_service_status(Service.CAPTURE, ServiceStatus.STOPPED)
def start_ingest(event): # Put metadata files on disk attachments = event.get_data().get('attach') workflow_config = '' for attachment in attachments: value = attachment.get('data') if attachment.get('fmttype') == 'application/text': workflow_def, workflow_config = get_config_params(value) filename = attachment.get('x-apple-filename') with open(os.path.join(event.directory(), filename), 'wb') as f: f.write(value.encode('utf-8')) # If we are a backup CA, we don't want to actually upload anything. So # let's just quit here. if config()['agent']['backup_mode']: return True # Upload everything set_service_status(Service.INGEST, ServiceStatus.BUSY) recording_state(event.uid, 'uploading') update_event_status(event, Status.UPLOADING) try: ingest(event.get_tracks(), event.directory(), event.uid, workflow_def, workflow_config) except: logger.error('Something went wrong during the upload') logger.error(traceback.format_exc()) # Update state if something went wrong recording_state(event.uid, 'upload_error') update_event_status(event, Status.FAILED_UPLOADING) set_service_status_immediate(Service.INGEST, ServiceStatus.IDLE) return False # Update state recording_state(event.uid, 'upload_finished') update_event_status(event, Status.FINISHED_UPLOADING) set_service_status_immediate(Service.INGEST, ServiceStatus.IDLE) return True
def control_loop(): '''Main loop of the capture agent, retrieving and checking the schedule as well as starting the capture process if necessry. ''' set_service_status_immediate(Service.INGEST, ServiceStatus.IDLE) notify.notify('READY=1') notify.notify('STATUS=Running') while not terminate(): notify.notify('WATCHDOG=1') # Get next recording session = get_session() event = session.query(RecordedEvent)\ .filter(RecordedEvent.status == Status.FINISHED_RECORDING).first() if event: delay = random.randint(config('ingest', 'delay_min'), config('ingest', 'delay_max')) logger.info("Delaying ingest for %s seconds", delay) time.sleep(delay) safe_start_ingest(event) session.close() time.sleep(1.0) logger.info('Shutting down ingest service') set_service_status(Service.INGEST, ServiceStatus.STOPPED)
def ingest(event): '''Ingest a finished recording to the Opencast server. ''' # Update status set_service_status(Service.INGEST, ServiceStatus.BUSY) notify.notify('STATUS=Uploading') recording_state(event.uid, 'uploading') update_event_status(event, Status.UPLOADING) # Select ingest service # The ingest service to use is selected at random from the available # ingest services to ensure that not every capture agent uses the same # service at the same time service_url = service('ingest', force_update=True) service_url = service_url[random.randrange(0, len(service_url))] logger.info('Selecting ingest service to use: ' + service_url) # create mediapackage logger.info('Creating new mediapackage') mediapackage = http_request(service_url + '/createMediaPackage') # extract workflow_def, workflow_config and add DC catalogs prop = 'org.opencastproject.capture.agent.properties' dcns = 'http://www.opencastproject.org/xsd/1.0/dublincore/' for attachment in event.get_data().get('attach'): data = attachment.get('data') if attachment.get('x-apple-filename') == prop: workflow_def, workflow_config = get_config_params(data) # dublin core catalogs elif attachment.get('fmttype') == 'application/xml' \ and dcns in data \ and config('ingest', 'upload_catalogs'): name = attachment.get('x-apple-filename', '').rsplit('.', 1)[0] logger.info('Adding %s DC catalog', name) fields = [('mediaPackage', mediapackage), ('flavor', 'dublincore/%s' % name), ('dublinCore', data.encode('utf-8'))] mediapackage = http_request(service_url + '/addDCCatalog', fields) else: logger.info('Not uploading %s', attachment.get('x-apple-filename')) continue # add track for (flavor, track) in event.get_tracks(): logger.info('Adding track (%s -> %s)', flavor, track) track = track.encode('ascii', 'ignore') fields = [('mediaPackage', mediapackage), ('flavor', flavor), ('BODY1', (pycurl.FORM_FILE, track))] mediapackage = http_request(service_url + '/addTrack', fields) # ingest logger.info('Ingest recording') fields = [('mediaPackage', mediapackage)] if workflow_def: fields.append(('workflowDefinitionId', workflow_def)) if event.uid: fields.append( ('workflowInstanceId', event.uid.encode('ascii', 'ignore'))) fields += workflow_config mediapackage = http_request(service_url + '/ingest', fields) # Update status recording_state(event.uid, 'upload_finished') update_event_status(event, Status.FINISHED_UPLOADING) if config('ingest', 'delete_after_upload'): directory = event.directory() logger.info("Removing uploaded event directory %s", directory) shutil.rmtree(directory) notify.notify('STATUS=Running') set_service_status_immediate(Service.INGEST, ServiceStatus.IDLE) logger.info('Finished ingest')