def start_capture(upcoming_event):
'''Start the capture process, creating all necessary files and directories
as well as ingesting the captured files if no backup mode is configured.
'''
logger.info('Start recording')
# First move event to recording_event table
db = get_session()
event = db.query(RecordedEvent)\
.filter(RecordedEvent.uid == upcoming_event.uid)\
.filter(RecordedEvent.start == upcoming_event.start)\
.first()
if not event:
event = RecordedEvent(upcoming_event)
db.add(event)
db.commit()
duration = event.end - timestamp()
try_mkdir(config()['capture']['directory'])
os.mkdir(event.directory())
# Set state
set_service_status_immediate(Service.CAPTURE, ServiceStatus.BUSY)
recording_state(event.uid, 'capturing')
update_event_status(event, Status.RECORDING)
# Recording
tracks = recording_command(event.directory(), event.name(), duration)
event.set_tracks(tracks)
db.commit()
# Set status
set_service_status_immediate(Service.CAPTURE, ServiceStatus.IDLE)
update_event_status(event, Status.FINISHED_RECORDING)
def control_loop():
'''Main loop, retrieving the schedule.
'''
set_service_status_immediate(Service.SCHEDULE, ServiceStatus.BUSY)
notify.notify('READY=1')
while not terminate():
notify.notify('WATCHDOG=1')
# Try getting an updated schedule
get_schedule()
session = get_session()
next_event = session.query(UpcomingEvent)\
.filter(UpcomingEvent.end > timestamp())\
.order_by(UpcomingEvent.start)\
.first()
if next_event:
logger.info('Next scheduled recording: %s',
datetime.fromtimestamp(next_event.start))
notify.notify('STATUS=Next scheduled recording: %s' %
datetime.fromtimestamp(next_event.start))
else:
logger.info('No scheduled recording')
notify.notify('STATUS=No scheduled recording')
session.close()
next_update = timestamp() + config('agent', 'update_frequency')
while not terminate() and timestamp() < next_update:
time.sleep(0.1)
logger.info('Shutting down schedule service')
set_service_status_immediate(Service.SCHEDULE, ServiceStatus.STOPPED)
def ingest(event):
'''Ingest a finished recording to the Opencast server.
'''
# Update status
set_service_status(Service.INGEST, ServiceStatus.BUSY)
recording_state(event.uid, 'uploading')
update_event_status(event, Status.UPLOADING)
# Select ingest service
# The ingest service to use is selected at random from the available
# ingest services to ensure that not every capture agent uses the same
# service at the same time
service = config('service-ingest')
service = service[randrange(0, len(service))]
logger.info('Selecting ingest service to use: ' + service)
# create mediapackage
logger.info('Creating new mediapackage')
mediapackage = http_request(service + '/createMediaPackage')
# extract workflow_def, workflow_config and add DC catalogs
prop = 'org.opencastproject.capture.agent.properties'
dcns = 'http://www.opencastproject.org/xsd/1.0/dublincore/'
for attachment in event.get_data().get('attach'):
data = attachment.get('data')
if attachment.get('x-apple-filename') == prop:
workflow_def, workflow_config = get_config_params(data)
# Check for dublincore catalogs
elif attachment.get('fmttype') == 'application/xml' and dcns in data:
name = attachment.get('x-apple-filename', '').rsplit('.', 1)[0]
logger.info('Adding %s DC catalog' % name)
fields = [('mediaPackage', mediapackage),
('flavor', 'dublincore/%s' % name),
('dublinCore', data.encode('utf-8'))]
mediapackage = http_request(service + '/addDCCatalog', fields)
# add track
for (flavor, track) in event.get_tracks():
logger.info('Adding track ({0} -> {1})'.format(flavor, track))
track = track.encode('ascii', 'ignore')
fields = [('mediaPackage', mediapackage), ('flavor', flavor),
('BODY1', (pycurl.FORM_FILE, track))]
mediapackage = http_request(service + '/addTrack', fields)
# ingest
logger.info('Ingest recording')
fields = [('mediaPackage', mediapackage)]
if workflow_def:
fields.append(('workflowDefinitionId', workflow_def))
if event.uid:
fields.append(('workflowInstanceId',
event.uid.encode('ascii', 'ignore')))
fields += workflow_config
mediapackage = http_request(service + '/ingest', fields)
# Update status
recording_state(event.uid, 'upload_finished')
update_event_status(event, Status.FINISHED_UPLOADING)
set_service_status_immediate(Service.INGEST, ServiceStatus.IDLE)
def start_capture(upcoming_event):
'''Start the capture process, creating all necessary files and directories
as well as ingesting the captured files if no backup mode is configured.
'''
logger.info('Start recording')
# First move event to recording_event table
db = get_session()
event = db.query(RecordedEvent)\
.filter(RecordedEvent.uid == upcoming_event.uid)\
.filter(RecordedEvent.start == upcoming_event.start)\
.first()
if not event:
event = RecordedEvent(upcoming_event)
db.add(event)
db.commit()
try_mkdir(config()['capture']['directory'])
os.mkdir(event.directory())
# Set state
update_event_status(event, Status.RECORDING)
recording_state(event.uid, 'capturing')
set_service_status_immediate(Service.CAPTURE, ServiceStatus.BUSY)
# Recording
tracks = recording_command(event)
event.set_tracks(tracks)
db.commit()
# Set status
update_event_status(event, Status.FINISHED_RECORDING)
recording_state(event.uid, 'capture_finished')
set_service_status_immediate(Service.CAPTURE, ServiceStatus.IDLE)
def safe_start_ingest(event):
'''Start a capture process but make sure to catch any errors during this
process, log them but otherwise ignore them.
'''
try:
ingest(event)
except Exception:
logger.exception('Something went wrong during the upload')
# Update state if something went wrong
recording_state(event.uid, 'upload_error')
update_event_status(event, Status.FAILED_UPLOADING)
set_service_status_immediate(Service.INGEST, ServiceStatus.IDLE)
def safe_start_capture(event):
'''Start a capture process but make sure to catch any errors during this
process, log them but otherwise ignore them.
'''
try:
start_capture(event)
except Exception:
logger.error('Recording failed')
logger.error(traceback.format_exc())
# Update state
recording_state(event.uid, 'capture_error')
update_event_status(event, Status.FAILED_RECORDING)
set_service_status_immediate(Service.CAPTURE, ServiceStatus.IDLE)
def safe_start_capture(event):
'''Start a capture process but make sure to catch any errors during this
process, log them but otherwise ignore them.
'''
try:
start_capture(event)
return True
except Exception:
logger.error('Recording failed')
logger.error(traceback.format_exc())
# Update state
recording_state(event.uid, 'capture_error')
update_event_status(event, Status.FAILED_RECORDING)
set_service_status_immediate(Service.CAPTURE, ServiceStatus.IDLE)
return False
def safe_start_capture(event):
'''Start a capture process but make sure to catch any errors during this
process, log them but otherwise ignore them.
'''
try:
start_capture(event)
except Exception:
logger.exception('Recording failed')
# Update current status in Opencast
try:
set_service_status_immediate(Service.CAPTURE, ServiceStatus.IDLE)
recording_state(event.uid, 'capture_error')
update_event_status(event, Status.FAILED_RECORDING)
except Exception:
logger.exception('Could not update recording status')
def start_capture(db, upcoming_event):
'''Start the capture process, creating all necessary files and directories
as well as ingesting the captured files if no backup mode is configured.
'''
logger.info('Start recording')
# First move event to recording_event table
event = db.query(RecordedEvent)\
.filter(RecordedEvent.uid == upcoming_event.uid)\
.filter(RecordedEvent.start == upcoming_event.start)\
.first()
if not event:
event = RecordedEvent(upcoming_event)
db.add(event)
db.commit()
try_mkdir(config('capture', 'directory'))
try_mkdir(event.directory())
# Set state
update_event_status(event, Status.RECORDING)
recording_state(event.uid, 'capturing')
set_service_status_immediate(Service.CAPTURE, ServiceStatus.BUSY)
# Recording
files = recording_command(event)
# [(flavor,path),…]
event.set_tracks(list(zip(config('capture', 'flavors'), files)))
db.commit()
# Set status
# If part files exist, its an partial recording
part_files = any([glob.glob(f'{f}-part-*') for f in files])
if part_files:
state = Status.PARTIAL_RECORDING
elif config('agent', 'backup_mode'):
state = Status.PAUSED_AFTER_RECORDING
else:
state = Status.FINISHED_RECORDING
logger.info("Set %s to %s", event.uid, Status.str(state))
update_event_status(event, state)
recording_state(event.uid, 'capture_finished')
set_service_status_immediate(Service.CAPTURE, ServiceStatus.IDLE)
logger.info('Finished recording')
def control_loop():
'''Main loop of the capture agent, retrieving and checking the schedule as
well as starting the capture process if necessry.
'''
set_service_status_immediate(Service.INGEST, ServiceStatus.IDLE)
notify.notify('READY=1')
notify.notify('STATUS=Running')
while not terminate():
notify.notify('WATCHDOG=1')
# Get next recording
event = get_session().query(RecordedEvent)\
.filter(RecordedEvent.status ==
Status.FINISHED_RECORDING).first()
if event:
safe_start_ingest(event)
time.sleep(1.0)
logger.info('Shutting down ingest service')
set_service_status(Service.INGEST, ServiceStatus.STOPPED)
def control_loop():
'''Main loop of the capture agent, retrieving and checking the schedule as
well as starting the capture process if necessry.
'''
set_service_status_immediate(Service.CAPTURE, ServiceStatus.IDLE)
notify.notify('READY=1')
notify.notify('STATUS=Waiting')
while not terminate():
notify.notify('WATCHDOG=1')
# Get next recording
event = get_session().query(UpcomingEvent)\
.filter(UpcomingEvent.start <= timestamp())\
.filter(UpcomingEvent.end > timestamp())\
.first()
if event:
safe_start_capture(event)
time.sleep(1.0)
logger.info('Shutting down capture service')
set_service_status(Service.CAPTURE, ServiceStatus.STOPPED)
def start_ingest(event):
# Put metadata files on disk
attachments = event.get_data().get('attach')
workflow_config = ''
for attachment in attachments:
value = attachment.get('data')
if attachment.get('fmttype') == 'application/text':
workflow_def, workflow_config = get_config_params(value)
filename = attachment.get('x-apple-filename')
with open(os.path.join(event.directory(), filename), 'wb') as f:
f.write(value.encode('utf-8'))
# If we are a backup CA, we don't want to actually upload anything. So
# let's just quit here.
if config()['agent']['backup_mode']:
return True
# Upload everything
set_service_status(Service.INGEST, ServiceStatus.BUSY)
recording_state(event.uid, 'uploading')
update_event_status(event, Status.UPLOADING)
try:
ingest(event.get_tracks(), event.directory(), event.uid, workflow_def,
workflow_config)
except:
logger.error('Something went wrong during the upload')
logger.error(traceback.format_exc())
# Update state if something went wrong
recording_state(event.uid, 'upload_error')
update_event_status(event, Status.FAILED_UPLOADING)
set_service_status_immediate(Service.INGEST, ServiceStatus.IDLE)
return False
# Update state
recording_state(event.uid, 'upload_finished')
update_event_status(event, Status.FINISHED_UPLOADING)
set_service_status_immediate(Service.INGEST, ServiceStatus.IDLE)
return True
def test_set_service_status_immediate(self):
utils.http_request = lambda x, y=False: b''
utils.set_service_status_immediate(db.Service.SCHEDULE,
db.ServiceStatus.IDLE)
utils.set_service_status_immediate(db.Service.INGEST,
db.ServiceStatus.BUSY)
utils.set_service_status_immediate(db.Service.CAPTURE,
db.ServiceStatus.BUSY)
def test_set_service_status_immediate(self):
utils.http_request = lambda x, y=False: b''
utils.set_service_status_immediate(db.Service.SCHEDULE,
db.ServiceStatus.IDLE)
utils.set_service_status_immediate(db.Service.INGEST,
db.ServiceStatus.BUSY)
utils.set_service_status_immediate(db.Service.CAPTURE,
db.ServiceStatus.BUSY)
def control_loop():
'''Main loop of the capture agent, retrieving and checking the schedule as
well as starting the capture process if necessry.
'''
set_service_status_immediate(Service.INGEST, ServiceStatus.IDLE)
notify.notify('READY=1')
notify.notify('STATUS=Running')
while not terminate():
notify.notify('WATCHDOG=1')
# Get next recording
session = get_session()
event = session.query(RecordedEvent)\
.filter(RecordedEvent.status ==
Status.FINISHED_RECORDING).first()
if event:
delay = random.randint(config('ingest', 'delay_min'),
config('ingest', 'delay_max'))
logger.info("Delaying ingest for %s seconds", delay)
time.sleep(delay)
safe_start_ingest(event)
session.close()
time.sleep(1.0)
logger.info('Shutting down ingest service')
set_service_status(Service.INGEST, ServiceStatus.STOPPED)
def ingest(event):
'''Ingest a finished recording to the Opencast server.
'''
# Update status
set_service_status(Service.INGEST, ServiceStatus.BUSY)
notify.notify('STATUS=Uploading')
recording_state(event.uid, 'uploading')
update_event_status(event, Status.UPLOADING)
# Select ingest service
# The ingest service to use is selected at random from the available
# ingest services to ensure that not every capture agent uses the same
# service at the same time
service_url = service('ingest', force_update=True)
service_url = service_url[random.randrange(0, len(service_url))]
logger.info('Selecting ingest service to use: ' + service_url)
# create mediapackage
logger.info('Creating new mediapackage')
mediapackage = http_request(service_url + '/createMediaPackage')
# extract workflow_def, workflow_config and add DC catalogs
prop = 'org.opencastproject.capture.agent.properties'
dcns = 'http://www.opencastproject.org/xsd/1.0/dublincore/'
for attachment in event.get_data().get('attach'):
data = attachment.get('data')
if attachment.get('x-apple-filename') == prop:
workflow_def, workflow_config = get_config_params(data)
# dublin core catalogs
elif attachment.get('fmttype') == 'application/xml' \
and dcns in data \
and config('ingest', 'upload_catalogs'):
name = attachment.get('x-apple-filename', '').rsplit('.', 1)[0]
logger.info('Adding %s DC catalog', name)
fields = [('mediaPackage', mediapackage),
('flavor', 'dublincore/%s' % name),
('dublinCore', data.encode('utf-8'))]
mediapackage = http_request(service_url + '/addDCCatalog', fields)
else:
logger.info('Not uploading %s', attachment.get('x-apple-filename'))
continue
# add track
for (flavor, track) in event.get_tracks():
logger.info('Adding track (%s -> %s)', flavor, track)
track = track.encode('ascii', 'ignore')
fields = [('mediaPackage', mediapackage), ('flavor', flavor),
('BODY1', (pycurl.FORM_FILE, track))]
mediapackage = http_request(service_url + '/addTrack', fields)
# ingest
logger.info('Ingest recording')
fields = [('mediaPackage', mediapackage)]
if workflow_def:
fields.append(('workflowDefinitionId', workflow_def))
if event.uid:
fields.append(
('workflowInstanceId', event.uid.encode('ascii', 'ignore')))
fields += workflow_config
mediapackage = http_request(service_url + '/ingest', fields)
# Update status
recording_state(event.uid, 'upload_finished')
update_event_status(event, Status.FINISHED_UPLOADING)
if config('ingest', 'delete_after_upload'):
directory = event.directory()
logger.info("Removing uploaded event directory %s", directory)
shutil.rmtree(directory)
notify.notify('STATUS=Running')
set_service_status_immediate(Service.INGEST, ServiceStatus.IDLE)
logger.info('Finished ingest')
