def remove_old_audio(self, date_today, keep_days, quiet=False):
if not self.exists(self._laftp_config['audio_path']):
return
import datetime
import schedule
pwd = self.pwd()
self.cwd(self._laftp_config['audio_path'])
remote_list = self.nlst()
deleted = []
for fname in remote_list:
if fname.startswith('.'):
continue
schedule_data = schedule.schedule_from_audio_file_name(fname)
if schedule_data is not None:
keep_days_this_file = schedule_data.get('extra', {}).get('days', [])
if keep_days_this_file:
try:
keep_days_this_file = int(keep_days_this_file[0])
except ValueError:
keep_days_this_file = keep_days
else:
keep_days_this_file = keep_days
earliest_keep_date = date_today - datetime.timedelta(days=keep_days_this_file-1)
if schedule_data['date'] < earliest_keep_date:
if not quiet:
print 'Deleting remote file:', fname, '...',
self.delete(fname)
if not quiet:
print 'done.'
deleted.append(fname)
self.cwd(pwd)
return deleted
def make_playlist_item(audio_fname):
import datetime
import os
import urllib
from rsab.listenagain import config, ListenAgainConfigError
import schedule
import utils
if not config.has_option('ftp', 'audio_path'):
raise ListenAgainConfigError('Missing option', 'ftp', 'audio_path')
audio_path = config.get('ftp', 'audio_path')
if not audio_path.endswith('/'):
audio_path += '/'
details = schedule.schedule_from_audio_file_name(audio_fname)
if details is None or details['show'].startswith('xx'):
return ''
template = Template('playlist-item')
template.title = utils.get_show_title_string(
details['show'],
details.get('presenters'),
)
template.date = details['date'].strftime('%a %d %b %Y')
template.date_compact = details['date'].strftime('%Y%m%d')
template.day_short = details['date'].strftime('%a').lower()
template.start_time = details['start'].strftime('%H:%M')
template.start_compact = details['start'].strftime('%H%M')
template.end_time = details['end'].strftime('%H:%M')
template.end_compact = details['end'].strftime('%H%M')
duration = (
datetime.datetime.combine(details['date'], details['end'])
- datetime.datetime.combine(details['date'], details['start'])
).seconds / 60.0 # approx minutes
ROUND_TO_MINUTES = 1
if duration % ROUND_TO_MINUTES < (ROUND_TO_MINUTES / 2.0):
duration = divmod(duration, ROUND_TO_MINUTES)[0] * ROUND_TO_MINUTES
else:
duration = (divmod(duration, ROUND_TO_MINUTES)[0] + 1) * ROUND_TO_MINUTES
duration_h, duration_m = divmod(duration, 60)
if duration_m:
duration_string = '%dh %dm' % (duration_h, duration_m)
else:
duration_string = '%dh' % (duration_h,)
template.duration = duration_string
hidden_input = '<input type="hidden" disabled="disabled" name="%s" value="%s" />'
template.show_names = hidden_input % ('show', details['show'])
template.presenter_names = '\n'.join([
hidden_input % ('presenter', presenter)
for presenter in details.get('presenters', [])
if presenter and presenter != details['show']
])
template.url = audio_path + urllib.quote(os.path.split(audio_fname)[1])
return template
def make_index_file(date, audio_fname_list, output_fname=None):
import datetime
import os
import schedule
import utils
from rsab.listenagain import config, ListenAgainConfigError
if not config.has_option('DEFAULT', 'output'):
raise ListenAgainConfigError('No [DEFAULT]/output config defined')
if output_fname is None:
output_fname = 'index.html'
output_fname = os.path.join(config.get('DEFAULT', 'output'), output_fname)
output_file = open(output_fname, 'w')
template = Template('player')
playlist_items = []
details_for_audio_files = []
show_name_mapping = {}
presenter_name_mapping = {}
for audio_fname in audio_fname_list:
playlist_items.append(str(make_playlist_item(audio_fname)))
details_for_audio_files.append(schedule.schedule_from_audio_file_name(audio_fname))
live_schedule = schedule.get_schedule(date + datetime.timedelta(days=1))
for details in details_for_audio_files + live_schedule:
if details is None:
continue
show_name = details['show']
show_title = utils.get_message(show_name, 'show', default=None)
if show_title is None:
show_title = utils.get_message(show_name, 'presenter', default=show_name)
if show_name and show_name not in show_name_mapping:
show_name_mapping[show_name] = show_title
for presenter in details.get('presenters', []):
if not presenter or presenter == show_name:
continue
if presenter not in presenter_name_mapping:
presenter_name_mapping[presenter] = utils.get_message(presenter, 'presenter', default=presenter)
template.playlist_items = '\n'.join(filter(None, playlist_items))
hidden_input = '<input type="hidden" disabled="disabled" name="%s" value="%s" />'
template.show_name_mapping = '\n'.join([
hidden_input % ('showname', '%s:%s' % pair)
for pair in show_name_mapping.items()
])
template.presenter_name_mapping = '\n'.join([
hidden_input % ('presentername', '%s:%s' % pair)
for pair in presenter_name_mapping.items()
])
template.live_schedule = '\n'.join([
hidden_input % (
'live_schedule',
'%s:%s' % (
details['start'].strftime('%H:%M'),
','.join([details['show']] + details.get('presenters', [])),
),
)
for details in schedule.get_schedule(date + datetime.timedelta(days=1))
])
output_file.write(str(template))
output_file.close()
return output_fname
def notify_all(uploaded_files):
from rsab.listenagain import config
import html
import schedule
from email.MIMEText import MIMEText
import os
import sets
if uploaded_files is None:
return
mapping = { None: sets.Set(uploaded_files), }
for fname in uploaded_files:
details = schedule.schedule_from_audio_file_name(fname)
for name in [details['show']] + details.get('presenters', []):
if name not in mapping:
mapping[name] = sets.Set()
mapping[name].add(fname)
config_sections = [
sname
for sname in config.sections()
if sname == 'notify' or sname.startswith('notify.')
]
if config.has_option('ftp', 'keep_days'):
keep_days = config.getint('ftp', 'keep_days')
else:
keep_days = 7
if config.has_option('main', 'userfacingserveraddress'):
remote_server = config.get('main', 'userfacingserveraddress')
else:
remote_server = 'http://listenagain.rsab.org'
if config.has_option('ftp', 'audio_path'):
remote_path = config.get('ftp', 'audio_path')
else:
remote_path = '/audio'
if config.has_option('email', 'from'):
email_from = config.get('email', 'from')
else:
email_from = DEFAULT_FROM
print
print 'Notification emails:'
messages = []
for section in config_sections:
print
print 'Section name:', section
if config.has_option(section, 'email'):
addresses = filter(None, [x.strip() for x in config.get(section, 'email').split(',')])
else:
addresses = []
print 'Addresses:', (addresses and ', '.join(addresses) or '(none)')
if not addresses:
print 'Skipping'
continue
if config.has_option(section, 'match'):
match_names = filter(None, [x.strip() for x in config.get(section, 'match').split(',')])
else:
match_names = []
print 'Requested:', ('*' in match_names and '(all)' or ', '.join(match_names))
if '*' in match_names:
notify_files = list(mapping[None])
elif match_names:
notify_files = sets.Set()
for name in match_names:
notify_files.update(mapping.get(name, []))
notify_files = list(notify_files)
else:
notify_files = []
if not notify_files:
print 'No files matched.'
continue
notify_files.sort()
if config.has_option(section, 'subject'):
subject = config.get(section, 'subject')
else:
subject = 'New files uploaded by Radio St Austell Bay'
if config.has_option(section, 'template'):
template_name = config.get(section, 'template')
else:
template_name = 'mail-notify-default.txt'
template = html.Template(template_name) # Not actually HTML!
template.keep_days = keep_days
files_string = '\n'.join([
' %s%s/%s' % (
remote_server,
remote_path,
os.path.split(fname)[1],
)
for fname in notify_files
])
template.files = files_string
print 'Files:'
print files_string
message = MIMEText(str(template))
message['Subject'] = subject
message['From'] = email_from
message['To'] = ', '.join(addresses)
messages.append(message)
print
print 'Sending messages...'
print
sent = _send(messages)
print
print 'Done. Sent %s of %s messages' % (sent, len(messages))
def run():
from optparse import OptionParser
import datetime
import glob
import os
import sys
import time
import audio
import html
import notify
import utils
import recorder
import remote
import schedule
option_parser = OptionParser()
option_parser.add_option(
"-p",
"--print",
dest="print_schedule",
action="store_true",
default=False,
help="Print information about the date but don't do anything",
)
option_parser.add_option("-w", "--wav", dest="wavs", action="store_true", default=False, help="Construct WAVs")
option_parser.add_option("-e", "--encode", dest="encode", action="store_true", default=False, help="Encode MP3s")
option_parser.add_option(
"-i", "--index", dest="index", action="store_true", default=False, help="Generate index pages"
)
option_parser.add_option(
"-u", "--upload", dest="upload", action="store_true", default=False, help="Upload data to web server"
)
option_parser.add_option(
"-c", "--config", dest="config_file", help="Specify alternative config file", metavar="FILE"
)
option_parser.add_option(
"-f",
"--filter",
dest="filter",
action="append",
help="Filter schedule to items containing at least one of the given show/presenter",
metavar="NAME",
)
options, args = option_parser.parse_args()
task_options = ["print_schedule", "wavs", "encode", "index", "upload"]
num_task_options_supplied = len([None for option_name in task_options if getattr(options, option_name, False)])
# No specific do-something options were given. Do everything.
if num_task_options_supplied == 0:
for option_name in task_options:
setattr(options, option_name, True)
config_files = utils.default_config_files()
if options.config_file is not None:
config_files.append(options.config_file)
config = utils.init_config(config_files)
date_string = "yesterday"
if len(args) == 1:
date_string = args[0]
date = utils.interpret_date_string(date_string)
start_time = time.time()
recorder.init_module()
bounds_and_files = recorder.get_bounds_and_files_for_date(date)
schedule_list = schedule.get_schedule(date, filter_items=options.filter)
if options.wavs or options.print_schedule:
if options.filter:
print "Schedule (filtered):"
else:
print "Schedule:"
schedule.print_schedule(schedule_list)
print
print "Recordings:"
recorder.print_bounds_and_files(bounds_and_files)
wav_files = None
if options.wavs:
wav_files = audio.make_wav_files(bounds_and_files, schedule_list)
mp3_files = None
if options.encode:
# Always rebuild WAVs list in case any are hanging around from before.
if True: # wav_files is None:
if config.has_option("main", "wavs"):
wavs_dir = config.get("main", "wavs")
else:
wavs_dir = os.getcwd()
wav_files = glob.glob(os.path.join(wavs_dir, "*.wav"))
# XXX Delete working WAVs? Only if MP3 was created for it.
mp3_files = [audio.encode_file(path) for path in wav_files]
if True: # XXX look for no-delete option later
print "Deleting local copies of WAVs..."
for (wav, mp3) in zip(wav_files, mp3_files):
if mp3 is not None and os.path.isfile(wav):
os.unlink(wav)
print " ", wav
print "done."
print
ftp_conn = None
remote_audio_files = []
if options.upload or options.index:
if config.has_option("ftp", "keep_days"):
keep_days = config.getint("ftp", "keep_days")
else:
keep_days = 7
earliest_keep_date = date - datetime.timedelta(days=keep_days - 1)
ftp_conn = remote.connect()
remote_audio_files = ftp_conn.get_list_of_audio_files()
# First make an index with no old files:
# XXX For now this ignores per-file limits, so will remove everything
# over N days old from the index temporarily. If a file has a higher
# number of days defined, it will be restored to the index later when
# it's not deleted -- but for a file with a lower number of days
# defined, it'll disappear later than it should.
audio_files_for_first_index = [
fname
for (fname, details) in [
(fname, schedule.schedule_from_audio_file_name(fname)) for fname in remote_audio_files
]
if details is not None and details["date"] >= earliest_keep_date
]
index_fname = html.make_index_file(date, audio_files_for_first_index)
if options.upload:
ftp_conn.storlines("STOR index.html", open(index_fname, "r"))
if options.upload:
ftp_conn.remove_old_audio(date, keep_days)
# XXX Here we should delete local copies of MP3s that are more than N
# days old, in case the upload has failed for more than N days.
pass
# Always build the list again as we can pick up files we missed before.
if True: # mp3_files is None:
if config.has_option("main", "mp3s"):
mp3s_dir = config.get("main", "mp3s")
else:
mp3s_dir = os.getcwd()
mp3_files = glob.glob(os.path.join(mp3s_dir, "*.mp3"))
try:
uploaded = remote.upload_audio(ftp_conn, mp3_files)
except:
import traceback
print "Exception uploading files"
traceback.print_exc(file=sys.stdout)
print "Continuing..."
# Reconnect (or grab the cached connection) in case there were failures
# during the upload. A better structure would see us making this
# completely transparent across all remote calls, but for now we focus
# on the big upload.
ftp_conn = remote.connect()
if True: # XXX look for no-delete option later
print "Deleting local copies of MP3s..."
for mp3_path in mp3_files:
if os.path.split(mp3_path)[1] in uploaded and os.path.isfile(mp3_path):
print " ", mp3_path
os.unlink(mp3_path)
print "done."
print
notify.notify_all(mp3_files)
if options.index:
# Second index file: whatever's on the server.
remote_audio_files = ftp_conn.get_list_of_audio_files()
index_fname = html.make_index_file(date, remote_audio_files)
if options.upload:
ftp_conn.storlines("STOR index.html", open(index_fname, "r"))
# XXX Now also sync up anything that's in the www directory
# (resource files such as JS, CSS, images, jPlayer...).
pass
if ftp_conn is not None:
ftp_conn.quit()
end_time = time.time()
if not options.print_schedule:
duration = end_time - start_time
print "Took %2.2dm %2.2ds" % divmod(duration, 60)
return 0
