def send_stream(path, download_name=None):
"""
Downloads a stream from the server. The path is callsign-date_duration.mp3
* callsign: The callsign returned by /stats
* date: in the format YYYYMMDDHHMM such as 201508011005 for
2015-08-01 10:05
* duration: A value, in minutes, to return.
The mp3 extension should be used regardless of the actual format of the stream -
although the audio returned will be in the streams' native format.
The streams are created and sent on-demand, so there may be a slight delay before
it starts.
"""
DB.incr('hits-dl')
base_dir = "%s%s/" % (config['storage'], misc.DIR_SLICES)
if not path.startswith(config['callsign']):
path = "%s-%s" % (config['callsign'], path)
if not path.endswith('.mp3'):
path = "%s.mp3" % path
file_name = base_dir + path
# If the file doesn't exist, then we need to slice it and create it based on our query.
# Also, if it's a zero byte file, then we try to create it again.
if not os.path.isfile(file_name) or os.path.getsize(file_name) == 0:
cloud.register_stream_list()
# This tells us that if it were to exist, it would be something
# like this.
request_info = audio.stream_info(file_name)
logging.info(("expected value", request_info))
# we can do something rather specific here ...
#
# first we get our generic stream list using our start_minute from the info.
stream_list, episode_list = cloud.find_streams(start_list=[request_info['start_minute']], duration_min=request_info['duration_sec'] / 60.0)
for ep in episode_list:
episode = ep[0]
first_slice = episode[0]
if first_slice['week_number'] == request_info['week_number']:
# This means that we've found the episode that we want
# We will block on this.
relative_start_minute = request_info['start_minute'] - first_slice['start_minute']
logging.info(episode)
audio.stitch_and_slice_process(file_list=episode, relative_start_minute=relative_start_minute, duration_minute=request_info['duration_sec'] / 60.0, destination_path=path)
# And break out of our loop ... now everything should exist.
break
return send_file_partial("%s/%s" % (base_dir, path), requested_path=path, file_name=download_name)
def get_next(info_query):
# Given a file, we look to see if there's another one which could come after -- we won't look in the database
if type(info_query) is str:
info_query = stream_info(info_query)
#
# We are looking for a file with the closest start time to
# the end time of our stream whose file size is greater than a
# certain threshold
#
target_time = info_query['start_date'] + timedelta(seconds=info_query['duration_sec'])
return get_file_for_ts(target_time=target_time, bias=None, exclude_path=info_query['name'])
def register_stream_list(reindex=False):
import lib.misc as misc
# Find the local streams and make sure they are all registered in the sqlite3 database.
#
# Get the existing streams as a set
#
# If we are asked to re-index (due to trying to fix a bug) then we ignore what we have
# and just go ahead and do everything.
#
if reindex:
all_registered = set([])
else:
all_registered = set(DB.all('streams', ['name']))
# There should be a smarter way to do this ... you'd think. We should also
# be more faithfully giving things extensions since it's not 100% mp3
all_files = set(glob('%s/*.mp3' % misc.DIR_STREAMS))
diff = all_files.difference(all_registered)
# This is a list of files we haven't scanned yet...
if not diff: return True
# This basically means we could still be writing
# this file.
#
# We take the cascade time and then buffer it by a minute, just
# to be sure.
#
# If the creation time is less then this then we don't register this
# until later.
cutoff = time.mktime((
datetime.now() -
timedelta(minutes=1, seconds=misc.config['cascade_time'])).timetuple())
for fname in diff:
if len(fname) == 0 or os.path.getctime(fname) > cutoff:
next
info = stream_info(fname)
if not info:
continue
DB.register_stream(info)
if not misc.manager_is_running():
logging.info("Manager is gone, shutting down")
raise Exception()
def get_next(info_query):
# Given a file, we look to see if there's another one which could come after -- we won't look in the database
if type(info_query) is str:
info_query = stream_info(info_query)
#
# We are looking for a file with the closest start time to
# the end time of our stream whose file size is greater than a
# certain threshold
#
target_time = info_query['start_date'] + timedelta(
seconds=info_query['duration_sec'])
return get_file_for_ts(target_time=target_time,
bias=None,
exclude_path=info_query['name'])
def register_stream_list(reindex=False):
import lib.misc as misc
# Find the local streams and make sure they are all registered in the sqlite3 database.
#
# Get the existing streams as a set
#
# If we are asked to re-index (due to trying to fix a bug) then we ignore what we have
# and just go ahead and do everything.
#
if reindex:
all_registered = set([])
else:
all_registered = set(DB.all('streams', ['name']))
# There should be a smarter way to do this ... you'd think. We should also
# be more faithfully giving things extensions since it's not 100% mp3
all_files = set(glob('%s/*.mp3' % misc.DIR_STREAMS))
diff = all_files.difference(all_registered)
# This is a list of files we haven't scanned yet...
if not diff: return True
# This basically means we could still be writing
# this file.
#
# We take the cascade time and then buffer it by a minute, just
# to be sure.
#
# If the creation time is less then this then we don't register this
# until later.
cutoff = time.mktime((datetime.now() - timedelta(minutes=1, seconds=misc.config['cascade_time'])).timetuple())
for fname in diff:
if len(fname) == 0 or os.path.getctime(fname) > cutoff:
next
info = stream_info(fname)
if not info:
continue
DB.register_stream(info)
if not misc.manager_is_running():
logging.info("Manager is gone, shutting down")
raise Exception()
def send_stream(path, download_name=None):
"""
Downloads a stream from the server. The path is callsign-date_duration.mp3
* callsign: The callsign returned by /stats
* date: in the format YYYYMMDDHHMM such as 201508011005 for
2015-08-01 10:05
* duration: A value, in minutes, to return.
The mp3 extension should be used regardless of the actual format of the stream -
although the audio returned will be in the streams' native format.
The streams are created and sent on-demand, so there may be a slight delay before
it starts.
"""
DB.incr('hits-dl')
base_dir = "%s%s/" % (config['storage'], misc.DIR_SLICES)
if not path.startswith(config['callsign']):
path = "%s-%s" % (config['callsign'], path)
if not path.endswith('.mp3'):
path = "%s.mp3" % path
file_name = base_dir + path
# If the file doesn't exist, then we need to slice it and create it based on our query.
# Also, if it's a zero byte file, then we try to create it again.
if not os.path.isfile(file_name) or os.path.getsize(file_name) == 0:
cloud.register_stream_list()
# This tells us that if it were to exist, it would be something
# like this.
request_info = audio.stream_info(file_name)
logging.info(("expected value", request_info))
# we can do something rather specific here ...
#
# first we get our generic stream list using our start_minute from the info.
stream_list, episode_list = cloud.find_streams(
start_list=[request_info['start_minute']],
duration_min=request_info['duration_sec'] / 60.0)
for ep in episode_list:
episode = ep[0]
first_slice = episode[0]
if first_slice['week_number'] == request_info['week_number']:
# This means that we've found the episode that we want
# We will block on this.
relative_start_minute = request_info[
'start_minute'] - first_slice['start_minute']
logging.info(episode)
audio.stitch_and_slice_process(
file_list=episode,
relative_start_minute=relative_start_minute,
duration_minute=request_info['duration_sec'] / 60.0,
destination_path=path)
# And break out of our loop ... now everything should exist.
break
return send_file_partial("%s/%s" % (base_dir, path),
requested_path=path,
file_name=download_name)
def stream_download(callsign, url, my_pid, file_name):
# Curl interfacing which downloads the stream to disk.
# Follows redirects and parses out basic m3u.
#pid = misc.change_proc_name("%s-download" % callsign)
nl = {'stream': None, 'curl_handle': None, 'pid': my_pid, 'ix': 0, 'ct': 0, 'start': 0}
def progress(download_t, download_d, upload_t, upload_d):
if nl['start'] == 0:
nl['start'] = time.time()
nl['ct'] += 1
if nl['pid'] <= g_download_kill_pid - 2:
logging.info("Stopping download #%d through progress" % nl['pid'])
raise TypeError("Download Stop")
return False
#logging.info("progress: %f %d %d" % ((time.time() - nl['start']) / nl['ct'], nl['pid'], download_d ));
def catchall(which, what):
logging.info("%d: %s %s" % (nl['pid'], which, str(what)))
def catch_read(what):
return catchall('read', what)
def catch_debug(what, origin):
if what == pycurl.INFOTYPE_TEXT:
return catchall('debug', json.dumps([what, str(origin)], ensure_ascii=False))
def cback(data):
global g_download_kill_pid
"""
if len(data):
catchall('download', json.dumps([g_download_kill_pid, nl['pid'], len(data)]))
else:
catchall('download', json.dumps([g_download_kill_pid, 'no data']))
"""
# print nl['pid'], g_download_kill_pid
if nl['pid'] <= g_download_kill_pid or not data:
logging.info("Stopping download #%d" % nl['pid'])
return False
# misc.params can fail based on a shutdown sequence.
if misc is None or misc.params is None or not misc.manager_is_running():
# if misc is not None:
# misc.shutdown()
return False
elif not misc.params['shutdown_time']:
if not misc.download_ipc.empty():
what, value = misc.download_ipc.get(False)
if what == 'shutdown_time':
misc.params['shutdown_time'] = value
elif TS.unixtime('dl') > misc.params['shutdown_time']:
raise TypeError("Download Stop")
if misc.params['isFirst'] == True:
misc.params['isFirst'] = False
if len(data) < 800:
try:
data_string = data.decode('utf-8')
if re.match('https?://', data_string):
# If we are getting a redirect then we don't mind, we
# just put it in the stream and then we leave
misc.queue.put(('stream', data_string.strip()))
return False
# A pls style playlist
elif re.findall('File\d', data_string, re.M):
logging.info('%d: Found a pls, using the File1 parameter' % (nl['pid'], ))
matches = re.findall('File1=(.*)\n', data_string, re.M)
misc.queue.put(('stream', matches[0].strip()))
return False
# If it gets here it's binary ... I guess that's fine.
except:
pass
# This provides a reliable way to determine bitrate. We look at how much
# data we've received between two time periods
misc.queue.put(('heartbeat', (TS.unixtime('hb'), nl['pid'], len(data))))
if not nl['stream']:
try:
nl['stream'] = open(file_name, 'wb')
except Exception as exc:
logging.critical("%d: Unable to open %s. Can't record. Must exit." % (nl['pid'], file_name))
return False
nl['stream'].write(data)
misc.params['isFirst'] = True
curl_handle = pycurl.Curl()
curl_handle.setopt(curl_handle.URL, url)
curl_handle.setopt(pycurl.WRITEFUNCTION, cback)
curl_handle.setopt(pycurl.FOLLOWLOCATION, True)
curl_handle.setopt(pycurl.NOPROGRESS, False)
try:
curl_handle.setopt(pycurl.XFERINFOFUNCTION, progress)
except (NameError, AttributeError):
curl_handle.setopt(pycurl.PROGRESSFUNCTION, progress)
curl_handle.setopt(pycurl.VERBOSE, 1)
#curl_handle.setopt(pycurl.READFUNCTION, catch_read)
curl_handle.setopt(pycurl.DEBUGFUNCTION, catch_debug)
curl_handle.setopt(pycurl.USERAGENT, "indycast %s. See http://indycast.net/ for more info." % misc.__version__)
nl['curl_handle'] = curl_handle
try:
curl_handle.perform()
except pycurl.error as exc:
if exc.args[0] != 23:
logging.warning("%d: Couldn't resolve or connect to %s." % (nl['pid'], url))
except:
logging.warning("%d: Unhandled exception" % (nl['pid'], ))
pass
curl_handle.close()
if nl['stream'] and type(nl['stream']) != bool:
nl['stream'].close()
# This is where we are sure of the stats on this file, because
# we just closed it ... so we can register it here.
info = audio.stream_info(file_name)
DB.register_stream(info)
def get_file_for_ts(target_time, bias=None, exclude_path=None):
import lib.misc as misc
# Given a datetime target_time, this finds the closest file either with a bias
# of +1 for after, -1 for before (or within) or no bias for the closest match.
#
# An exclude_path can be set to remove it from the candidates to be searched
best_before_time = None
best_before_info = None
best_after_time = None
best_after_info = None
time_to_beat = None
current_winner = None
#print "-----------------------"
for candidate_path in glob('%s/*.mp3' % misc.DIR_STREAMS):
if candidate_path == exclude_path: continue
info_candidate = stream_info(candidate_path)
if not info_candidate or info_candidate['duration_sec'] is None or info_candidate['duration_sec'] < 10.0:
next
difference = info_candidate['start_date'] - target_time
# This means we want to be strictly later
# If our difference is before, which means we are earlier,
# then we exclude this
#
# BUGBUG: There's a hole in here ... pretend there's an expansive file starting at t0 and
# a small one at t1 where start time of t0 < t1 so t1 is the file that is selected even though
# t0 is a better candidate.
#
if difference < timedelta() and (not best_before_time or difference > best_before_time):
best_before_time = difference
best_before_info = info_candidate
# If we want something earlier and the start date is AFTER
# our target time then we bail
elif difference > timedelta() and (not best_after_time or difference < best_after_time):
best_after_time = difference
best_after_info = info_candidate
# print(target_time, "\n", best_before_time, best_before_info, "\n", best_after_time, best_after_info, bias)
if not best_before_info:
if best_after_info:
best_before_time = -best_after_time
best_before_info = best_after_info
else:
# if we couldn't find anything
return None, None
if bias == -1:
# Make sure that our candidate has our time within it
# print best_before_info['start_date'], timedelta(seconds=best_before_info['duration_sec']) , target_time
if best_before_info and best_before_info['start_date'] + timedelta(seconds=best_before_info['duration_sec']) > target_time:
# This means that we have found a valid file and we can return the successful target_time
# and our info
return best_before_info, target_time
# Otherwise that means that our best time doesn't actually have our target time!
# So we return where we ought to start and the file we can start at
if best_after_info:
return best_after_info, best_after_info['start_date']
else:
return None, None
if bias == None:
if not best_after_info or best_before_info and (abs(best_before_time) < abs(best_after_time)):
return best_before_info, max(target_time, best_before_info['start_date'])
return best_after_info, min(target_time, best_after_info['start_date'])
if bias == +1:
# print best_after_info, best_before_info, exclude_path
if not best_after_info:
return None, target_time
return best_after_info, min(target_time, best_after_info['start_date'])
def find_streams(start_list, duration_min):
import lib.misc as misc
# Given a start week minute this looks for streams in the storage
# directory that match it - regardless of duration ... so it may return
# partial shows results.
stream_list = []
if type(start_list) is int:
start_list = [start_list]
# Sort nominally - since we have unix time in the name, this should come out
# as sorted by time for us for free.
stitch_list = []
episode_list = []
# So we have a start list, we are about to query our database using the start_minute
# and end_minute field ... to get end_minue we need to make use of our duration.
#
# timeline ->
#
# ################### << Region we want
# start_sea#ch end_search # << Search
# V V
# | | | | | | | << Minute
# a b b c
#
# so we want
# (a) start_minute < start_search and end_minute >= start_search ||
# (b) start_minute > start_search and end_minute <= end_search ||
# (c) start_minute < end_search and end_minute >= end_search
#
condition_list = []
for start in start_list:
end_search = (start + duration_min) % TS.MINUTES_PER_WEEK
# print start, duration_min, end_search
condition_list.append('start_minute < %d and end_minute >= %d' % (start, start))
condition_list.append('start_minute >= %d and end_minute >= %d and end_minute <= %d' % (start, start, end_search))
condition_list.append('start_minute < %d and end_minute >= %d' % (end_search, end_search))
condition_query = "((%s))" % ') or ('.join(condition_list)
# see https://github.com/kristopolous/DRR/issues/50 - nah this shit is buggy
condition_query += " and start_unix < datetime(%d, 'unixepoch', 'localtime')" % (TS.sec_now() - misc.config['cascade_time'] + 3)
full_query = "select * from streams where {} order by strftime('%Y', start_unix) * 524160 + week_number * 10080 + start_minute asc".format(condition_query)
entry_list = DB.map(DB.run(full_query).fetchall(), 'streams')
#logging.info(full_query)
#logging.info(entry_list)
# print full_query, len(entry_list)
# We want to make sure that we break down the stream_list into days. We can't JUST look at the week
# number since we permit feed requests for shows which may have multiple days. Since this is leaky
# data that we don't keep via our separation of concerns, we use a little hack to figure this out.
by_episode = []
episode = []
cutoff_minute = 0
current_week = 0
for entry in entry_list:
# look at start minute, if it's > 12 * cascade time (by default 3 hours), then we presume this is a new episode.
if entry['start_minute'] > cutoff_minute or entry['week_number'] != current_week:
if len(episode):
by_episode.append(episode)
episode = []
cutoff_minute = entry['start_minute'] + (12 * misc.config['cascade_time']) % TS.MINUTES_PER_WEEK
current_week = entry['week_number']
# We know by definition that every entry in our stream_list is a valid thing we need
# to look at. We just need to make sure we break them down by episode
episode.append(entry)
if len(episode):
by_episode.append(episode)
#print len(by_episode), condition_query
# Start the creation of the audio files.
for episode in by_episode:
# We blur the test start to a bigger window
test_start = (episode[0]['start_minute'] / (60 * 4))
for week_start in start_list:
# Blur the query start to the same window
query_start = week_start / (60 * 4)
# This shouldn't be necessary but let's do it anyway
if abs(query_start - test_start) <= 1:
# Under these conditions we can say that this episode
# can be associated with this particular start time
# The start_minute is based on the week
offset_start = week_start - episode[0]['start_minute']
fname = audio.stream_name(episode, week_start, duration_min)
# print '--name',episode[0]['name'], fname
# We get the name that it will be and then append that
stream_list.append(stream_info(fname))
# print offset_start, duration_min, episode
episode_list.append((episode, offset_start, duration_min))
break
# print stream_list, "\nbreak\n", episode_list, "\nasfdasdf\n"
return stream_list, episode_list
def stream_download(callsign, url, my_pid, file_name):
# Curl interfacing which downloads the stream to disk.
# Follows redirects and parses out basic m3u.
pid = misc.change_proc_name("%s-download" % callsign)
nl = {'stream': None, 'curl_handle': None}
def dl_stop(signal, frame):
sys.exit(0)
def cback(data):
if not misc.params['shutdown_time']:
if not misc.download_ipc.empty():
what, value = misc.download_ipc.get(False)
if what == 'shutdown_time':
misc.params['shutdown_time'] = value
elif TS.unixtime('dl') > misc.params['shutdown_time']:
sys.exit(0)
if misc.params['isFirst'] == True:
misc.params['isFirst'] = False
if len(data) < 800:
if re.match('https?://', data):
# If we are getting a redirect then we don't mind, we
# just put it in the stream and then we leave
misc.queue.put(('stream', data.strip()))
return True
# A pls style playlist
elif re.findall('File\d', data, re.M):
logging.info('Found a pls, using the File1 parameter')
matches = re.findall('File1=(.*)\n', data, re.M)
misc.queue.put(('stream', matches[0].strip()))
return True
# This provides a reliable way to determine bitrate. We look at how much
# data we've received between two time periods
misc.queue.put(('heartbeat', (TS.unixtime('hb'), len(data))))
if not nl['stream']:
try:
nl['stream'] = open(file_name, 'w')
except Exception as exc:
logging.critical(
"Unable to open %s. Can't record. Must exit." % file_name)
sys.exit(-1)
nl['stream'].write(data)
if not misc.manager_is_running():
misc.shutdown()
# signal.signal(signal.SIGTERM, dl_stop)
misc.params['isFirst'] = True
curl_handle = pycurl.Curl()
curl_handle.setopt(curl_handle.URL, url)
curl_handle.setopt(pycurl.WRITEFUNCTION, cback)
curl_handle.setopt(pycurl.FOLLOWLOCATION, True)
nl['curl_handle'] = curl_handle
try:
curl_handle.perform()
except TypeError as exc:
logging.info('Properly shutting down.')
except Exception as exc:
logging.warning("Couldn't resolve or connect to %s." % url)
curl_handle.close()
if nl['stream'] and type(nl['stream']) != bool:
nl['stream'].close()
# This is where we are sure of the stats on this file, because
# we just closed it ... so we can register it here.
info = audio.stream_info(file_name)
DB.register_stream(info)
def get_file_for_ts(target_time, bias=None, exclude_path=None):
import lib.misc as misc
# Given a datetime target_time, this finds the closest file either with a bias
# of +1 for after, -1 for before (or within) or no bias for the closest match.
#
# An exclude_path can be set to remove it from the candidates to be searched
best_before_time = None
best_before_info = None
best_after_time = None
best_after_info = None
time_to_beat = None
current_winner = None
#print "-----------------------"
for candidate_path in glob('%s/*.mp3' % misc.DIR_STREAMS):
if candidate_path == exclude_path: continue
info_candidate = stream_info(candidate_path)
if not info_candidate or info_candidate[
'duration_sec'] is None or info_candidate[
'duration_sec'] < 10.0:
next
difference = info_candidate['start_date'] - target_time
# This means we want to be strictly later
# If our difference is before, which means we are earlier,
# then we exclude this
#
# BUGBUG: There's a hole in here ... pretend there's an expansive file starting at t0 and
# a small one at t1 where start time of t0 < t1 so t1 is the file that is selected even though
# t0 is a better candidate.
#
if difference < timedelta() and (not best_before_time
or difference > best_before_time):
best_before_time = difference
best_before_info = info_candidate
# If we want something earlier and the start date is AFTER
# our target time then we bail
elif difference > timedelta() and (not best_after_time
or difference < best_after_time):
best_after_time = difference
best_after_info = info_candidate
# print(target_time, "\n", best_before_time, best_before_info, "\n", best_after_time, best_after_info, bias)
if not best_before_info:
if best_after_info:
best_before_time = -best_after_time
best_before_info = best_after_info
else:
# if we couldn't find anything
return None, None
if bias == -1:
# Make sure that our candidate has our time within it
# print best_before_info['start_date'], timedelta(seconds=best_before_info['duration_sec']) , target_time
if best_before_info and best_before_info['start_date'] + timedelta(
seconds=best_before_info['duration_sec']) > target_time:
# This means that we have found a valid file and we can return the successful target_time
# and our info
return best_before_info, target_time
# Otherwise that means that our best time doesn't actually have our target time!
# So we return where we ought to start and the file we can start at
if best_after_info:
return best_after_info, best_after_info['start_date']
else:
return None, None
if bias == None:
if not best_after_info or best_before_info and (abs(best_before_time) <
abs(best_after_time)):
return best_before_info, max(target_time,
best_before_info['start_date'])
return best_after_info, min(target_time, best_after_info['start_date'])
if bias == +1:
# print best_after_info, best_before_info, exclude_path
if not best_after_info:
return None, target_time
return best_after_info, min(target_time, best_after_info['start_date'])
def find_streams(start_list, duration_min):
import lib.misc as misc
# Given a start week minute this looks for streams in the storage
# directory that match it - regardless of duration ... so it may return
# partial shows results.
stream_list = []
if type(start_list) is int:
start_list = [start_list]
# Sort nominally - since we have unix time in the name, this should come out
# as sorted by time for us for free.
stitch_list = []
episode_list = []
# So we have a start list, we are about to query our database using the start_minute
# and end_minute field ... to get end_minue we need to make use of our duration.
#
# timeline ->
#
# ################### << Region we want
# start_sea#ch end_search # << Search
# V V
# | | | | | | | << Minute
# a b b c
#
# so we want
# (a) start_minute < start_search and end_minute >= start_search ||
# (b) start_minute > start_search and end_minute <= end_search ||
# (c) start_minute < end_search and end_minute >= end_search
#
condition_list = []
for start in start_list:
end_search = (start + duration_min) % TS.MINUTES_PER_WEEK
# print start, duration_min, end_search
condition_list.append('start_minute < %d and end_minute >= %d' %
(start, start))
condition_list.append(
'start_minute >= %d and end_minute >= %d and end_minute <= %d' %
(start, start, end_search))
condition_list.append('start_minute < %d and end_minute >= %d' %
(end_search, end_search))
condition_query = "((%s))" % ') or ('.join(condition_list)
# see https://github.com/kristopolous/DRR/issues/50 - nah this shit is buggy
condition_query += " and start_unix < datetime(%d, 'unixepoch', 'localtime')" % (
TS.sec_now() - misc.config['cascade_time'] + 3)
full_query = "select * from streams where %s order by week_number * 10080 + start_minute asc" % condition_query
entry_list = DB.map(DB.run(full_query).fetchall(), 'streams')
#logging.info(full_query)
#logging.info(entry_list)
# print full_query, len(entry_list)
# We want to make sure that we break down the stream_list into days. We can't JUST look at the week
# number since we permit feed requests for shows which may have multiple days. Since this is leaky
# data that we don't keep via our separation of concerns, we use a little hack to figure this out.
by_episode = []
episode = []
cutoff_minute = 0
current_week = 0
for entry in entry_list:
# look at start minute, if it's > 12 * cascade time (by default 3 hours), then we presume this is a new episode.
if entry['start_minute'] > cutoff_minute or entry[
'week_number'] != current_week:
if len(episode):
by_episode.append(episode)
episode = []
cutoff_minute = entry['start_minute'] + (
12 * misc.config['cascade_time']) % TS.MINUTES_PER_WEEK
current_week = entry['week_number']
# We know by definition that every entry in our stream_list is a valid thing we need
# to look at. We just need to make sure we break them down by episode
episode.append(entry)
if len(episode):
by_episode.append(episode)
#print len(by_episode), condition_query
# Start the creation of the audio files.
for episode in by_episode:
# We blur the test start to a bigger window
test_start = (episode[0]['start_minute'] / (60 * 4))
for week_start in start_list:
# Blur the query start to the same window
query_start = week_start / (60 * 4)
# This shouldn't be necessary but let's do it anyway
if abs(query_start - test_start) <= 1:
# Under these conditions we can say that this episode
# can be associated with this particular start time
# The start_minute is based on the week
offset_start = week_start - episode[0]['start_minute']
fname = audio.stream_name(episode, week_start, duration_min)
# print '--name',episode[0]['name'], fname
# We get the name that it will be and then append that
stream_list.append(stream_info(fname))
# print offset_start, duration_min, episode
episode_list.append((episode, offset_start, duration_min))
break
# print stream_list, "\nbreak\n", episode_list, "\nasfdasdf\n"
return stream_list, episode_list
#!/usr/bin/python3
import sys
import lib.audio as audio
for file_name in sys.argv[1:]:
print("%s:" % file_name)
print(audio.stream_info(file_name))
sig, block = audio.signature(file_name)
print({'block_count': len(block), 'format': audio._LASTFORMAT})
def stream_download(callsign, url, my_pid, file_name):
# Curl interfacing which downloads the stream to disk.
# Follows redirects and parses out basic m3u.
#pid = misc.change_proc_name("%s-download" % callsign)
nl = {
'stream': None,
'curl_handle': None,
'pid': my_pid,
'ix': 0,
'ct': 0,
'start': 0
}
def progress(download_t, download_d, upload_t, upload_d):
if nl['start'] == 0:
nl['start'] = time.time()
nl['ct'] += 1
if nl['pid'] <= g_download_kill_pid - 2:
logging.info("Stopping download #%d through progress" % nl['pid'])
raise TypeError("Download Stop")
return False
#logging.info("progress: %f %d %d" % ((time.time() - nl['start']) / nl['ct'], nl['pid'], download_d ));
def catchall(which, what):
logging.info("%d: %s %s" % (nl['pid'], which, str(what)))
def catch_read(what):
return catchall('read', what)
def catch_debug(what, origin):
if what == pycurl.INFOTYPE_TEXT:
return catchall(
'debug', json.dumps([what, str(origin)], ensure_ascii=False))
def cback(data):
global g_download_kill_pid
"""
if len(data):
catchall('download', json.dumps([g_download_kill_pid, nl['pid'], len(data)]))
else:
catchall('download', json.dumps([g_download_kill_pid, 'no data']))
"""
# print nl['pid'], g_download_kill_pid
if nl['pid'] <= g_download_kill_pid or not data:
logging.info("Stopping download #%d" % nl['pid'])
return False
# misc.params can fail based on a shutdown sequence.
if misc is None or misc.params is None or not misc.manager_is_running(
):
# if misc is not None:
# misc.shutdown()
return False
elif not misc.params['shutdown_time']:
if not misc.download_ipc.empty():
what, value = misc.download_ipc.get(False)
if what == 'shutdown_time':
misc.params['shutdown_time'] = value
elif TS.unixtime('dl') > misc.params['shutdown_time']:
raise TypeError("Download Stop")
if misc.params['isFirst'] == True:
misc.params['isFirst'] = False
if len(data) < 800:
try:
data_string = data.decode('utf-8')
if re.match('https?://', data_string):
# If we are getting a redirect then we don't mind, we
# just put it in the stream and then we leave
misc.queue.put(('stream', data_string.strip()))
return False
# A pls style playlist
elif re.findall('File\d', data_string, re.M):
logging.info(
'%d: Found a pls, using the File1 parameter' %
(nl['pid'], ))
matches = re.findall('File1=(.*)\n', data_string, re.M)
misc.queue.put(('stream', matches[0].strip()))
return False
# If it gets here it's binary ... I guess that's fine.
except:
pass
# This provides a reliable way to determine bitrate. We look at how much
# data we've received between two time periods
misc.queue.put(
('heartbeat', (TS.unixtime('hb'), nl['pid'], len(data))))
if not nl['stream']:
try:
nl['stream'] = open(file_name, 'wb')
except Exception as exc:
logging.critical(
"%d: Unable to open %s. Can't record. Must exit." %
(nl['pid'], file_name))
return False
nl['stream'].write(data)
misc.params['isFirst'] = True
curl_handle = pycurl.Curl()
curl_handle.setopt(curl_handle.URL, url)
curl_handle.setopt(pycurl.WRITEFUNCTION, cback)
curl_handle.setopt(pycurl.FOLLOWLOCATION, True)
curl_handle.setopt(pycurl.NOPROGRESS, False)
try:
curl_handle.setopt(pycurl.XFERINFOFUNCTION, progress)
except (NameError, AttributeError):
curl_handle.setopt(pycurl.PROGRESSFUNCTION, progress)
curl_handle.setopt(pycurl.VERBOSE, 1)
#curl_handle.setopt(pycurl.READFUNCTION, catch_read)
curl_handle.setopt(pycurl.DEBUGFUNCTION, catch_debug)
curl_handle.setopt(
pycurl.USERAGENT,
"indycast %s. See http://indycast.net/ for more info." %
misc.__version__)
nl['curl_handle'] = curl_handle
try:
curl_handle.perform()
except pycurl.error as exc:
if exc.args[0] != 23:
logging.warning("%d: Couldn't resolve or connect to %s." %
(nl['pid'], url))
except:
logging.warning("%d: Unhandled exception" % (nl['pid'], ))
pass
curl_handle.close()
if nl['stream'] and type(nl['stream']) != bool:
nl['stream'].close()
# This is where we are sure of the stats on this file, because
# we just closed it ... so we can register it here.
info = audio.stream_info(file_name)
DB.register_stream(info)
def stream_download(callsign, url, my_pid, file_name):
# Curl interfacing which downloads the stream to disk.
# Follows redirects and parses out basic m3u.
pid = misc.change_proc_name("%s-download" % callsign)
nl = {'stream': None, 'curl_handle': None}
def dl_stop(signal, frame):
sys.exit(0)
def cback(data):
if not misc.params['shutdown_time']:
if not misc.download_ipc.empty():
what, value = misc.download_ipc.get(False)
if what == 'shutdown_time':
misc.params['shutdown_time'] = value
elif TS.unixtime('dl') > misc.params['shutdown_time']:
sys.exit(0)
if misc.params['isFirst'] == True:
misc.params['isFirst'] = False
if len(data) < 800:
if re.match('https?://', data):
# If we are getting a redirect then we don't mind, we
# just put it in the stream and then we leave
misc.queue.put(('stream', data.strip()))
return True
# A pls style playlist
elif re.findall('File\d', data, re.M):
logging.info('Found a pls, using the File1 parameter')
matches = re.findall('File1=(.*)\n', data, re.M)
misc.queue.put(('stream', matches[0].strip()))
return True
# This provides a reliable way to determine bitrate. We look at how much
# data we've received between two time periods
misc.queue.put(('heartbeat', (TS.unixtime('hb'), len(data))))
if not nl['stream']:
try:
nl['stream'] = open(file_name, 'w')
except Exception as exc:
logging.critical("Unable to open %s. Can't record. Must exit." % file_name)
sys.exit(-1)
nl['stream'].write(data)
if not misc.manager_is_running():
misc.shutdown()
# signal.signal(signal.SIGTERM, dl_stop)
misc.params['isFirst'] = True
curl_handle = pycurl.Curl()
curl_handle.setopt(curl_handle.URL, url)
curl_handle.setopt(pycurl.WRITEFUNCTION, cback)
curl_handle.setopt(pycurl.FOLLOWLOCATION, True)
nl['curl_handle'] = curl_handle
try:
curl_handle.perform()
except TypeError as exc:
logging.info('Properly shutting down.')
except Exception as exc:
logging.warning("Couldn't resolve or connect to %s." % url)
curl_handle.close()
if nl['stream'] and type(nl['stream']) != bool:
nl['stream'].close()
# This is where we are sure of the stats on this file, because
# we just closed it ... so we can register it here.
info = audio.stream_info(file_name)
DB.register_stream(info)
#!/usr/bin/python3
import sys
import lib.audio as audio
for file_name in sys.argv[1:]:
print("%s:" % file_name)
print(audio.stream_info(file_name))
sig, block = audio.signature(file_name)
print({'block_count': len(block), 'format': audio._LASTFORMAT})