def register_stream_list(reindex=False):
import lib.misc as misc
# Find the local streams and make sure they are all registered in the sqlite3 database.
#
# Get the existing streams as a set
#
# If we are asked to re-index (due to trying to fix a bug) then we ignore what we have
# and just go ahead and do everything.
#
if reindex:
all_registered = set([])
else:
all_registered = set(DB.all('streams', ['name']))
# There should be a smarter way to do this ... you'd think. We should also
# be more faithfully giving things extensions since it's not 100% mp3
all_files = set(glob('%s/*.mp3' % misc.DIR_STREAMS))
diff = all_files.difference(all_registered)
# This is a list of files we haven't scanned yet...
if not diff: return True
# This basically means we could still be writing
# this file.
#
# We take the cascade time and then buffer it by a minute, just
# to be sure.
#
# If the creation time is less then this then we don't register this
# until later.
cutoff = time.mktime((
datetime.now() -
timedelta(minutes=1, seconds=misc.config['cascade_time'])).timetuple())
for fname in diff:
if len(fname) == 0 or os.path.getctime(fname) > cutoff:
next
info = stream_info(fname)
if not info:
continue
DB.register_stream(info)
if not misc.manager_is_running():
logging.info("Manager is gone, shutting down")
raise Exception()
def register_stream_list(reindex=False):
import lib.misc as misc
# Find the local streams and make sure they are all registered in the sqlite3 database.
#
# Get the existing streams as a set
#
# If we are asked to re-index (due to trying to fix a bug) then we ignore what we have
# and just go ahead and do everything.
#
if reindex:
all_registered = set([])
else:
all_registered = set(DB.all('streams', ['name']))
# There should be a smarter way to do this ... you'd think. We should also
# be more faithfully giving things extensions since it's not 100% mp3
all_files = set(glob('%s/*.mp3' % misc.DIR_STREAMS))
diff = all_files.difference(all_registered)
# This is a list of files we haven't scanned yet...
if not diff: return True
# This basically means we could still be writing
# this file.
#
# We take the cascade time and then buffer it by a minute, just
# to be sure.
#
# If the creation time is less then this then we don't register this
# until later.
cutoff = time.mktime((datetime.now() - timedelta(minutes=1, seconds=misc.config['cascade_time'])).timetuple())
for fname in diff:
if len(fname) == 0 or os.path.getctime(fname) > cutoff:
next
info = stream_info(fname)
if not info:
continue
DB.register_stream(info)
if not misc.manager_is_running():
logging.info("Manager is gone, shutting down")
raise Exception()
def stream_download(callsign, url, my_pid, file_name):
# Curl interfacing which downloads the stream to disk.
# Follows redirects and parses out basic m3u.
#pid = misc.change_proc_name("%s-download" % callsign)
nl = {'stream': None, 'curl_handle': None, 'pid': my_pid, 'ix': 0, 'ct': 0, 'start': 0}
def progress(download_t, download_d, upload_t, upload_d):
if nl['start'] == 0:
nl['start'] = time.time()
nl['ct'] += 1
if nl['pid'] <= g_download_kill_pid - 2:
logging.info("Stopping download #%d through progress" % nl['pid'])
raise TypeError("Download Stop")
return False
#logging.info("progress: %f %d %d" % ((time.time() - nl['start']) / nl['ct'], nl['pid'], download_d ));
def catchall(which, what):
logging.info("%d: %s %s" % (nl['pid'], which, str(what)))
def catch_read(what):
return catchall('read', what)
def catch_debug(what, origin):
if what == pycurl.INFOTYPE_TEXT:
return catchall('debug', json.dumps([what, str(origin)], ensure_ascii=False))
def cback(data):
global g_download_kill_pid
"""
if len(data):
catchall('download', json.dumps([g_download_kill_pid, nl['pid'], len(data)]))
else:
catchall('download', json.dumps([g_download_kill_pid, 'no data']))
"""
# print nl['pid'], g_download_kill_pid
if nl['pid'] <= g_download_kill_pid or not data:
logging.info("Stopping download #%d" % nl['pid'])
return False
# misc.params can fail based on a shutdown sequence.
if misc is None or misc.params is None or not misc.manager_is_running():
# if misc is not None:
# misc.shutdown()
return False
elif not misc.params['shutdown_time']:
if not misc.download_ipc.empty():
what, value = misc.download_ipc.get(False)
if what == 'shutdown_time':
misc.params['shutdown_time'] = value
elif TS.unixtime('dl') > misc.params['shutdown_time']:
raise TypeError("Download Stop")
if misc.params['isFirst'] == True:
misc.params['isFirst'] = False
if len(data) < 800:
try:
data_string = data.decode('utf-8')
if re.match('https?://', data_string):
# If we are getting a redirect then we don't mind, we
# just put it in the stream and then we leave
misc.queue.put(('stream', data_string.strip()))
return False
# A pls style playlist
elif re.findall('File\d', data_string, re.M):
logging.info('%d: Found a pls, using the File1 parameter' % (nl['pid'], ))
matches = re.findall('File1=(.*)\n', data_string, re.M)
misc.queue.put(('stream', matches[0].strip()))
return False
# If it gets here it's binary ... I guess that's fine.
except:
pass
# This provides a reliable way to determine bitrate. We look at how much
# data we've received between two time periods
misc.queue.put(('heartbeat', (TS.unixtime('hb'), nl['pid'], len(data))))
if not nl['stream']:
try:
nl['stream'] = open(file_name, 'wb')
except Exception as exc:
logging.critical("%d: Unable to open %s. Can't record. Must exit." % (nl['pid'], file_name))
return False
nl['stream'].write(data)
misc.params['isFirst'] = True
curl_handle = pycurl.Curl()
curl_handle.setopt(curl_handle.URL, url)
curl_handle.setopt(pycurl.WRITEFUNCTION, cback)
curl_handle.setopt(pycurl.FOLLOWLOCATION, True)
curl_handle.setopt(pycurl.NOPROGRESS, False)
try:
curl_handle.setopt(pycurl.XFERINFOFUNCTION, progress)
except (NameError, AttributeError):
curl_handle.setopt(pycurl.PROGRESSFUNCTION, progress)
curl_handle.setopt(pycurl.VERBOSE, 1)
#curl_handle.setopt(pycurl.READFUNCTION, catch_read)
curl_handle.setopt(pycurl.DEBUGFUNCTION, catch_debug)
curl_handle.setopt(pycurl.USERAGENT, "indycast %s. See http://indycast.net/ for more info." % misc.__version__)
nl['curl_handle'] = curl_handle
try:
curl_handle.perform()
except pycurl.error as exc:
if exc.args[0] != 23:
logging.warning("%d: Couldn't resolve or connect to %s." % (nl['pid'], url))
except:
logging.warning("%d: Unhandled exception" % (nl['pid'], ))
pass
curl_handle.close()
if nl['stream'] and type(nl['stream']) != bool:
nl['stream'].close()
# This is where we are sure of the stats on this file, because
# we just closed it ... so we can register it here.
info = audio.stream_info(file_name)
DB.register_stream(info)
def stream_download(callsign, url, my_pid, file_name):
# Curl interfacing which downloads the stream to disk.
# Follows redirects and parses out basic m3u.
pid = misc.change_proc_name("%s-download" % callsign)
nl = {'stream': None, 'curl_handle': None}
def dl_stop(signal, frame):
sys.exit(0)
def cback(data):
if not misc.params['shutdown_time']:
if not misc.download_ipc.empty():
what, value = misc.download_ipc.get(False)
if what == 'shutdown_time':
misc.params['shutdown_time'] = value
elif TS.unixtime('dl') > misc.params['shutdown_time']:
sys.exit(0)
if misc.params['isFirst'] == True:
misc.params['isFirst'] = False
if len(data) < 800:
if re.match('https?://', data):
# If we are getting a redirect then we don't mind, we
# just put it in the stream and then we leave
misc.queue.put(('stream', data.strip()))
return True
# A pls style playlist
elif re.findall('File\d', data, re.M):
logging.info('Found a pls, using the File1 parameter')
matches = re.findall('File1=(.*)\n', data, re.M)
misc.queue.put(('stream', matches[0].strip()))
return True
# This provides a reliable way to determine bitrate. We look at how much
# data we've received between two time periods
misc.queue.put(('heartbeat', (TS.unixtime('hb'), len(data))))
if not nl['stream']:
try:
nl['stream'] = open(file_name, 'w')
except Exception as exc:
logging.critical(
"Unable to open %s. Can't record. Must exit." % file_name)
sys.exit(-1)
nl['stream'].write(data)
if not misc.manager_is_running():
misc.shutdown()
# signal.signal(signal.SIGTERM, dl_stop)
misc.params['isFirst'] = True
curl_handle = pycurl.Curl()
curl_handle.setopt(curl_handle.URL, url)
curl_handle.setopt(pycurl.WRITEFUNCTION, cback)
curl_handle.setopt(pycurl.FOLLOWLOCATION, True)
nl['curl_handle'] = curl_handle
try:
curl_handle.perform()
except TypeError as exc:
logging.info('Properly shutting down.')
except Exception as exc:
logging.warning("Couldn't resolve or connect to %s." % url)
curl_handle.close()
if nl['stream'] and type(nl['stream']) != bool:
nl['stream'].close()
# This is where we are sure of the stats on this file, because
# we just closed it ... so we can register it here.
info = audio.stream_info(file_name)
DB.register_stream(info)
def stream_download(callsign, url, my_pid, file_name):
# Curl interfacing which downloads the stream to disk.
# Follows redirects and parses out basic m3u.
#pid = misc.change_proc_name("%s-download" % callsign)
nl = {
'stream': None,
'curl_handle': None,
'pid': my_pid,
'ix': 0,
'ct': 0,
'start': 0
}
def progress(download_t, download_d, upload_t, upload_d):
if nl['start'] == 0:
nl['start'] = time.time()
nl['ct'] += 1
if nl['pid'] <= g_download_kill_pid - 2:
logging.info("Stopping download #%d through progress" % nl['pid'])
raise TypeError("Download Stop")
return False
#logging.info("progress: %f %d %d" % ((time.time() - nl['start']) / nl['ct'], nl['pid'], download_d ));
def catchall(which, what):
logging.info("%d: %s %s" % (nl['pid'], which, str(what)))
def catch_read(what):
return catchall('read', what)
def catch_debug(what, origin):
if what == pycurl.INFOTYPE_TEXT:
return catchall(
'debug', json.dumps([what, str(origin)], ensure_ascii=False))
def cback(data):
global g_download_kill_pid
"""
if len(data):
catchall('download', json.dumps([g_download_kill_pid, nl['pid'], len(data)]))
else:
catchall('download', json.dumps([g_download_kill_pid, 'no data']))
"""
# print nl['pid'], g_download_kill_pid
if nl['pid'] <= g_download_kill_pid or not data:
logging.info("Stopping download #%d" % nl['pid'])
return False
# misc.params can fail based on a shutdown sequence.
if misc is None or misc.params is None or not misc.manager_is_running(
):
# if misc is not None:
# misc.shutdown()
return False
elif not misc.params['shutdown_time']:
if not misc.download_ipc.empty():
what, value = misc.download_ipc.get(False)
if what == 'shutdown_time':
misc.params['shutdown_time'] = value
elif TS.unixtime('dl') > misc.params['shutdown_time']:
raise TypeError("Download Stop")
if misc.params['isFirst'] == True:
misc.params['isFirst'] = False
if len(data) < 800:
try:
data_string = data.decode('utf-8')
if re.match('https?://', data_string):
# If we are getting a redirect then we don't mind, we
# just put it in the stream and then we leave
misc.queue.put(('stream', data_string.strip()))
return False
# A pls style playlist
elif re.findall('File\d', data_string, re.M):
logging.info(
'%d: Found a pls, using the File1 parameter' %
(nl['pid'], ))
matches = re.findall('File1=(.*)\n', data_string, re.M)
misc.queue.put(('stream', matches[0].strip()))
return False
# If it gets here it's binary ... I guess that's fine.
except:
pass
# This provides a reliable way to determine bitrate. We look at how much
# data we've received between two time periods
misc.queue.put(
('heartbeat', (TS.unixtime('hb'), nl['pid'], len(data))))
if not nl['stream']:
try:
nl['stream'] = open(file_name, 'wb')
except Exception as exc:
logging.critical(
"%d: Unable to open %s. Can't record. Must exit." %
(nl['pid'], file_name))
return False
nl['stream'].write(data)
misc.params['isFirst'] = True
curl_handle = pycurl.Curl()
curl_handle.setopt(curl_handle.URL, url)
curl_handle.setopt(pycurl.WRITEFUNCTION, cback)
curl_handle.setopt(pycurl.FOLLOWLOCATION, True)
curl_handle.setopt(pycurl.NOPROGRESS, False)
try:
curl_handle.setopt(pycurl.XFERINFOFUNCTION, progress)
except (NameError, AttributeError):
curl_handle.setopt(pycurl.PROGRESSFUNCTION, progress)
curl_handle.setopt(pycurl.VERBOSE, 1)
#curl_handle.setopt(pycurl.READFUNCTION, catch_read)
curl_handle.setopt(pycurl.DEBUGFUNCTION, catch_debug)
curl_handle.setopt(
pycurl.USERAGENT,
"indycast %s. See http://indycast.net/ for more info." %
misc.__version__)
nl['curl_handle'] = curl_handle
try:
curl_handle.perform()
except pycurl.error as exc:
if exc.args[0] != 23:
logging.warning("%d: Couldn't resolve or connect to %s." %
(nl['pid'], url))
except:
logging.warning("%d: Unhandled exception" % (nl['pid'], ))
pass
curl_handle.close()
if nl['stream'] and type(nl['stream']) != bool:
nl['stream'].close()
# This is where we are sure of the stats on this file, because
# we just closed it ... so we can register it here.
info = audio.stream_info(file_name)
DB.register_stream(info)
def stream_download(callsign, url, my_pid, file_name):
# Curl interfacing which downloads the stream to disk.
# Follows redirects and parses out basic m3u.
pid = misc.change_proc_name("%s-download" % callsign)
nl = {'stream': None, 'curl_handle': None}
def dl_stop(signal, frame):
sys.exit(0)
def cback(data):
if not misc.params['shutdown_time']:
if not misc.download_ipc.empty():
what, value = misc.download_ipc.get(False)
if what == 'shutdown_time':
misc.params['shutdown_time'] = value
elif TS.unixtime('dl') > misc.params['shutdown_time']:
sys.exit(0)
if misc.params['isFirst'] == True:
misc.params['isFirst'] = False
if len(data) < 800:
if re.match('https?://', data):
# If we are getting a redirect then we don't mind, we
# just put it in the stream and then we leave
misc.queue.put(('stream', data.strip()))
return True
# A pls style playlist
elif re.findall('File\d', data, re.M):
logging.info('Found a pls, using the File1 parameter')
matches = re.findall('File1=(.*)\n', data, re.M)
misc.queue.put(('stream', matches[0].strip()))
return True
# This provides a reliable way to determine bitrate. We look at how much
# data we've received between two time periods
misc.queue.put(('heartbeat', (TS.unixtime('hb'), len(data))))
if not nl['stream']:
try:
nl['stream'] = open(file_name, 'w')
except Exception as exc:
logging.critical("Unable to open %s. Can't record. Must exit." % file_name)
sys.exit(-1)
nl['stream'].write(data)
if not misc.manager_is_running():
misc.shutdown()
# signal.signal(signal.SIGTERM, dl_stop)
misc.params['isFirst'] = True
curl_handle = pycurl.Curl()
curl_handle.setopt(curl_handle.URL, url)
curl_handle.setopt(pycurl.WRITEFUNCTION, cback)
curl_handle.setopt(pycurl.FOLLOWLOCATION, True)
nl['curl_handle'] = curl_handle
try:
curl_handle.perform()
except TypeError as exc:
logging.info('Properly shutting down.')
except Exception as exc:
logging.warning("Couldn't resolve or connect to %s." % url)
curl_handle.close()
if nl['stream'] and type(nl['stream']) != bool:
nl['stream'].close()
# This is where we are sure of the stats on this file, because
# we just closed it ... so we can register it here.
info = audio.stream_info(file_name)
DB.register_stream(info)
