def register_stream_list(reindex=False): import lib.misc as misc # Find the local streams and make sure they are all registered in the sqlite3 database. # # Get the existing streams as a set # # If we are asked to re-index (due to trying to fix a bug) then we ignore what we have # and just go ahead and do everything. # if reindex: all_registered = set([]) else: all_registered = set(DB.all('streams', ['name'])) # There should be a smarter way to do this ... you'd think. We should also # be more faithfully giving things extensions since it's not 100% mp3 all_files = set(glob('%s/*.mp3' % misc.DIR_STREAMS)) diff = all_files.difference(all_registered) # This is a list of files we haven't scanned yet... if not diff: return True # This basically means we could still be writing # this file. # # We take the cascade time and then buffer it by a minute, just # to be sure. # # If the creation time is less then this then we don't register this # until later. cutoff = time.mktime(( datetime.now() - timedelta(minutes=1, seconds=misc.config['cascade_time'])).timetuple()) for fname in diff: if len(fname) == 0 or os.path.getctime(fname) > cutoff: next info = stream_info(fname) if not info: continue DB.register_stream(info) if not misc.manager_is_running(): logging.info("Manager is gone, shutting down") raise Exception()
def register_stream_list(reindex=False): import lib.misc as misc # Find the local streams and make sure they are all registered in the sqlite3 database. # # Get the existing streams as a set # # If we are asked to re-index (due to trying to fix a bug) then we ignore what we have # and just go ahead and do everything. # if reindex: all_registered = set([]) else: all_registered = set(DB.all('streams', ['name'])) # There should be a smarter way to do this ... you'd think. We should also # be more faithfully giving things extensions since it's not 100% mp3 all_files = set(glob('%s/*.mp3' % misc.DIR_STREAMS)) diff = all_files.difference(all_registered) # This is a list of files we haven't scanned yet... if not diff: return True # This basically means we could still be writing # this file. # # We take the cascade time and then buffer it by a minute, just # to be sure. # # If the creation time is less then this then we don't register this # until later. cutoff = time.mktime((datetime.now() - timedelta(minutes=1, seconds=misc.config['cascade_time'])).timetuple()) for fname in diff: if len(fname) == 0 or os.path.getctime(fname) > cutoff: next info = stream_info(fname) if not info: continue DB.register_stream(info) if not misc.manager_is_running(): logging.info("Manager is gone, shutting down") raise Exception()
def stream_download(callsign, url, my_pid, file_name): # Curl interfacing which downloads the stream to disk. # Follows redirects and parses out basic m3u. #pid = misc.change_proc_name("%s-download" % callsign) nl = {'stream': None, 'curl_handle': None, 'pid': my_pid, 'ix': 0, 'ct': 0, 'start': 0} def progress(download_t, download_d, upload_t, upload_d): if nl['start'] == 0: nl['start'] = time.time() nl['ct'] += 1 if nl['pid'] <= g_download_kill_pid - 2: logging.info("Stopping download #%d through progress" % nl['pid']) raise TypeError("Download Stop") return False #logging.info("progress: %f %d %d" % ((time.time() - nl['start']) / nl['ct'], nl['pid'], download_d )); def catchall(which, what): logging.info("%d: %s %s" % (nl['pid'], which, str(what))) def catch_read(what): return catchall('read', what) def catch_debug(what, origin): if what == pycurl.INFOTYPE_TEXT: return catchall('debug', json.dumps([what, str(origin)], ensure_ascii=False)) def cback(data): global g_download_kill_pid """ if len(data): catchall('download', json.dumps([g_download_kill_pid, nl['pid'], len(data)])) else: catchall('download', json.dumps([g_download_kill_pid, 'no data'])) """ # print nl['pid'], g_download_kill_pid if nl['pid'] <= g_download_kill_pid or not data: logging.info("Stopping download #%d" % nl['pid']) return False # misc.params can fail based on a shutdown sequence. if misc is None or misc.params is None or not misc.manager_is_running(): # if misc is not None: # misc.shutdown() return False elif not misc.params['shutdown_time']: if not misc.download_ipc.empty(): what, value = misc.download_ipc.get(False) if what == 'shutdown_time': misc.params['shutdown_time'] = value elif TS.unixtime('dl') > misc.params['shutdown_time']: raise TypeError("Download Stop") if misc.params['isFirst'] == True: misc.params['isFirst'] = False if len(data) < 800: try: data_string = data.decode('utf-8') if re.match('https?://', data_string): # If we are getting a redirect then we don't mind, we # just put it in the stream and then we leave misc.queue.put(('stream', data_string.strip())) return False # A pls style playlist elif re.findall('File\d', data_string, re.M): logging.info('%d: Found a pls, using the File1 parameter' % (nl['pid'], )) matches = re.findall('File1=(.*)\n', data_string, re.M) misc.queue.put(('stream', matches[0].strip())) return False # If it gets here it's binary ... I guess that's fine. except: pass # This provides a reliable way to determine bitrate. We look at how much # data we've received between two time periods misc.queue.put(('heartbeat', (TS.unixtime('hb'), nl['pid'], len(data)))) if not nl['stream']: try: nl['stream'] = open(file_name, 'wb') except Exception as exc: logging.critical("%d: Unable to open %s. Can't record. Must exit." % (nl['pid'], file_name)) return False nl['stream'].write(data) misc.params['isFirst'] = True curl_handle = pycurl.Curl() curl_handle.setopt(curl_handle.URL, url) curl_handle.setopt(pycurl.WRITEFUNCTION, cback) curl_handle.setopt(pycurl.FOLLOWLOCATION, True) curl_handle.setopt(pycurl.NOPROGRESS, False) try: curl_handle.setopt(pycurl.XFERINFOFUNCTION, progress) except (NameError, AttributeError): curl_handle.setopt(pycurl.PROGRESSFUNCTION, progress) curl_handle.setopt(pycurl.VERBOSE, 1) #curl_handle.setopt(pycurl.READFUNCTION, catch_read) curl_handle.setopt(pycurl.DEBUGFUNCTION, catch_debug) curl_handle.setopt(pycurl.USERAGENT, "indycast %s. See http://indycast.net/ for more info." % misc.__version__) nl['curl_handle'] = curl_handle try: curl_handle.perform() except pycurl.error as exc: if exc.args[0] != 23: logging.warning("%d: Couldn't resolve or connect to %s." % (nl['pid'], url)) except: logging.warning("%d: Unhandled exception" % (nl['pid'], )) pass curl_handle.close() if nl['stream'] and type(nl['stream']) != bool: nl['stream'].close() # This is where we are sure of the stats on this file, because # we just closed it ... so we can register it here. info = audio.stream_info(file_name) DB.register_stream(info)
def stream_download(callsign, url, my_pid, file_name): # Curl interfacing which downloads the stream to disk. # Follows redirects and parses out basic m3u. pid = misc.change_proc_name("%s-download" % callsign) nl = {'stream': None, 'curl_handle': None} def dl_stop(signal, frame): sys.exit(0) def cback(data): if not misc.params['shutdown_time']: if not misc.download_ipc.empty(): what, value = misc.download_ipc.get(False) if what == 'shutdown_time': misc.params['shutdown_time'] = value elif TS.unixtime('dl') > misc.params['shutdown_time']: sys.exit(0) if misc.params['isFirst'] == True: misc.params['isFirst'] = False if len(data) < 800: if re.match('https?://', data): # If we are getting a redirect then we don't mind, we # just put it in the stream and then we leave misc.queue.put(('stream', data.strip())) return True # A pls style playlist elif re.findall('File\d', data, re.M): logging.info('Found a pls, using the File1 parameter') matches = re.findall('File1=(.*)\n', data, re.M) misc.queue.put(('stream', matches[0].strip())) return True # This provides a reliable way to determine bitrate. We look at how much # data we've received between two time periods misc.queue.put(('heartbeat', (TS.unixtime('hb'), len(data)))) if not nl['stream']: try: nl['stream'] = open(file_name, 'w') except Exception as exc: logging.critical( "Unable to open %s. Can't record. Must exit." % file_name) sys.exit(-1) nl['stream'].write(data) if not misc.manager_is_running(): misc.shutdown() # signal.signal(signal.SIGTERM, dl_stop) misc.params['isFirst'] = True curl_handle = pycurl.Curl() curl_handle.setopt(curl_handle.URL, url) curl_handle.setopt(pycurl.WRITEFUNCTION, cback) curl_handle.setopt(pycurl.FOLLOWLOCATION, True) nl['curl_handle'] = curl_handle try: curl_handle.perform() except TypeError as exc: logging.info('Properly shutting down.') except Exception as exc: logging.warning("Couldn't resolve or connect to %s." % url) curl_handle.close() if nl['stream'] and type(nl['stream']) != bool: nl['stream'].close() # This is where we are sure of the stats on this file, because # we just closed it ... so we can register it here. info = audio.stream_info(file_name) DB.register_stream(info)
def stream_download(callsign, url, my_pid, file_name): # Curl interfacing which downloads the stream to disk. # Follows redirects and parses out basic m3u. #pid = misc.change_proc_name("%s-download" % callsign) nl = { 'stream': None, 'curl_handle': None, 'pid': my_pid, 'ix': 0, 'ct': 0, 'start': 0 } def progress(download_t, download_d, upload_t, upload_d): if nl['start'] == 0: nl['start'] = time.time() nl['ct'] += 1 if nl['pid'] <= g_download_kill_pid - 2: logging.info("Stopping download #%d through progress" % nl['pid']) raise TypeError("Download Stop") return False #logging.info("progress: %f %d %d" % ((time.time() - nl['start']) / nl['ct'], nl['pid'], download_d )); def catchall(which, what): logging.info("%d: %s %s" % (nl['pid'], which, str(what))) def catch_read(what): return catchall('read', what) def catch_debug(what, origin): if what == pycurl.INFOTYPE_TEXT: return catchall( 'debug', json.dumps([what, str(origin)], ensure_ascii=False)) def cback(data): global g_download_kill_pid """ if len(data): catchall('download', json.dumps([g_download_kill_pid, nl['pid'], len(data)])) else: catchall('download', json.dumps([g_download_kill_pid, 'no data'])) """ # print nl['pid'], g_download_kill_pid if nl['pid'] <= g_download_kill_pid or not data: logging.info("Stopping download #%d" % nl['pid']) return False # misc.params can fail based on a shutdown sequence. if misc is None or misc.params is None or not misc.manager_is_running( ): # if misc is not None: # misc.shutdown() return False elif not misc.params['shutdown_time']: if not misc.download_ipc.empty(): what, value = misc.download_ipc.get(False) if what == 'shutdown_time': misc.params['shutdown_time'] = value elif TS.unixtime('dl') > misc.params['shutdown_time']: raise TypeError("Download Stop") if misc.params['isFirst'] == True: misc.params['isFirst'] = False if len(data) < 800: try: data_string = data.decode('utf-8') if re.match('https?://', data_string): # If we are getting a redirect then we don't mind, we # just put it in the stream and then we leave misc.queue.put(('stream', data_string.strip())) return False # A pls style playlist elif re.findall('File\d', data_string, re.M): logging.info( '%d: Found a pls, using the File1 parameter' % (nl['pid'], )) matches = re.findall('File1=(.*)\n', data_string, re.M) misc.queue.put(('stream', matches[0].strip())) return False # If it gets here it's binary ... I guess that's fine. except: pass # This provides a reliable way to determine bitrate. We look at how much # data we've received between two time periods misc.queue.put( ('heartbeat', (TS.unixtime('hb'), nl['pid'], len(data)))) if not nl['stream']: try: nl['stream'] = open(file_name, 'wb') except Exception as exc: logging.critical( "%d: Unable to open %s. Can't record. Must exit." % (nl['pid'], file_name)) return False nl['stream'].write(data) misc.params['isFirst'] = True curl_handle = pycurl.Curl() curl_handle.setopt(curl_handle.URL, url) curl_handle.setopt(pycurl.WRITEFUNCTION, cback) curl_handle.setopt(pycurl.FOLLOWLOCATION, True) curl_handle.setopt(pycurl.NOPROGRESS, False) try: curl_handle.setopt(pycurl.XFERINFOFUNCTION, progress) except (NameError, AttributeError): curl_handle.setopt(pycurl.PROGRESSFUNCTION, progress) curl_handle.setopt(pycurl.VERBOSE, 1) #curl_handle.setopt(pycurl.READFUNCTION, catch_read) curl_handle.setopt(pycurl.DEBUGFUNCTION, catch_debug) curl_handle.setopt( pycurl.USERAGENT, "indycast %s. See http://indycast.net/ for more info." % misc.__version__) nl['curl_handle'] = curl_handle try: curl_handle.perform() except pycurl.error as exc: if exc.args[0] != 23: logging.warning("%d: Couldn't resolve or connect to %s." % (nl['pid'], url)) except: logging.warning("%d: Unhandled exception" % (nl['pid'], )) pass curl_handle.close() if nl['stream'] and type(nl['stream']) != bool: nl['stream'].close() # This is where we are sure of the stats on this file, because # we just closed it ... so we can register it here. info = audio.stream_info(file_name) DB.register_stream(info)
def stream_download(callsign, url, my_pid, file_name): # Curl interfacing which downloads the stream to disk. # Follows redirects and parses out basic m3u. pid = misc.change_proc_name("%s-download" % callsign) nl = {'stream': None, 'curl_handle': None} def dl_stop(signal, frame): sys.exit(0) def cback(data): if not misc.params['shutdown_time']: if not misc.download_ipc.empty(): what, value = misc.download_ipc.get(False) if what == 'shutdown_time': misc.params['shutdown_time'] = value elif TS.unixtime('dl') > misc.params['shutdown_time']: sys.exit(0) if misc.params['isFirst'] == True: misc.params['isFirst'] = False if len(data) < 800: if re.match('https?://', data): # If we are getting a redirect then we don't mind, we # just put it in the stream and then we leave misc.queue.put(('stream', data.strip())) return True # A pls style playlist elif re.findall('File\d', data, re.M): logging.info('Found a pls, using the File1 parameter') matches = re.findall('File1=(.*)\n', data, re.M) misc.queue.put(('stream', matches[0].strip())) return True # This provides a reliable way to determine bitrate. We look at how much # data we've received between two time periods misc.queue.put(('heartbeat', (TS.unixtime('hb'), len(data)))) if not nl['stream']: try: nl['stream'] = open(file_name, 'w') except Exception as exc: logging.critical("Unable to open %s. Can't record. Must exit." % file_name) sys.exit(-1) nl['stream'].write(data) if not misc.manager_is_running(): misc.shutdown() # signal.signal(signal.SIGTERM, dl_stop) misc.params['isFirst'] = True curl_handle = pycurl.Curl() curl_handle.setopt(curl_handle.URL, url) curl_handle.setopt(pycurl.WRITEFUNCTION, cback) curl_handle.setopt(pycurl.FOLLOWLOCATION, True) nl['curl_handle'] = curl_handle try: curl_handle.perform() except TypeError as exc: logging.info('Properly shutting down.') except Exception as exc: logging.warning("Couldn't resolve or connect to %s." % url) curl_handle.close() if nl['stream'] and type(nl['stream']) != bool: nl['stream'].close() # This is where we are sure of the stats on this file, because # we just closed it ... so we can register it here. info = audio.stream_info(file_name) DB.register_stream(info)