def _cmp ( a, b ): t = a['broadcasts'][0]['transmission_time']\ - b['broadcasts'][0]['transmission_time'] t = util.total_seconds(t) if t < 0: return -1 if t > 0: return 1 return 0
def _cmp(a, b): t = a['broadcasts'][0]['transmission_time']\ - b['broadcasts'][0]['transmission_time'] t = util.total_seconds(t) if t < 0: return -1 if t > 0: return 1 return 0
def grab ( epg, channels, start, stop ): import multiprocessing as mp # Filter the channel list (only include those we have listing for) channels = filter_channels(channels) days = util.total_seconds(stop - start) / 86400 channels = sorted(channels, cmp=lambda a,b: cmp(a.number,b.number)) log.info('atlas - epg grab %d channels for %d days' % (len(channels), days)) # Config grab_thread_cnt = conf.get('atlas_grab_threads', 32) data_thread_cnt = conf.get('atlas_data_threads', 0) if grab_thread_cnt <= 0: grab_thread_cnt = len(channels) if data_thread_cnt <= 0: data_thread_cnt = mp.cpu_count() * 2 data_thread_cnt = min(data_thread_cnt, len(channels)) grab_thread_cnt = min(grab_thread_cnt, len(channels)) # Create input/output queues inq = ChannelQueue(channels) outq = DataQueue(len(channels)) # Create grab threads grab_threads = [] for i in range(grab_thread_cnt): t = GrabThread(i, inq, outq, start, stop) grab_threads.append(t) # Create data threads data_threads = [] for i in range(data_thread_cnt): t = DataThread(i, outq, epg) data_threads.append(t) # Start threads for t in grab_threads: t.start() for t in data_threads: t.start() # Wait for completion (inq first) ins = outs = len(channels) while True: s = inq.remain() if s != ins: ins = s log.info('atlas - grab %3d/%3d channels remain' % (s, len(channels))) s = outq.remain() if s != outs: outs = s log.info('atlas - proc %3d/%3d channels remain' % (s, len(channels))) if not ins and not outs: break # Safety checks i = 0 for t in grab_threads: if t.isAlive(): i = i + 1 if not i and ins: log.error('atlas - grab threads have died prematurely') break i = 0 for t in data_threads: if t.isAlive(): i = i + 1 if not i and outs: log.error('atlas - proc threads have died prematurely') break time.sleep(1.0)
def grab(epg, channels, start, stop): import multiprocessing as mp # Filter the channel list (only include those we have listing for) channels = filter_channels(channels) days = util.total_seconds(stop - start) / 86400 channels = sorted(channels, cmp=lambda a, b: cmp(a.number, b.number)) log.info('atlas - epg grab %d channels for %d days' % (len(channels), days)) # Config grab_thread_cnt = conf.get('atlas_grab_threads', 32) data_thread_cnt = conf.get('atlas_data_threads', 0) if grab_thread_cnt <= 0: grab_thread_cnt = len(channels) if data_thread_cnt <= 0: data_thread_cnt = mp.cpu_count() * 2 data_thread_cnt = min(data_thread_cnt, len(channels)) grab_thread_cnt = min(grab_thread_cnt, len(channels)) # Create input/output queues inq = ChannelQueue(channels) outq = DataQueue(len(channels)) # Create grab threads grab_threads = [] for i in range(grab_thread_cnt): t = GrabThread(i, inq, outq, start, stop) grab_threads.append(t) # Create data threads data_threads = [] for i in range(data_thread_cnt): t = DataThread(i, outq, epg) data_threads.append(t) # Start threads for t in grab_threads: t.start() for t in data_threads: t.start() # Wait for completion (inq first) ins = outs = len(channels) while True: s = inq.remain() if s != ins: ins = s log.info('atlas - grab %3d/%3d channels remain' % (s, len(channels))) s = outq.remain() if s != outs: outs = s log.info('atlas - proc %3d/%3d channels remain' % (s, len(channels))) if not ins and not outs: break # Safety checks i = 0 for t in grab_threads: if t.isAlive(): i = i + 1 if not i and ins: log.error('atlas - grab threads have died prematurely') break i = 0 for t in data_threads: if t.isAlive(): i = i + 1 if not i and outs: log.error('atlas - proc threads have died prematurely') break time.sleep(1.0)