Пример #1
0
def fix_plus_n(epg, channels):
    import re
    from copy import copy
    from datetime import timedelta

    epg_chns = epg.get_channels()
    exp = re.compile('(\+\d+)$')
    fix = []

    # Find potential candidates for a fix
    for c in channels:
        if c not in epg_chns:
            r = exp.search(c.uri)
            if r:
                os = int(r.group(1))
                uri = c.uri.replace(r.group(1), '')
                for c1 in channels:
                    if uri == c1.uri:
                        fix.append((c, os * 60, c1))
                        break

    # Fix the channels
    for (plus, offset, base) in fix:
        sched = epg.get_schedule(base)
        if not sched: continue
        log.info('pyepg - fix missing plusN channel %s' % plus.title)
        for e in sched:
            n = copy(e)
            n.channel = plus
            n.start = n.start + timedelta(minutes=offset)
            n.stop = n.stop + timedelta(minutes=offset)
            epg.add_broadcast(n)
Пример #2
0
def fix_plus_n ( epg, channels ):
  import re
  from copy     import copy
  from datetime import timedelta

  epg_chns = epg.get_channels()
  exp      = re.compile('(\+\d+)$')
  fix      = []

  # Find potential candidates for a fix
  for c in channels:
    if c not in epg_chns:
      r = exp.search(c.uri)
      if r:
        os  = int(r.group(1))
        uri = c.uri.replace(r.group(1), '')
        for c1 in channels:
          if uri == c1.uri:
            fix.append((c, os*60, c1))
            break

  # Fix the channels
  for (plus, offset, base) in fix:
    sched = epg.get_schedule(base)
    if not sched: continue
    log.info('pyepg - fix missing plusN channel %s' % plus.title)
    for e in sched:
      n         = copy(e)
      n.channel = plus
      n.start   = n.start + timedelta(minutes=offset)
      n.stop    = n.stop  + timedelta(minutes=offset)
      epg.add_broadcast(n)
Пример #3
0
def _channels():

    # Fetch remote data
    log.info('fetch free to air channel info')
    chn_data = cache.get_data('uk_satellite_channels.csv', ttl=86400 * 7)
    reg_data = cache.get_data('uk_satellite_regions.csv', ttl=86400 * 7)

    # Channels list
    log.info('processing channel list')
    regional = []
    chns = []
    for l in chn_data.splitlines()[1:]:
        p = l.strip().split(',')
        if len(p) < 9: continue
        try:
            c = Channel()
            c.extra['stream'] = [(int(p[0]), p[1])]
            c.uri = p[2]
            c.title = p[3]
            c.extra['freesat_number'] = int(p[4])
            c.number = c.extra['sky_number'] = int(p[5])
            c.hd = p[6] == '1'
            c.radio = p[8] == '1'
            if (p[10]):
                c.image = p[10]
            else:
                c.image = p[9]

            # Skip
            if not c.uri: continue

            # Already included
            if c in chns:
                for t in chns:
                    if t == c:
                        t.extra['stream'].extend(c.extra['stream'])
                        break
                continue

            # Regional channel
            if p[7] == '1':
                regional.append(c)

            # Store
            elif c.extra['stream'][0][0]:
                chns.append(c)

        except Exception, e:
            log.error('failed to process [%s] [e=%s]' % (l, str(e)))
Пример #4
0
def _channels ():

  # Fetch remote data
  log.info('fetch free to air channel info')
  chn_data = cache.get_data('uk_satellite_channels.csv', ttl=86400*7)
  reg_data = cache.get_data('uk_satellite_regions.csv', ttl=86400*7)

  # Channels list
  log.info('processing channel list')
  regional = []
  chns     = []
  for l in chn_data.splitlines()[1:]:
    p = l.strip().split(',')
    if len(p) < 9: continue
    try:
      c = Channel()
      c.extra['stream']                = [ (int(p[0]), p[1]) ] 
      c.uri                            = p[2]
      c.title                          = p[3]
      c.extra['freesat_number']        = int(p[4])
      c.number = c.extra['sky_number'] = int(p[5])
      c.hd                             = p[6] == '1'
      c.radio                          = p[8] == '1'
      if (p[10]):
        c.image                        = p[10]
      else:
        c.image                        = p[9]

      # Skip
      if not c.uri: continue

      # Already included
      if c in chns:
        for t in chns:
          if t == c:
            t.extra['stream'].extend(c.extra['stream'])
            break
        continue

      # Regional channel
      if p[7] == '1': 
        regional.append(c)

      # Store
      elif c.extra['stream'][0][0]:
        chns.append(c)

    except Exception, e:
      log.error('failed to process [%s] [e=%s]' % (l, str(e)))
Пример #5
0
def load_channels ():
  ret = set()
  log.info('get atlas channel list')

  # Process
  data = cache.get_data('atlas_channels.csv')
  if data:
    for l in data.splitlines():
      p = map(lambda x: x.strip(), l.split(','))
      if len(p) < 3: continue
      c = Channel()
      c.uri       = p[0]
      c.shortid   = p[1]
      c.publisher = [ p[3] ]
      ret.add(c)
  return ret
Пример #6
0
def load_channels():
    ret = set()
    log.info('get atlas channel list')

    # Process
    data = cache.get_data('atlas_channels.csv')
    if data:
        for l in data.splitlines():
            p = map(lambda x: x.strip(), l.split(','))
            if len(p) < 3: continue
            c = Channel()
            c.uri = p[0]
            c.shortid = p[1]
            c.publisher = [p[3]]
            ret.add(c)
    return ret
Пример #7
0
def grab ( epg, channels, start, stop ):
  import multiprocessing as mp

  # Filter the channel list (only include those we have listing for)
  channels = filter_channels(channels)
  days     = util.total_seconds(stop - start) / 86400
  channels = sorted(channels, cmp=lambda a,b: cmp(a.number,b.number))
  log.info('atlas - epg grab %d channels for %d days' % (len(channels), days))

  # Config
  grab_thread_cnt = conf.get('atlas_grab_threads', 32)
  data_thread_cnt = conf.get('atlas_data_threads', 0)
  if grab_thread_cnt <= 0:
    grab_thread_cnt = len(channels)
  if data_thread_cnt <= 0:
    data_thread_cnt = mp.cpu_count() * 2
  data_thread_cnt = min(data_thread_cnt, len(channels))
  grab_thread_cnt = min(grab_thread_cnt, len(channels))

  # Create input/output queues
  inq  = ChannelQueue(channels)
  outq = DataQueue(len(channels))

  # Create grab threads
  grab_threads = []
  for i in range(grab_thread_cnt):
    t = GrabThread(i, inq, outq, start, stop)
    grab_threads.append(t)

  # Create data threads
  data_threads = []
  for i in range(data_thread_cnt):
    t = DataThread(i, outq, epg)
    data_threads.append(t)

  # Start threads
  for t in grab_threads: t.start()
  for t in data_threads: t.start()

  # Wait for completion (inq first)
  ins = outs = len(channels)
  while True:
    s = inq.remain()
    if s != ins:
      ins = s
      log.info('atlas - grab %3d/%3d channels remain' % (s, len(channels)))
    s = outq.remain()
    if s != outs:
      outs = s
      log.info('atlas - proc %3d/%3d channels remain' % (s, len(channels)))
    if not ins and not outs: break
  
    # Safety checks
    i = 0
    for t in grab_threads:
      if t.isAlive(): i = i + 1
    if not i and ins:
      log.error('atlas - grab threads have died prematurely')
      break
    i = 0
    for t in data_threads:
      if t.isAlive(): i = i + 1
    if not i and outs:
      log.error('atlas - proc threads have died prematurely')
      break
    time.sleep(1.0)
Пример #8
0
def grab(epg, channels, start, stop):
    import multiprocessing as mp

    # Filter the channel list (only include those we have listing for)
    channels = filter_channels(channels)
    days = util.total_seconds(stop - start) / 86400
    channels = sorted(channels, cmp=lambda a, b: cmp(a.number, b.number))
    log.info('atlas - epg grab %d channels for %d days' %
             (len(channels), days))

    # Config
    grab_thread_cnt = conf.get('atlas_grab_threads', 32)
    data_thread_cnt = conf.get('atlas_data_threads', 0)
    if grab_thread_cnt <= 0:
        grab_thread_cnt = len(channels)
    if data_thread_cnt <= 0:
        data_thread_cnt = mp.cpu_count() * 2
    data_thread_cnt = min(data_thread_cnt, len(channels))
    grab_thread_cnt = min(grab_thread_cnt, len(channels))

    # Create input/output queues
    inq = ChannelQueue(channels)
    outq = DataQueue(len(channels))

    # Create grab threads
    grab_threads = []
    for i in range(grab_thread_cnt):
        t = GrabThread(i, inq, outq, start, stop)
        grab_threads.append(t)

    # Create data threads
    data_threads = []
    for i in range(data_thread_cnt):
        t = DataThread(i, outq, epg)
        data_threads.append(t)

    # Start threads
    for t in grab_threads:
        t.start()
    for t in data_threads:
        t.start()

    # Wait for completion (inq first)
    ins = outs = len(channels)
    while True:
        s = inq.remain()
        if s != ins:
            ins = s
            log.info('atlas - grab %3d/%3d channels remain' %
                     (s, len(channels)))
        s = outq.remain()
        if s != outs:
            outs = s
            log.info('atlas - proc %3d/%3d channels remain' %
                     (s, len(channels)))
        if not ins and not outs: break

        # Safety checks
        i = 0
        for t in grab_threads:
            if t.isAlive(): i = i + 1
        if not i and ins:
            log.error('atlas - grab threads have died prematurely')
            break
        i = 0
        for t in data_threads:
            if t.isAlive(): i = i + 1
        if not i and outs:
            log.error('atlas - proc threads have died prematurely')
            break
        time.sleep(1.0)
Пример #9
0
def grab(opts, args):

    # Initialise EPG
    epg = EPG()

    # Get config
    days = conf.get('days', 7)
    today = datetime.datetime.today()

    # Get grabber/formatter
    grabber = get_grabber()
    formatter = get_formatter()

    # Channels
    channels = get_channels()

    # Get EPG
    log.info('grabbing EPG for %d days' % days)
    grabber.grab(epg, channels, today, today + datetime.timedelta(days=days))

    # Attempt to deal with missing +N channels
    fix_plus_n(epg, channels)

    # Finish the EPG (will tidy it up)
    epg.finish()

    # Output
    formatter.format(epg, sys.stdout)

    # Stats
    log.info('')
    log.info('Statistics:')
    log.info('--------------------------------------')
    log.info('Channel  Count: %d' % len(epg.get_channels()))
    log.info('Brand    Count: %d' % len(epg.get_brands()))
    log.info('Series   Count: %d' % len(epg.get_series()))
    log.info('Episode  Count: %d' % len(epg.get_episodes()))
    log.info('Schedule Count: %d' % epg.get_sched_count())
Пример #10
0
def grab ( opts, args ):

  # Initialise EPG
  epg = EPG()

  # Get config
  days     = conf.get('days', 7)
  today    = datetime.datetime.today()

  # Get grabber/formatter
  grabber   = get_grabber()
  formatter = get_formatter()

  # Channels
  channels  = get_channels()

  # Get EPG
  log.info('grabbing EPG for %d days' % days)
  grabber.grab(epg, channels, today, today + datetime.timedelta(days=days))

  # Attempt to deal with missing +N channels
  fix_plus_n(epg, channels)

  # Finish the EPG (will tidy it up)
  epg.finish()

  # Output
  formatter.format(epg, sys.stdout)

  # Stats
  log.info('')
  log.info('Statistics:')
  log.info('--------------------------------------')
  log.info('Channel  Count: %d' % len(epg.get_channels()))
  log.info('Brand    Count: %d' % len(epg.get_brands()))
  log.info('Series   Count: %d' % len(epg.get_series()))
  log.info('Episode  Count: %d' % len(epg.get_episodes()))
  log.info('Schedule Count: %d' % epg.get_sched_count())