def fix_plus_n(epg, channels):
import re
from copy import copy
from datetime import timedelta
epg_chns = epg.get_channels()
exp = re.compile('(\+\d+)$')
fix = []
# Find potential candidates for a fix
for c in channels:
if c not in epg_chns:
r = exp.search(c.uri)
if r:
os = int(r.group(1))
uri = c.uri.replace(r.group(1), '')
for c1 in channels:
if uri == c1.uri:
fix.append((c, os * 60, c1))
break
# Fix the channels
for (plus, offset, base) in fix:
sched = epg.get_schedule(base)
if not sched: continue
log.info('pyepg - fix missing plusN channel %s' % plus.title)
for e in sched:
n = copy(e)
n.channel = plus
n.start = n.start + timedelta(minutes=offset)
n.stop = n.stop + timedelta(minutes=offset)
epg.add_broadcast(n)
def fix_plus_n ( epg, channels ):
import re
from copy import copy
from datetime import timedelta
epg_chns = epg.get_channels()
exp = re.compile('(\+\d+)$')
fix = []
# Find potential candidates for a fix
for c in channels:
if c not in epg_chns:
r = exp.search(c.uri)
if r:
os = int(r.group(1))
uri = c.uri.replace(r.group(1), '')
for c1 in channels:
if uri == c1.uri:
fix.append((c, os*60, c1))
break
# Fix the channels
for (plus, offset, base) in fix:
sched = epg.get_schedule(base)
if not sched: continue
log.info('pyepg - fix missing plusN channel %s' % plus.title)
for e in sched:
n = copy(e)
n.channel = plus
n.start = n.start + timedelta(minutes=offset)
n.stop = n.stop + timedelta(minutes=offset)
epg.add_broadcast(n)
def _channels():
# Fetch remote data
log.info('fetch free to air channel info')
chn_data = cache.get_data('uk_satellite_channels.csv', ttl=86400 * 7)
reg_data = cache.get_data('uk_satellite_regions.csv', ttl=86400 * 7)
# Channels list
log.info('processing channel list')
regional = []
chns = []
for l in chn_data.splitlines()[1:]:
p = l.strip().split(',')
if len(p) < 9: continue
try:
c = Channel()
c.extra['stream'] = [(int(p[0]), p[1])]
c.uri = p[2]
c.title = p[3]
c.extra['freesat_number'] = int(p[4])
c.number = c.extra['sky_number'] = int(p[5])
c.hd = p[6] == '1'
c.radio = p[8] == '1'
if (p[10]):
c.image = p[10]
else:
c.image = p[9]
# Skip
if not c.uri: continue
# Already included
if c in chns:
for t in chns:
if t == c:
t.extra['stream'].extend(c.extra['stream'])
break
continue
# Regional channel
if p[7] == '1':
regional.append(c)
# Store
elif c.extra['stream'][0][0]:
chns.append(c)
except Exception, e:
log.error('failed to process [%s] [e=%s]' % (l, str(e)))
def _channels ():
# Fetch remote data
log.info('fetch free to air channel info')
chn_data = cache.get_data('uk_satellite_channels.csv', ttl=86400*7)
reg_data = cache.get_data('uk_satellite_regions.csv', ttl=86400*7)
# Channels list
log.info('processing channel list')
regional = []
chns = []
for l in chn_data.splitlines()[1:]:
p = l.strip().split(',')
if len(p) < 9: continue
try:
c = Channel()
c.extra['stream'] = [ (int(p[0]), p[1]) ]
c.uri = p[2]
c.title = p[3]
c.extra['freesat_number'] = int(p[4])
c.number = c.extra['sky_number'] = int(p[5])
c.hd = p[6] == '1'
c.radio = p[8] == '1'
if (p[10]):
c.image = p[10]
else:
c.image = p[9]
# Skip
if not c.uri: continue
# Already included
if c in chns:
for t in chns:
if t == c:
t.extra['stream'].extend(c.extra['stream'])
break
continue
# Regional channel
if p[7] == '1':
regional.append(c)
# Store
elif c.extra['stream'][0][0]:
chns.append(c)
except Exception, e:
log.error('failed to process [%s] [e=%s]' % (l, str(e)))
def load_channels ():
ret = set()
log.info('get atlas channel list')
# Process
data = cache.get_data('atlas_channels.csv')
if data:
for l in data.splitlines():
p = map(lambda x: x.strip(), l.split(','))
if len(p) < 3: continue
c = Channel()
c.uri = p[0]
c.shortid = p[1]
c.publisher = [ p[3] ]
ret.add(c)
return ret
def load_channels():
ret = set()
log.info('get atlas channel list')
# Process
data = cache.get_data('atlas_channels.csv')
if data:
for l in data.splitlines():
p = map(lambda x: x.strip(), l.split(','))
if len(p) < 3: continue
c = Channel()
c.uri = p[0]
c.shortid = p[1]
c.publisher = [p[3]]
ret.add(c)
return ret
def grab ( epg, channels, start, stop ):
import multiprocessing as mp
# Filter the channel list (only include those we have listing for)
channels = filter_channels(channels)
days = util.total_seconds(stop - start) / 86400
channels = sorted(channels, cmp=lambda a,b: cmp(a.number,b.number))
log.info('atlas - epg grab %d channels for %d days' % (len(channels), days))
# Config
grab_thread_cnt = conf.get('atlas_grab_threads', 32)
data_thread_cnt = conf.get('atlas_data_threads', 0)
if grab_thread_cnt <= 0:
grab_thread_cnt = len(channels)
if data_thread_cnt <= 0:
data_thread_cnt = mp.cpu_count() * 2
data_thread_cnt = min(data_thread_cnt, len(channels))
grab_thread_cnt = min(grab_thread_cnt, len(channels))
# Create input/output queues
inq = ChannelQueue(channels)
outq = DataQueue(len(channels))
# Create grab threads
grab_threads = []
for i in range(grab_thread_cnt):
t = GrabThread(i, inq, outq, start, stop)
grab_threads.append(t)
# Create data threads
data_threads = []
for i in range(data_thread_cnt):
t = DataThread(i, outq, epg)
data_threads.append(t)
# Start threads
for t in grab_threads: t.start()
for t in data_threads: t.start()
# Wait for completion (inq first)
ins = outs = len(channels)
while True:
s = inq.remain()
if s != ins:
ins = s
log.info('atlas - grab %3d/%3d channels remain' % (s, len(channels)))
s = outq.remain()
if s != outs:
outs = s
log.info('atlas - proc %3d/%3d channels remain' % (s, len(channels)))
if not ins and not outs: break
# Safety checks
i = 0
for t in grab_threads:
if t.isAlive(): i = i + 1
if not i and ins:
log.error('atlas - grab threads have died prematurely')
break
i = 0
for t in data_threads:
if t.isAlive(): i = i + 1
if not i and outs:
log.error('atlas - proc threads have died prematurely')
break
time.sleep(1.0)
def grab(epg, channels, start, stop):
import multiprocessing as mp
# Filter the channel list (only include those we have listing for)
channels = filter_channels(channels)
days = util.total_seconds(stop - start) / 86400
channels = sorted(channels, cmp=lambda a, b: cmp(a.number, b.number))
log.info('atlas - epg grab %d channels for %d days' %
(len(channels), days))
# Config
grab_thread_cnt = conf.get('atlas_grab_threads', 32)
data_thread_cnt = conf.get('atlas_data_threads', 0)
if grab_thread_cnt <= 0:
grab_thread_cnt = len(channels)
if data_thread_cnt <= 0:
data_thread_cnt = mp.cpu_count() * 2
data_thread_cnt = min(data_thread_cnt, len(channels))
grab_thread_cnt = min(grab_thread_cnt, len(channels))
# Create input/output queues
inq = ChannelQueue(channels)
outq = DataQueue(len(channels))
# Create grab threads
grab_threads = []
for i in range(grab_thread_cnt):
t = GrabThread(i, inq, outq, start, stop)
grab_threads.append(t)
# Create data threads
data_threads = []
for i in range(data_thread_cnt):
t = DataThread(i, outq, epg)
data_threads.append(t)
# Start threads
for t in grab_threads:
t.start()
for t in data_threads:
t.start()
# Wait for completion (inq first)
ins = outs = len(channels)
while True:
s = inq.remain()
if s != ins:
ins = s
log.info('atlas - grab %3d/%3d channels remain' %
(s, len(channels)))
s = outq.remain()
if s != outs:
outs = s
log.info('atlas - proc %3d/%3d channels remain' %
(s, len(channels)))
if not ins and not outs: break
# Safety checks
i = 0
for t in grab_threads:
if t.isAlive(): i = i + 1
if not i and ins:
log.error('atlas - grab threads have died prematurely')
break
i = 0
for t in data_threads:
if t.isAlive(): i = i + 1
if not i and outs:
log.error('atlas - proc threads have died prematurely')
break
time.sleep(1.0)
def grab(opts, args):
# Initialise EPG
epg = EPG()
# Get config
days = conf.get('days', 7)
today = datetime.datetime.today()
# Get grabber/formatter
grabber = get_grabber()
formatter = get_formatter()
# Channels
channels = get_channels()
# Get EPG
log.info('grabbing EPG for %d days' % days)
grabber.grab(epg, channels, today, today + datetime.timedelta(days=days))
# Attempt to deal with missing +N channels
fix_plus_n(epg, channels)
# Finish the EPG (will tidy it up)
epg.finish()
# Output
formatter.format(epg, sys.stdout)
# Stats
log.info('')
log.info('Statistics:')
log.info('--------------------------------------')
log.info('Channel Count: %d' % len(epg.get_channels()))
log.info('Brand Count: %d' % len(epg.get_brands()))
log.info('Series Count: %d' % len(epg.get_series()))
log.info('Episode Count: %d' % len(epg.get_episodes()))
log.info('Schedule Count: %d' % epg.get_sched_count())
def grab ( opts, args ):
# Initialise EPG
epg = EPG()
# Get config
days = conf.get('days', 7)
today = datetime.datetime.today()
# Get grabber/formatter
grabber = get_grabber()
formatter = get_formatter()
# Channels
channels = get_channels()
# Get EPG
log.info('grabbing EPG for %d days' % days)
grabber.grab(epg, channels, today, today + datetime.timedelta(days=days))
# Attempt to deal with missing +N channels
fix_plus_n(epg, channels)
# Finish the EPG (will tidy it up)
epg.finish()
# Output
formatter.format(epg, sys.stdout)
# Stats
log.info('')
log.info('Statistics:')
log.info('--------------------------------------')
log.info('Channel Count: %d' % len(epg.get_channels()))
log.info('Brand Count: %d' % len(epg.get_brands()))
log.info('Series Count: %d' % len(epg.get_series()))
log.info('Episode Count: %d' % len(epg.get_episodes()))
log.info('Schedule Count: %d' % epg.get_sched_count())
