def fetch(self, utc_date = None, utc_time = None, day = None, region = 'us', days = 1):
region = region
days = days
sql = "SELECT id, name, slug FROM epgsdotcom_channels WHERE region = '%s'" % region
self.cursor.execute(sql)
channels = self.cursor.fetchall() # a list
for channel in channels[:]: # a tuple
xmltv_url = self.epgs_url + 'checksum=' + self.checksum + '&channel=' + str(channel[0]) + '&days=' + str(days)
xml = urllib2.urlopen(xmltv_url) # a file-like handle
epgs = xmltv.read_programmes(xml)
self.store(channel = channel[1], data = epgs)
def parse_broadcasts():
broadcasts = []
for element in xmltv.read_programmes(open(filename, 'r')):
channel = retrieve_channel(element['channel'])
broadcasts.append({
'channel': channel['id'],
'title': retrieve_title(element),
'start_time': format_time(element['start']).strftime("%Y-%m-%d %H:%M"),
'end_time': format_time(element['stop']).strftime("%Y-%m-%d %H:%M"),
'blurb': retrieve_blurb(element)
})
return broadcasts
def __load_programs(self):
for program in xmltv.read_programmes(open(self.path)):
error_msg = ""
if not program.has_key("title"):
continue
elif not program.has_key("start"):
error_msg = "missing start time"
elif not program.has_key("stop"):
error_msg = "missing stop time"
if error_msg:
print >>sys.stderr, 'Unable to add program "%s" to Listings: %s' % (program["title"][0][0], error_msg)
continue
channel_name = self.__get_channel_name(program)
p = Program(program)
self.listings.add_program(p, channel_name)
if self.config.debug:
print('Added program "%s" to Listings.' % p.title)
def tv(self, irc, msg, args):
"""takes no arguments
Returns the list of current TV shows.
"""
global filenames
now = datetime.datetime.today()
show_list = ""
show_count = 0
for filename in filenames:
print filename
try:
chans = xmltv.read_channels(open(filename, 'r'))
for chan in chans:
try:
chan_name = chan['display-name'][0][0]
print chan_name
shows = xmltv.read_programmes(open(filename, 'r'))
for show in shows:
try:
start_time = parse(show['start'],ignoretz=True)
stop_time = parse(show['stop'],ignoretz=True)
if (start_time <= now and stop_time >= now):
show_count = show_count + 1
show_list = show_list + "\x0F" + "\x02" + chan_name + "\x0F" + " " + str(start_time.strftime("%H:%M")) + " " + (show['title'][0][0]).encode("UTF-8") + " "
# if not (show_count % 3):
# irc.reply(show_list)
# show_list = ""
except:
pass
except:
pass
except:
pass
irc.reply(show_list)
print show_list
def write_cats(root, cont, is_channel_cats):
global cat_id
for item in cont.keys():
if not item[0]:
continue
elem = SubElement(root, 'Category')
if item[1]:
elem.set('lang', item[1])
elem.set('id', str(cat_id))
cat_id += 1
if is_channel_cats:
for ch in cont[item]:
ch_elem = SubElement(elem, 'Channel')
ch_elem.set('id', ch)
elem.set('channel', 'true')
elem.set('name', item[0])
if len(sys.argv) != 2:
print 'Usage: %s epg.xml' % sys.argv[0]
sys.exit(-1)
channel_cats = find_cats(xmltv.read_channels(open(sys.argv[1])))
programmes_cats = find_cats(xmltv.read_programmes(open(sys.argv[1])))
root = Element('Categories')
et = ElementTree(root)
write_cats(root, channel_cats, True)
write_cats(root, programmes_cats, False)
et.write(sys.stdout, 'UTF-8')
def get_xmltv():
"""
Download XMLTV url and store channels and programs in the database.
:return:
None
:return:
"""
# http://wiki.xmltv.org/index.php/Main_Page/xmltvfileformat.html
import urllib2
import gzip
import StringIO
import xmltv
url = cfg.TVGURL
# Download XMLTV source
out_file_path = url.split("/")[-1][:-3]
print('Downloading TV program from: {}'.format(url))
response = urllib2.urlopen(url)
compressed_file = StringIO.StringIO(response.read())
decompressed_file = gzip.GzipFile(fileobj=compressed_file)
# Extract XMLTV
with open(out_file_path, 'w') as outfile:
outfile.write(decompressed_file.read())
# Print XMLTV header
xmltv_data = xmltv.read_data(open(out_file_path, 'r'))
ic(xmltv_data)
# Read xml channels
xmlchannels = xmltv.read_channels(open(out_file_path, 'r'))
print("Got {} channels from XMLTV source".format(len(xmlchannels)))
# Drop content of XMLChannel
XMLChannel.query.delete()
db.session.commit()
# Populate XMLChannel with channels from XMLTV source
for xc in xmlchannels:
xmlchannel = XMLChannel(id=int(xc['id']),
label=xc['display-name'][0][0].strip())
db.session.add(xmlchannel)
db.session.commit()
programs = xmltv.read_programmes(open(out_file_path, 'r'))
chunk = 1024
index = 0
for pr in programs:
desc = ""
try:
desc = pr['desc'][0][0]
except KeyError:
pass
a_category = Category.query.filter(
Category.name == pr['category'][0][0]).first()
if a_category:
p = Program(channel=int(pr['channel']),
title=pr['title'][0][0],
start=duparse(pr['start']),
stop=duparse(pr['stop']),
desc=desc,
category_id=a_category.id)
db.session.add(p)
else:
py = Category(name=pr['category'][0][0])
Program(channel=int(pr['channel']),
title=pr['title'][0][0],
start=duparse(pr['start']),
stop=duparse(pr['stop']),
desc=desc,
category=py)
db.session.add(py)
index += 1
if index % chunk == 0:
db.session.commit()
db.session.commit()
categories = [x.name for x in Category.query.all()]
ic(u', '.join(categories))
channel, created = Channel.objects.get_or_create(
type=Channel.TYPE_TV_CHANNEL,
fetcher__cid=id,
fetcher__name='xmltv',
defaults=defaults(),
)
if created:
Fetcher(name='xmltv', channel=channel, cid=id).save()
if not created and self.opt.get('force'):
if channel.icon:
channel.icon.delete()
for (n, v) in defaults():
setattr(channel, n, v)
channel.save()
return
for id, programs in groupby(xmltv.read_programmes(input_file),
lambda p: p['channel']):
try:
channel = Channel.objects.get(fetcher__name='xmltv',
fetcher__cid=id)
except Channel.DoesNotExist, e:
self.logger.warning(
"channel %s does not exist, try --update-channels", id)
continue
last_time = channel.last_screening_time
last_time = last_time and pytz.utc.localize(last_time)
max_time = None
for p in programs:
directors = p.get('credits', {}).get('director')
fetcher__cid=id,
fetcher__name='xmltv',
defaults=defaults(),
)
if created:
Fetcher(name='xmltv',
channel=channel,
cid=id).save()
if not created and self.opt.get('force'):
if channel.icon:
channel.icon.delete()
for (n, v) in defaults():
setattr(channel, n, v)
channel.save()
return
for id, programs in groupby(xmltv.read_programmes(input_file), lambda p:p['channel']):
try:
channel = Channel.objects.get(fetcher__name='xmltv', fetcher__cid=id)
except Channel.DoesNotExist, e:
self.logger.warning("channel %s does not exist, try --update-channels", id)
continue
last_time = channel.last_screening_time
last_time = last_time and pytz.utc.localize(last_time)
max_time = None
for p in programs:
directors = p.get('credits',{}).get('director')
title = p.get('title')
title = title and title[0][0]
subtitle = p.get('sub-title')
if(len(c['display-name']) > 4):
display_name = c['display-name'][4][0]
else:
display_name = c['display-name'][3][0]
display_name = display_name.replace(' Low Power', '')
display_name = display_name.replace(' Affiliate', '')
data[id] = {}
data[id]['id'] = id
data[id]['callsign'] = callsign
data[id]['network'] = display_name
# Print programmes
programs = xmltv.read_programmes(open(filename, 'r'))
next = 0
for p in programs:
t = p['channel']
if(t in data):
start = int(str(p['start']).replace(' -0600', ''))
end = int(str(p['stop']).replace(' -0600', ''))
if(next == 1):
data[t]['next'] = p['title'][0][0]
next = 0
callsign = data[t]['callsign']
data[callsign] = {}
data[callsign] = data[t]
def parse_raw_xmltv_files():
"""Parse all xmltv files to deserialize info in dict."""
print('\n# Parse all raw XMLTV files', flush=True)
all_channels = {} # Key: channel id in xmltv file, Value: channel dict
all_data = {} # Key: grabber, Value: data dict
all_programmes = {
} # Key: channel id, Value: List of dict programme with UTC time
all_programmes_local = {
} # Key: channel id, Value: List of dict programme with local time
for grabber, grabber_infos in GRABBERS.items():
print('\n\t* Processing of {} grabber\'s raw files:'.format(grabber),
flush=True)
for offset in range(-10, 20):
date = TODAY + timedelta(days=offset)
xmltv_fp = RAW_DIRECTORY + grabber_infos['raw'].format(
'_' + date.strftime("%Y%m%d"))
if not os.path.exists(xmltv_fp):
continue
print('\t\t* Parse {} file'.format(xmltv_fp), flush=True)
# Deserialize xmltv file
try:
data = xmltv.read_data(xmltv_fp)
channels = xmltv.read_channels(xmltv_fp)
programmes = xmltv.read_programmes(xmltv_fp)
except Exception as e:
print(
'\t\t\t- This file seems to be corrupt :-/, delete it ({})'
.format(e),
flush=True)
os.remove(xmltv_fp)
continue
# Data
if grabber not in all_data:
all_data[grabber] = data
# Channels
print('\t\t\t- This file contains {} channels'.format(
len(channels)),
flush=True)
for channel in channels:
channel_id = channel['id']
if channel_id not in all_channels:
all_channels[channel_id] = channel
if channel_id not in all_programmes:
all_programmes[channel_id] = []
if channel_id not in all_programmes_local:
all_programmes_local[channel_id] = []
# Programmes
# If there is no programme, delete this raw xmltv file
if not programmes:
print(
'\t\t\t- This file does not contain any TV shows :-/, delete it',
flush=True)
os.remove(xmltv_fp)
else:
print('\t\t\t- This file contains {} TV shows'.format(
len(programmes)),
flush=True)
for programme in programmes:
channel_id = programme['channel']
all_programmes_local[channel_id].append(dict(programme))
# Replace local datetime by UTC datetime
programme_utc = dict(programme)
for elt in ['start', 'stop']:
if elt not in programme_utc:
continue
s = programme_utc[elt]
# Remove timezone part to get %Y%m%d%H%M%S format
s = s.split(' ')[0]
# Get the naive datetime object
d = datetime.strptime(s, DATE_FORMAT_NOTZ)
# Add correct timezone
tz = pytz.timezone(grabber_infos['tz'])
d = tz.localize(d)
# Convert to UTC timezone
utc_tz = pytz.UTC
d = d.astimezone(utc_tz)
# Finally replace the datetime with the UTC one
s = d.strftime(DATE_FORMAT_NOTZ)
# print('Replace {} by {}'.format(programme_utc[elt], s))
programme_utc[elt] = s
all_programmes[channel_id].append(dict(programme_utc))
return (all_data, all_channels, all_programmes, all_programmes_local)
def __init__(self,xmltvfile):
self.stations = xmltv.read_channels(xmltvfile)
self.programm = xmltv.read_programmes(xmltvfile)
self.stations = self.get_stations(self.stations)
# If you need to change the locale:
# xmltv.locale = 'Latin-1'
# If you need to change the date format used in the XMLTV file:
# xmltv.date_format = '%Y%m%d%H%M%S %Z'
#https://www.bloggoiptv.com/fonti-per-le-guide-epg/
#http://www.epg-guide.com/it.gz
filename = '../it.xml'
def programme_to_pdrow(x):
x.pop('desc', None)
if not isinstance(x['stop'], datetime.datetime):
x['stop'] = parser.parse(x['stop'])
if not isinstance(x['start'], datetime.datetime):
x['start'] = parser.parse(x['start'])
x['duration'] = x['stop'] - x['start']
x['category'] = "|".join([z[0] for z in x.get('category', [])])
x['title'] = "|".join([z[0] for z in x.get('title', [])])
return x
with open(filename, 'r') as f:
data = xmltv.read_programmes(f)
p = pd.DataFrame([programme_to_pdrow(j) for j in data])
gg=p.groupby('title')[['title']].count()
vv=gg.title.sort_values(ascending=False)
def Update_Listings(self):
# Download the file from `url` and save it locally under `file_name`:
# urllib.request.urlretrieve("http://www.xmltv.co.uk/feed/6721", "tvlistings.xml")
self.channelsInputDict = xmltv.read_channels(open('tvlistings.xml', 'r'))
self.programmeInputDict = xmltv.read_programmes(open('tvlistings.xml', 'r'))
self.lastUpdatedListings = datetime.now()
'raw_fp': os.path.join(WD, '../raw/tv_guide_it.xml'),
'dst_fp': os.path.join(WD, '../tv_guide_it{}.xml'),
'tz': 'Europe/Rome'
}
}
for country_code, country_infos in countries.items():
print('* Processing of {} country:'.format(country_code))
# Parse channels, data and programmes in the raw xmltv file
try:
# country_infos['channels_list'] = xmltv.read_channels(open(country_infos['raw_fp'], 'r'))
# country_infos['programmes_list_local_datetime'] = xmltv.read_programmes(open(country_infos['raw_fp'], 'r'))
# country_infos['data_list'] = xmltv.read_data(open(country_infos['raw_fp'], 'r'))
channels_l = xmltv.read_channels(open(country_infos['raw_fp'], 'r'))
programmes_local_datetime_l = xmltv.read_programmes(
open(country_infos['raw_fp'], 'r'))
data_l = xmltv.read_data(open(country_infos['raw_fp'], 'r'))
except Exception:
continue
# XMLTV data stays untouched
country_infos['data_l'] = data_l
# If any filter on channels exists, remove unwanted channels
if 'channels_to_add' in country_infos:
country_infos['channels_l'] = []
for channel in channels_l:
if 'id' in channel and channel['id'] in country_infos[
'channels_to_add']:
country_infos['channels_l'].append(channel)
else:
sys.exit(-1)
print 'Connecting to DB...'
conn = db_connect()
print '(Re-)Creating tables...'
create_tables(conn)
print 'Parsing categories...'
try:
categories = parse_categories(sys.argv[2])
except Exception as e:
print 'Error while parsing %s: %s' % (sys.argv[2], e)
conn.close()
sys.exit(-1)
print 'Parsing EPG from %s...' % sys.argv[1]
try:
programmes = xmltv.read_programmes(open(sys.argv[1], 'r'))
except Exception as e:
print 'Error while parsing %s: %s' % (sys.argv[2], e)
conn.close()
sys.exit(-1)
programmes.sort(key = lambda program : int(calendar.timegm(parser.parse(program['start']).utctimetuple())))
print 'Inserting data to DB...'
insert_programs(conn, programmes, categories)
conn.close()
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import xmltv
if __name__ == '__main__':
target = '10117_e-entertainment.xml'
xml = open(target, 'r')
print xml
epgs = xmltv.read_programmes(xml)
for epg in epgs:
print epg['date']
print epg['start']
print epg['stop']
print epg['title'][0][0]
print epg['category'][0][0]
print epg
])
def channels():
channels = {}
for key in xmltv.read_channels(open(filename, 'r')):
name = map(itemgetter(0), key['display-name'])
id = key['id']
src = key['icon'][0]['src']
name = name[0]
rec = dict(zip(Channel._fields, [id, name, src]))
channel = Channel(**rec)
channels[channel.id] = channel
return channels
CHANNELS = channels()
def create_channel(id):
return CHANNELS[id]
def format_time(timestamp):
return datetime.datetime.strptime(timestamp[:12], "%Y%m%d%H%M%S")
if __name__ == "__main__":
for key in xmltv.read_programmes(open(filename, 'r')):
channel = create_channel(key['channel'])
titles = map(itemgetter(0), key['title'])
print "%s - %s - %s - %s" % (titles[0], channel.name, format_time(key['start']), format_time(key['stop']))
'''
import xmltv
import re
from pprint import pprint
from datetime import datetime, timedelta
import json
import pytz
timezone = pytz.timezone("Europe/Amsterdam")
tvseries = json.load(open("shows.json"))
filename = "guide.xml"
xmltv.read_data(open(filename, 'r'))
programmes = xmltv.read_programmes(open(filename, 'r'))
result = list()
def _should_get(num, numSet):
if not numSet:
return 1
for val in numSet:
if val.find("-") != -1:
sval = int(val.split("-")[0])
fval = int(val.split("-")[1])
def __init__(self, xmltvfile):
self.stations = xmltv.read_channels(xmltvfile)
self.programm = xmltv.read_programmes(xmltvfile)
self.stations = self.get_stations(self.stations)
