def fetch(self, utc_date = None, utc_time = None, day = None, region = 'us', days = 1): region = region days = days sql = "SELECT id, name, slug FROM epgsdotcom_channels WHERE region = '%s'" % region self.cursor.execute(sql) channels = self.cursor.fetchall() # a list for channel in channels[:]: # a tuple xmltv_url = self.epgs_url + 'checksum=' + self.checksum + '&channel=' + str(channel[0]) + '&days=' + str(days) xml = urllib2.urlopen(xmltv_url) # a file-like handle epgs = xmltv.read_programmes(xml) self.store(channel = channel[1], data = epgs)
def parse_broadcasts(): broadcasts = [] for element in xmltv.read_programmes(open(filename, 'r')): channel = retrieve_channel(element['channel']) broadcasts.append({ 'channel': channel['id'], 'title': retrieve_title(element), 'start_time': format_time(element['start']).strftime("%Y-%m-%d %H:%M"), 'end_time': format_time(element['stop']).strftime("%Y-%m-%d %H:%M"), 'blurb': retrieve_blurb(element) }) return broadcasts
def __load_programs(self): for program in xmltv.read_programmes(open(self.path)): error_msg = "" if not program.has_key("title"): continue elif not program.has_key("start"): error_msg = "missing start time" elif not program.has_key("stop"): error_msg = "missing stop time" if error_msg: print >>sys.stderr, 'Unable to add program "%s" to Listings: %s' % (program["title"][0][0], error_msg) continue channel_name = self.__get_channel_name(program) p = Program(program) self.listings.add_program(p, channel_name) if self.config.debug: print('Added program "%s" to Listings.' % p.title)
def tv(self, irc, msg, args): """takes no arguments Returns the list of current TV shows. """ global filenames now = datetime.datetime.today() show_list = "" show_count = 0 for filename in filenames: print filename try: chans = xmltv.read_channels(open(filename, 'r')) for chan in chans: try: chan_name = chan['display-name'][0][0] print chan_name shows = xmltv.read_programmes(open(filename, 'r')) for show in shows: try: start_time = parse(show['start'],ignoretz=True) stop_time = parse(show['stop'],ignoretz=True) if (start_time <= now and stop_time >= now): show_count = show_count + 1 show_list = show_list + "\x0F" + "\x02" + chan_name + "\x0F" + " " + str(start_time.strftime("%H:%M")) + " " + (show['title'][0][0]).encode("UTF-8") + " " # if not (show_count % 3): # irc.reply(show_list) # show_list = "" except: pass except: pass except: pass irc.reply(show_list) print show_list
def write_cats(root, cont, is_channel_cats): global cat_id for item in cont.keys(): if not item[0]: continue elem = SubElement(root, 'Category') if item[1]: elem.set('lang', item[1]) elem.set('id', str(cat_id)) cat_id += 1 if is_channel_cats: for ch in cont[item]: ch_elem = SubElement(elem, 'Channel') ch_elem.set('id', ch) elem.set('channel', 'true') elem.set('name', item[0]) if len(sys.argv) != 2: print 'Usage: %s epg.xml' % sys.argv[0] sys.exit(-1) channel_cats = find_cats(xmltv.read_channels(open(sys.argv[1]))) programmes_cats = find_cats(xmltv.read_programmes(open(sys.argv[1]))) root = Element('Categories') et = ElementTree(root) write_cats(root, channel_cats, True) write_cats(root, programmes_cats, False) et.write(sys.stdout, 'UTF-8')
def get_xmltv(): """ Download XMLTV url and store channels and programs in the database. :return: None :return: """ # http://wiki.xmltv.org/index.php/Main_Page/xmltvfileformat.html import urllib2 import gzip import StringIO import xmltv url = cfg.TVGURL # Download XMLTV source out_file_path = url.split("/")[-1][:-3] print('Downloading TV program from: {}'.format(url)) response = urllib2.urlopen(url) compressed_file = StringIO.StringIO(response.read()) decompressed_file = gzip.GzipFile(fileobj=compressed_file) # Extract XMLTV with open(out_file_path, 'w') as outfile: outfile.write(decompressed_file.read()) # Print XMLTV header xmltv_data = xmltv.read_data(open(out_file_path, 'r')) ic(xmltv_data) # Read xml channels xmlchannels = xmltv.read_channels(open(out_file_path, 'r')) print("Got {} channels from XMLTV source".format(len(xmlchannels))) # Drop content of XMLChannel XMLChannel.query.delete() db.session.commit() # Populate XMLChannel with channels from XMLTV source for xc in xmlchannels: xmlchannel = XMLChannel(id=int(xc['id']), label=xc['display-name'][0][0].strip()) db.session.add(xmlchannel) db.session.commit() programs = xmltv.read_programmes(open(out_file_path, 'r')) chunk = 1024 index = 0 for pr in programs: desc = "" try: desc = pr['desc'][0][0] except KeyError: pass a_category = Category.query.filter( Category.name == pr['category'][0][0]).first() if a_category: p = Program(channel=int(pr['channel']), title=pr['title'][0][0], start=duparse(pr['start']), stop=duparse(pr['stop']), desc=desc, category_id=a_category.id) db.session.add(p) else: py = Category(name=pr['category'][0][0]) Program(channel=int(pr['channel']), title=pr['title'][0][0], start=duparse(pr['start']), stop=duparse(pr['stop']), desc=desc, category=py) db.session.add(py) index += 1 if index % chunk == 0: db.session.commit() db.session.commit() categories = [x.name for x in Category.query.all()] ic(u', '.join(categories))
channel, created = Channel.objects.get_or_create( type=Channel.TYPE_TV_CHANNEL, fetcher__cid=id, fetcher__name='xmltv', defaults=defaults(), ) if created: Fetcher(name='xmltv', channel=channel, cid=id).save() if not created and self.opt.get('force'): if channel.icon: channel.icon.delete() for (n, v) in defaults(): setattr(channel, n, v) channel.save() return for id, programs in groupby(xmltv.read_programmes(input_file), lambda p: p['channel']): try: channel = Channel.objects.get(fetcher__name='xmltv', fetcher__cid=id) except Channel.DoesNotExist, e: self.logger.warning( "channel %s does not exist, try --update-channels", id) continue last_time = channel.last_screening_time last_time = last_time and pytz.utc.localize(last_time) max_time = None for p in programs: directors = p.get('credits', {}).get('director')
fetcher__cid=id, fetcher__name='xmltv', defaults=defaults(), ) if created: Fetcher(name='xmltv', channel=channel, cid=id).save() if not created and self.opt.get('force'): if channel.icon: channel.icon.delete() for (n, v) in defaults(): setattr(channel, n, v) channel.save() return for id, programs in groupby(xmltv.read_programmes(input_file), lambda p:p['channel']): try: channel = Channel.objects.get(fetcher__name='xmltv', fetcher__cid=id) except Channel.DoesNotExist, e: self.logger.warning("channel %s does not exist, try --update-channels", id) continue last_time = channel.last_screening_time last_time = last_time and pytz.utc.localize(last_time) max_time = None for p in programs: directors = p.get('credits',{}).get('director') title = p.get('title') title = title and title[0][0] subtitle = p.get('sub-title')
if(len(c['display-name']) > 4): display_name = c['display-name'][4][0] else: display_name = c['display-name'][3][0] display_name = display_name.replace(' Low Power', '') display_name = display_name.replace(' Affiliate', '') data[id] = {} data[id]['id'] = id data[id]['callsign'] = callsign data[id]['network'] = display_name # Print programmes programs = xmltv.read_programmes(open(filename, 'r')) next = 0 for p in programs: t = p['channel'] if(t in data): start = int(str(p['start']).replace(' -0600', '')) end = int(str(p['stop']).replace(' -0600', '')) if(next == 1): data[t]['next'] = p['title'][0][0] next = 0 callsign = data[t]['callsign'] data[callsign] = {} data[callsign] = data[t]
def parse_raw_xmltv_files(): """Parse all xmltv files to deserialize info in dict.""" print('\n# Parse all raw XMLTV files', flush=True) all_channels = {} # Key: channel id in xmltv file, Value: channel dict all_data = {} # Key: grabber, Value: data dict all_programmes = { } # Key: channel id, Value: List of dict programme with UTC time all_programmes_local = { } # Key: channel id, Value: List of dict programme with local time for grabber, grabber_infos in GRABBERS.items(): print('\n\t* Processing of {} grabber\'s raw files:'.format(grabber), flush=True) for offset in range(-10, 20): date = TODAY + timedelta(days=offset) xmltv_fp = RAW_DIRECTORY + grabber_infos['raw'].format( '_' + date.strftime("%Y%m%d")) if not os.path.exists(xmltv_fp): continue print('\t\t* Parse {} file'.format(xmltv_fp), flush=True) # Deserialize xmltv file try: data = xmltv.read_data(xmltv_fp) channels = xmltv.read_channels(xmltv_fp) programmes = xmltv.read_programmes(xmltv_fp) except Exception as e: print( '\t\t\t- This file seems to be corrupt :-/, delete it ({})' .format(e), flush=True) os.remove(xmltv_fp) continue # Data if grabber not in all_data: all_data[grabber] = data # Channels print('\t\t\t- This file contains {} channels'.format( len(channels)), flush=True) for channel in channels: channel_id = channel['id'] if channel_id not in all_channels: all_channels[channel_id] = channel if channel_id not in all_programmes: all_programmes[channel_id] = [] if channel_id not in all_programmes_local: all_programmes_local[channel_id] = [] # Programmes # If there is no programme, delete this raw xmltv file if not programmes: print( '\t\t\t- This file does not contain any TV shows :-/, delete it', flush=True) os.remove(xmltv_fp) else: print('\t\t\t- This file contains {} TV shows'.format( len(programmes)), flush=True) for programme in programmes: channel_id = programme['channel'] all_programmes_local[channel_id].append(dict(programme)) # Replace local datetime by UTC datetime programme_utc = dict(programme) for elt in ['start', 'stop']: if elt not in programme_utc: continue s = programme_utc[elt] # Remove timezone part to get %Y%m%d%H%M%S format s = s.split(' ')[0] # Get the naive datetime object d = datetime.strptime(s, DATE_FORMAT_NOTZ) # Add correct timezone tz = pytz.timezone(grabber_infos['tz']) d = tz.localize(d) # Convert to UTC timezone utc_tz = pytz.UTC d = d.astimezone(utc_tz) # Finally replace the datetime with the UTC one s = d.strftime(DATE_FORMAT_NOTZ) # print('Replace {} by {}'.format(programme_utc[elt], s)) programme_utc[elt] = s all_programmes[channel_id].append(dict(programme_utc)) return (all_data, all_channels, all_programmes, all_programmes_local)
def __init__(self,xmltvfile): self.stations = xmltv.read_channels(xmltvfile) self.programm = xmltv.read_programmes(xmltvfile) self.stations = self.get_stations(self.stations)
# If you need to change the locale: # xmltv.locale = 'Latin-1' # If you need to change the date format used in the XMLTV file: # xmltv.date_format = '%Y%m%d%H%M%S %Z' #https://www.bloggoiptv.com/fonti-per-le-guide-epg/ #http://www.epg-guide.com/it.gz filename = '../it.xml' def programme_to_pdrow(x): x.pop('desc', None) if not isinstance(x['stop'], datetime.datetime): x['stop'] = parser.parse(x['stop']) if not isinstance(x['start'], datetime.datetime): x['start'] = parser.parse(x['start']) x['duration'] = x['stop'] - x['start'] x['category'] = "|".join([z[0] for z in x.get('category', [])]) x['title'] = "|".join([z[0] for z in x.get('title', [])]) return x with open(filename, 'r') as f: data = xmltv.read_programmes(f) p = pd.DataFrame([programme_to_pdrow(j) for j in data]) gg=p.groupby('title')[['title']].count() vv=gg.title.sort_values(ascending=False)
def Update_Listings(self): # Download the file from `url` and save it locally under `file_name`: # urllib.request.urlretrieve("http://www.xmltv.co.uk/feed/6721", "tvlistings.xml") self.channelsInputDict = xmltv.read_channels(open('tvlistings.xml', 'r')) self.programmeInputDict = xmltv.read_programmes(open('tvlistings.xml', 'r')) self.lastUpdatedListings = datetime.now()
'raw_fp': os.path.join(WD, '../raw/tv_guide_it.xml'), 'dst_fp': os.path.join(WD, '../tv_guide_it{}.xml'), 'tz': 'Europe/Rome' } } for country_code, country_infos in countries.items(): print('* Processing of {} country:'.format(country_code)) # Parse channels, data and programmes in the raw xmltv file try: # country_infos['channels_list'] = xmltv.read_channels(open(country_infos['raw_fp'], 'r')) # country_infos['programmes_list_local_datetime'] = xmltv.read_programmes(open(country_infos['raw_fp'], 'r')) # country_infos['data_list'] = xmltv.read_data(open(country_infos['raw_fp'], 'r')) channels_l = xmltv.read_channels(open(country_infos['raw_fp'], 'r')) programmes_local_datetime_l = xmltv.read_programmes( open(country_infos['raw_fp'], 'r')) data_l = xmltv.read_data(open(country_infos['raw_fp'], 'r')) except Exception: continue # XMLTV data stays untouched country_infos['data_l'] = data_l # If any filter on channels exists, remove unwanted channels if 'channels_to_add' in country_infos: country_infos['channels_l'] = [] for channel in channels_l: if 'id' in channel and channel['id'] in country_infos[ 'channels_to_add']: country_infos['channels_l'].append(channel) else:
sys.exit(-1) print 'Connecting to DB...' conn = db_connect() print '(Re-)Creating tables...' create_tables(conn) print 'Parsing categories...' try: categories = parse_categories(sys.argv[2]) except Exception as e: print 'Error while parsing %s: %s' % (sys.argv[2], e) conn.close() sys.exit(-1) print 'Parsing EPG from %s...' % sys.argv[1] try: programmes = xmltv.read_programmes(open(sys.argv[1], 'r')) except Exception as e: print 'Error while parsing %s: %s' % (sys.argv[2], e) conn.close() sys.exit(-1) programmes.sort(key = lambda program : int(calendar.timegm(parser.parse(program['start']).utctimetuple()))) print 'Inserting data to DB...' insert_programs(conn, programmes, categories) conn.close()
#!/usr/bin/env python # -*- coding: utf-8 -*- import xmltv if __name__ == '__main__': target = '10117_e-entertainment.xml' xml = open(target, 'r') print xml epgs = xmltv.read_programmes(xml) for epg in epgs: print epg['date'] print epg['start'] print epg['stop'] print epg['title'][0][0] print epg['category'][0][0] print epg
]) def channels(): channels = {} for key in xmltv.read_channels(open(filename, 'r')): name = map(itemgetter(0), key['display-name']) id = key['id'] src = key['icon'][0]['src'] name = name[0] rec = dict(zip(Channel._fields, [id, name, src])) channel = Channel(**rec) channels[channel.id] = channel return channels CHANNELS = channels() def create_channel(id): return CHANNELS[id] def format_time(timestamp): return datetime.datetime.strptime(timestamp[:12], "%Y%m%d%H%M%S") if __name__ == "__main__": for key in xmltv.read_programmes(open(filename, 'r')): channel = create_channel(key['channel']) titles = map(itemgetter(0), key['title']) print "%s - %s - %s - %s" % (titles[0], channel.name, format_time(key['start']), format_time(key['stop']))
''' import xmltv import re from pprint import pprint from datetime import datetime, timedelta import json import pytz timezone = pytz.timezone("Europe/Amsterdam") tvseries = json.load(open("shows.json")) filename = "guide.xml" xmltv.read_data(open(filename, 'r')) programmes = xmltv.read_programmes(open(filename, 'r')) result = list() def _should_get(num, numSet): if not numSet: return 1 for val in numSet: if val.find("-") != -1: sval = int(val.split("-")[0]) fval = int(val.split("-")[1])
def __init__(self, xmltvfile): self.stations = xmltv.read_channels(xmltvfile) self.programm = xmltv.read_programmes(xmltvfile) self.stations = self.get_stations(self.stations)