def process(self):
"""process meetings"""
parser = etree.XMLParser(recover=True)
r = requests.get(self.timerange_url)
xml = r.text.encode('ascii','xmlcharrefreplace')
root = etree.fromstring(xml, parser=parser)
for item in root[1].iterchildren():
meeting = {}
for e in item.iterchildren():
meeting[e.tag] = e.text
meeting['start_date'] = parse_date(meeting['sisbvcs'])
meeting['end_date'] = parse_date(meeting['sisevcs'])
meeting['tz_start_date'] = parse_date(meeting['sisbvcs'], tzinfo=utc).astimezone(self.tzinfo)
meeting['tz_end_date'] = parse_date(meeting['sisevcs'], tzinfo=utc).astimezone(self.tzinfo)
silfdnr = meeting['silfdnr']
try:
result = self.process_agenda(silfdnr)
meeting.update(result)
except Exception, e:
meeting['ERROR'] = True
print e
meeting['_id'] = int(meeting['silfdnr'])
self.db.meetings.save(meeting)
def validate_no_bigamy(fams, indis):
ret_data = []
for iid in indis:
marriages = []
for fid in indis[iid]['FAMS']:
if any(fams[fid][tag] is None for tag in ['HUSB', 'WIFE', 'MARR']):
continue
begin_date = utils.parse_date(fams[fid]['MARR'])
end_date = None
if indis[fams[fid]['HUSB']]['DEAT'] is not None:
deat = utils.parse_date(indis[fams[fid]['HUSB']]['DEAT'])
end_date = (deat if end_date is None else min(end_date, deat))
if indis[fams[fid]['WIFE']]['DEAT'] is not None:
deat = utils.parse_date(indis[fams[fid]['WIFE']]['DEAT'])
end_date = (deat if end_date is None else min(end_date, deat))
if fams[fid]['DIV'] is not None:
div = utils.parse_date(fams[fid]['DIV'])
end_date = (div if end_date is None else min(end_date, div))
marriages.append((begin_date, end_date))
marriages.sort()
for i in range(len(marriages) - 1):
int1 = marriages[i]
int2 = marriages[i + 1]
if utils.interval_intersect(int1, int2):
ret_data.append((
iid,
f'Individual id={iid} was in two or more marriages at the same time'
))
break
return ret_data
def history(begin=None, end=None, week=False, month=False):
if week or month:
end = datetime.datetime.now()
if week:
begin = end - datetime.timedelta(days=7)
if month:
begin = end - datetime.timedelta(days=28)
else:
try:
begin = utils.parse_date(begin, dtime=True)
except ValueError:
return 'Error: unable to parse date %s' % begin
try:
end = utils.parse_date(end, dtime=True)
except ValueError:
return 'Error: unable to parse date %s' % end
cd = utils.compose_date
if not begin <= end:
return 'Error: %s is not before %s' % (cd(begin), cd(end))
db = utils.read_db()
days = [
utils.compose_date(begin + datetime.timedelta(days=x))
for x in xrange(0, (end - begin).days)
]
matches = ['from: %s to: %s' % (cd(begin), cd(end))]
for day in days:
matches.append('%s: %s' % (day, utils.daily_points(day, db)))
return '\n'.join(matches)
def validate_birth_before_marriage(fams, indis):
return_data = []
for fid in fams:
if fams[fid]['MARR'] is not None:
marriage_day = utils.parse_date(fams[fid]['MARR'])
husband_id = fams[fid]['HUSB']
wife_id = fams[fid]['WIFE']
husband_birth = indis[husband_id]['BIRT']
wife_birth = indis[wife_id]['BIRT']
if husband_birth is not None:
husband_birthday = utils.parse_date(husband_birth)
if wife_birth is not None:
wife_birthday = utils.parse_date(wife_birth)
if marriage_day < husband_birthday:
return_data.append((
husband_id,
f'Husband id={husband_id} in family id={fid} has marriage before birth.'
))
if marriage_day < wife_birthday:
return_data.append((
wife_id,
f'Wife id={wife_id} in family id={fid} has marriage before birth.'
))
return return_data
def validate_marriage_after_fourteen(fams, indis):
invalid_marriages = []
for fid in fams:
# if married
if fams[fid]['MARR'] is not None:
marriage_date = utils.parse_date(fams[fid]['MARR'])
husbandID, wifeID = fams[fid]['HUSB'], fams[fid]['WIFE']
husband_birth, wife_birth = indis[husbandID]['BIRT'], indis[
wifeID]['BIRT']
ages = []
if husband_birth is not None:
husband_birth = utils.parse_date(husband_birth)
husband_marriage_age = utils.get_age(husband_birth,
marriage_date)
ages.append(husband_marriage_age)
if wife_birth is not None:
wife_birth = utils.parse_date(wife_birth)
wife_marriage_age = utils.get_age(wife_birth, marriage_date)
ages.append(wife_marriage_age)
if len(ages) > 0 and min(ages) < 14:
invalid_marriages.append(
(fid, f'Family id={fid} has marriage before age 14'))
return invalid_marriages
def validate_divorce_before_death(fams, indis):
ret_data = []
for fid in fams:
if fams[fid]['DIV'] is not None:
divorce_date = utils.parse_date(fams[fid]['DIV'])
husband_id = fams[fid]['HUSB']
wife_id = fams[fid]['WIFE']
if indis[husband_id]['DEAT'] is not None:
death_date = utils.parse_date(indis[husband_id]['DEAT'])
if death_date < divorce_date:
ret_data.append((
husband_id,
f'Individual id={husband_id} has a divorce after his death'
))
if indis[wife_id]['DEAT'] is not None:
death_date = utils.parse_date(indis[wife_id]['DEAT'])
if death_date < divorce_date:
ret_data.append((
wife_id,
f'Individual id={wife_id} has a divorce after her death'
))
return ret_data
def process(self):
"""process meetings"""
parser = etree.XMLParser(recover=True)
r = requests.get(self.timerange_url)
xml = r.text.encode('ascii','xmlcharrefreplace')
root = etree.fromstring(xml, parser=parser)
start_at = root.find("list")
for item in start_at.iterchildren():
meeting = {}
for e in item.iterchildren():
meeting[e.tag] = e.text
meeting['start_date'] = parse_date(meeting['sisbvcs'])
meeting['end_date'] = parse_date(meeting['sisevcs'])
meeting['tz_start_date'] = parse_date(meeting['sisbvcs'], tzinfo=utc).astimezone(self.tzinfo)
meeting['tz_end_date'] = parse_date(meeting['sisevcs'], tzinfo=utc).astimezone(self.tzinfo)
silfdnr = meeting['silfdnr']
print "processing meeting %s" %silfdnr
try:
result = self.process_agenda(silfdnr)
meeting.update(result)
except Exception, e:
meeting['ERROR'] = True
print "exception when trying to parse meeting %s" %silfdnr
print e
meeting['meeting_id'] = str(meeting['silfdnr'])
meeting['_id'] = "%s:%s" %(self.city, silfdnr)
meeting['city'] = self.city
self.db.meetings.save(meeting)
def history(begin=None, end=None, week=False, month=False):
if week or month:
end = datetime.datetime.now()
if week:
begin = end - datetime.timedelta(days=7)
if month:
begin = end - datetime.timedelta(days=28)
else:
try:
begin = utils.parse_date(begin, dtime=True)
except ValueError:
return 'Error: unable to parse date %s' % begin
try:
end = utils.parse_date(end, dtime=True)
except ValueError:
return 'Error: unable to parse date %s' % end
cd = utils.compose_date
if not begin <= end:
return 'Error: %s is not before %s' % (cd(begin), cd(end))
db = utils.read_db()
days = [utils.compose_date(begin + datetime.timedelta(days=x)) for x in xrange(0,
(end-begin).days)]
matches = ['from: %s to: %s' % (cd(begin), cd(end))]
for day in days:
matches.append('%s: %s' % (day, utils.daily_points(day, db)))
return '\n'.join(matches)
def events():
try:
args = parse_url_arguments(request.url)
start_date = parse_date(args["start_date"])
end_date = parse_date(args["end_date"])
classification = args["classification"]
return get_events(start_date, end_date, classification)
except Exception as ex:
return str(ex)
def tweets(uid,min_date=None,max_date=None,limit=100):
"returns all the tweets for a user between min_date and max_date"
start = parse_date(min_date)
end = parse_date(max_date)
limit = int(limit) if limit else None
tweets = Tweet.find(
(Tweet.user_id==int(uid)) &
(Tweet.created_at.range(start,end)),
limit=limit,
sort=Tweet._id)
return [t.to_d() for t in tweets]
def trim_and_sort_events(events):
"""Sorts given events by event date, and
trims those that are past today.
"""
events = [
e for e in events \
if parse_date(e['event_date']) \
>= utc.localize(datetime.now())
]
events = sorted(events, key=lambda x: parse_date(x['event_date']))
return events
def __init__(self, timestamp, kwargs):
if isinstance(timestamp, str):
self.timestamp = parse_date(timestamp)
elif isinstance(timestamp, unicode):
self.timestamp = parse_date(timestamp)
elif isinstance(timestamp, datetime):
self.timestamp = timestamp
self.id = base64.b64decode(self.timestamp)
self.urls = {}
self.report_name, self.report_number = get_report_period(
self.timestamp)
for name, url in kwargs.items():
if url:
self.urls[name] = URL(self.id, name, url)
def validate_birth_before_death(fams, indis):
return_data = []
for cid in indis:
if indis[cid]['BIRT'] is not None:
birthday = utils.parse_date(indis[cid]['BIRT'])
if indis[cid]['DEAT'] is not None:
death_day = utils.parse_date(indis[cid]['DEAT'])
if death_day < birthday:
return_data.append(
(cid, f'Person id={cid} has death before birth.'))
return return_data
def parse_template_5(self, element):
"""
A template for a workshop with the conference acronym and year in the name
Examples:
- http://ceur-ws.org/Vol-958/
"""
workshop = {}
title = rex.rex(element[1], r'(.*)Edited\s*by.*', re.I | re.S).group(1)
workshop['volume_number'] = WorkshopSummaryParser.extract_volume_number(element[0].get('href'))
label_part = rex.rex(element[0].text, r'(.*)\sat\s(\w{2,})\s(\d{4})[\s\.]*', re.I | re.S)
workshop['label'] = label_part.group(1)
workshop['conf_acronym'] = label_part.group(2)
workshop['conf_year'] = label_part.group(3)
workshop['url'] = element[0].get('href')
workshop['time'] = utils.parse_date(title)
try:
workshop['edition'] = tonumber(
rex.rex(title,
r'.*Proceedings(\s*of)?(\s*the)?\s*(\d{1,}|first|second|third|forth|fourth|fifth)[thrd]*'
r'.*Workshop.*',
re.I, default=None).group(3))
except:
#'edition' property is optional
pass
self.add_workshop(workshop)
def validate_no_sextuples(fams, indis):
one_day = timedelta(days=1)
ret_data = []
for fid in fams:
child_births = [
indis[cid]['BIRT'] for cid in fams[fid]['CHIL']
if indis[cid]['BIRT'] is not None
]
if len(child_births) <= 5:
continue
birth_freq = {}
for birth in child_births:
if birth not in birth_freq:
birth_freq[birth] = 0
birth_freq[birth] += 1
dates = sorted(birth_freq.keys())
parsed_dates = [utils.parse_date(d) for d in dates]
for i in range(len(dates)):
count = birth_freq[dates[i]]
if i + 1 < len(dates) and parsed_dates[
i + 1] - parsed_dates[i] == one_day:
count = birth_freq[dates[i]] + birth_freq[dates[i + 1]]
if count > 5:
ret_data.append((
fid,
f'Family id={fid} has more than 5 children born together'))
break
return ret_data
def parse_template_2(self, element):
"""
A template for joint proceedings of two workshops:
Examples:
- http://ceur-ws.org/Vol-776/
"""
workshop_1 = {'id': 1}
workshop_2 = {'id': 2}
summary = rex.rex(element[1], r'^\s*(proceedings\s+of\s+joint.*on.*\((\w+)\-(\w+)\s+\d+\).*)Edited by.*',
re.I | re.S)
if len(summary.groups()) != 3:
raise DataNotFound()
title = summary.group(1)
workshop_1['volume_number'] = workshop_2['volume_number'] = \
WorkshopSummaryParser.extract_volume_number(element[0].get('href'))
workshop_1['url'] = workshop_2['url'] = element[0].get('href')
workshop_1['time'] = workshop_2['time'] = utils.parse_date(title)
workshop_1['short_label'] = summary.group(2)
workshop_2['short_label'] = summary.group(3)
self.add_workshop(workshop_1)
self.add_workshop(workshop_2)
def parse_template_3(self, element):
"""
A template for joint proceedings of two workshops.
Examples:
- http://ceur-ws.org/Vol-1098/
- http://ceur-ws.org/Vol-989/
"""
workshop_1 = {'id': 1}
workshop_2 = {'id': 2}
summary = self.rex(element[1], [
r"(joint\s+proceedings\s+of\s+([\s\w,]+)\(([a-zA-Z]+)['\s]?\d+\)[and,\s]+"
r"([:\s\w-]+)\(([a-zA-Z]+)['\s]?\d+\)([\w\s\-.,^\(]*|[,\s]+workshops\s+of.*|[,\s]+co-located.*))Edited by.*",
r"(proceedings\s+of\s+joint([\s\w,]+)\(([a-zA-Z]+)['\s]?\d{0,4}\)[and,\s]+"
r"([:,\s\w-]+)\(([a-zA-Z]+)['\s]?\d{0,4}\)([\w\s\-.,^\(]*|[,\s]+workshops\s+of.*|[,\s]+co-located.*))Edited by.*"
], re.I | re.S)
if len(summary.groups()) != 6:
raise DataNotFound()
title = summary.group(1)
workshop_1['volume_number'] = workshop_2['volume_number'] = \
WorkshopSummaryParser.extract_volume_number(element[0].get('href'))
workshop_1['url'] = workshop_2['url'] = element[0].get('href')
workshop_1['time'] = workshop_2['time'] = utils.parse_date(title)
workshop_1['label'] = summary.group(2)
workshop_1['short_label'] = summary.group(3)
workshop_2['label'] = summary.group(4)
workshop_2['short_label'] = summary.group(5)
self.add_workshop(workshop_1)
self.add_workshop(workshop_2)
def __init__(self, logger, sequences, reference, dateFormat):
super(sequence_set, self).__init__()
self.log = logger
# load sequences from the (parsed) JSON - don't forget to sort out dates
self.seqs = {}
for name, data in sequences.iteritems():
self.seqs[name] = SeqRecord(Seq(data["seq"], generic_dna),
id=name, name=name, description=name)
self.seqs[name].attributes = data["attributes"]
# tidy up dates
date_struc = parse_date(self.seqs[name].attributes["raw_date"], dateFormat)
self.seqs[name].attributes["num_date"] = date_struc[1]
self.seqs[name].attributes["date"] = date_struc[2]
# if the reference is to be analysed it'll already be in the (filtered & subsampled)
# sequences, so no need to add it here, and no need to care about attributes etc
# we do, however, need it for alignment
self.reference_in_dataset = reference["included"]
name = reference["strain"]
self.reference_seq = SeqRecord(Seq(reference["seq"], generic_dna),
id=name, name=name, description=name)
if "genes" in reference and len(reference["genes"]):
self.proteins = {k:FeatureLocation(start=v["start"], end=v["end"], strand=v["strand"]) for k, v in reference["genes"].iteritems()}
else:
self.proteins = None
# other things:
self.run_dir = '_'.join(['temp', time.strftime('%Y%m%d-%H%M%S',time.gmtime()), str(random.randint(0,1000000))])
self.nthreads = 2 # should load from config file
def convert_trait_to_numerical_date(self, trait, dateFormat):
for name, seq in self.seqs.iteritems():
try:
date_struc = parse_date(seq.attributes[trait], dateFormat)
seq.attributes[trait] = date_struc[1]
except KeyError:
self.log.warn("Attribute {} not found for sequence {}. Ignoring".format(trait, seq.name))
def parse_template_3(self, element):
"""
A template for joint proceedings of two workshops.
Examples:
- http://ceur-ws.org/Vol-1098/
- http://ceur-ws.org/Vol-989/
"""
workshop_1 = {'id': 1}
workshop_2 = {'id': 2}
summary = self.rex(element[1], [
r"(joint\s+proceedings\s+of\s+([\s\w,]+)\(([a-zA-Z]+)['\s]?\d+\)[and,\s]+"
r"([:\s\w-]+)\(([a-zA-Z]+)['\s]?\d+\)([\w\s\-.,^\(]*|[,\s]+workshops\s+of.*|[,\s]+co-located.*))Edited by.*",
r"(proceedings\s+of\s+joint([\s\w,]+)\(([a-zA-Z]+)['\s]?\d{0,4}\)[and,\s]+"
r"([:,\s\w-]+)\(([a-zA-Z]+)['\s]?\d{0,4}\)([\w\s\-.,^\(]*|[,\s]+workshops\s+of.*|[,\s]+co-located.*))Edited by.*"
], re.I | re.S)
if len(summary.groups()) != 6:
raise DataNotFound()
title = summary.group(1)
workshop_1['volume_number'] = workshop_2['volume_number'] = \
WorkshopSummaryParser.extract_volume_number(element[0].get('href'))
workshop_1['url'] = workshop_2['url'] = element[0].get('href')
workshop_1['time'] = workshop_2['time'] = utils.parse_date(title)
workshop_1['label'] = summary.group(2)
workshop_1['short_label'] = summary.group(3)
workshop_2['label'] = summary.group(4)
workshop_2['short_label'] = summary.group(5)
self.add_workshop(workshop_1)
self.add_workshop(workshop_2)
def map_video(config):
programId = config.get('programId')
kind = config.get('kind')
duration = int(config.get('duration') or 0) * \
60 or config.get('durationSeconds')
airdate = config.get('broadcastBegin')
if airdate is not None:
airdate = str(utils.parse_date(airdate))
return {
'label': utils.format_title_and_subtitle(config.get('title'), config.get('subtitle')),
'path': plugin.url_for('play', kind=kind, program_id=programId),
'thumbnail': config.get('imageUrl'),
'is_playable': True,
'info_type': 'video',
'info': {
'title': config.get('title'),
'duration': duration,
'genre': config.get('genrePresse'),
'plot': config.get('shortDescription') or config.get('fullDescription'),
'plotoutline': config.get('teaserText'),
# year is not correctly used by kodi :(
# the aired year will be used by kodi for production year :(
# 'year': int(config.get('productionYear')),
'country': [country.get('label') for country in config.get('productionCountries', [])],
'director': config.get('director'),
'aired': airdate
},
'properties': {
'fanart_image': config.get('imageUrl'),
}
}
def parse_standard_rss(self, url):
headers = self.set_headers()
req = urllib.request.Request(url, headers=headers)
parse_xml_url = urllib.request.urlopen(req)
xml_page = parse_xml_url.read()
parse_xml_url.close()
soup_page = BeautifulSoup(xml_page, "lxml")
channel = soup_page.find("channel")
news_list = channel.findAll("item")
links = []
for getfeed in news_list:
titolo = getfeed.title.text
description = ""
if getfeed.description.text:
description = getfeed.description.text
link_id = generate_link_id(titolo)
links.append({
'id': link_id,
'titolo': titolo,
'text': description,
'url': getfeed.link.nextSibling.rstrip(),
'data': parse_date(getfeed.pubdate.text)
})
if JUST_ONE_LINK:
break
return links
def parse_template_5(self, element):
"""
A template for a workshop with the conference acronym and year in the name
Examples:
- http://ceur-ws.org/Vol-958/
"""
workshop = {}
title = rex.rex(element[1], r'(.*)Edited\s*by.*', re.I | re.S).group(1)
workshop['volume_number'] = WorkshopSummaryParser.extract_volume_number(element[0].get('href'))
label_part = rex.rex(element[0].text, r'(.*)\sat\s(\w{2,})\s(\d{4})[\s\.]*', re.I | re.S)
workshop['label'] = label_part.group(1)
workshop['conf_acronym'] = label_part.group(2)
workshop['conf_year'] = label_part.group(3)
workshop['url'] = element[0].get('href')
workshop['time'] = utils.parse_date(title)
try:
workshop['edition'] = tonumber(
rex.rex(title,
r'.*Proceedings(\s*of)?(\s*the)?\s*(\d{1,}|first|second|third|forth|fourth|fifth)[thrd]*'
r'.*Workshop.*',
re.I, default=None).group(3))
except:
#'edition' property is optional
pass
self.add_workshop(workshop)
def parse_template_2(self, element):
"""
A template for joint proceedings of two workshops:
Examples:
- http://ceur-ws.org/Vol-776/
"""
workshop_1 = {'id': 1}
workshop_2 = {'id': 2}
summary = rex.rex(element[1], r'^\s*(proceedings\s+of\s+joint.*on.*\((\w+)\-(\w+)\s+\d+\).*)Edited by.*',
re.I | re.S)
if len(summary.groups()) != 3:
raise DataNotFound()
title = summary.group(1)
workshop_1['volume_number'] = workshop_2['volume_number'] = \
WorkshopSummaryParser.extract_volume_number(element[0].get('href'))
workshop_1['url'] = workshop_2['url'] = element[0].get('href')
workshop_1['time'] = workshop_2['time'] = utils.parse_date(title)
workshop_1['short_label'] = summary.group(2)
workshop_2['short_label'] = summary.group(3)
self.add_workshop(workshop_1)
self.add_workshop(workshop_2)
def command_received(self, cmd_data):
LOG_MSG('Poll command got response. Response: {0}.'.format(cmd_data))
for cmd in cmd_data:
# Obtain only new commands next time
cmd_date = parse_date(cmd['timestamp'])
if self.owner.timestamp is not None:
self.owner.timestamp = max(self.owner.timestamp, cmd_date)
else:
self.owner.timestamp = cmd_date
# device-application will use this deferred object to notify me about the command progress.
thiscmd = cmd
def ok(result):
self.command_done(thiscmd, result)
def err(reason):
self.command_failed(thiscmd, reason)
defer = Deferred()
defer.addCallbacks(ok, err)
try:
LOG_MSG('Executing command {0} handler.'.format(cmd))
self.owner.run_command(cmd, defer)
except Exception, err:
LOG_ERR(
'Failed to execute device-delegate on_command. Reason: <{0}>.'
.format(err))
self.command_failed(cmd, err)
def parse_data(data):
final_data = {}
for year, call_log in data.items():
print(f'Getting call data for {year}')
# UNCOMMENT THIS FOR CREATING A YEAR KEY FOR EACH YEAR (1 OF 3)
# final_data[year] = {}
for month, calls in call_log.items():
# UNCOMMENT THIS FOR CREATING A MONTH KEY FOR EACH MONTH (2 OF 3)
# month = parse_month(month=month, abbreviated=False)
# final_data[year][month] = {}
for call in calls:
call_data = call.split("-")
call_date = utils.parse_date(call_data[0].split("_")[0])
call_time = utils.parse_time(call_data[0].split("_")[1])
call_number = utils.parse_call_number(call_data[1])
call_direction = call_data[2].split(".")[0]
# UNCOMMENT THIS FOR CREATING A MONTH KEY FOR EACH MONTH (3 OF 3)
# final_data[year][month].setdefault(call_date, {})
# final_data[year][month][call_date].setdefault(call_direction, [])
# final_data[year][month][call_date][call_direction].append(f'{call_time} - {call_number}')
final_data.setdefault(call_date, [])
final_data[call_date].append(
f'{call_direction} - {call_time} - {call_number}')
return final_data
def gen_indi_table(fams, indis):
indi_table = PrettyTable()
indi_table.field_names = [
'ID', 'Name', 'Gender', 'Birthday', 'Age', 'Alive', 'Death', 'Child',
'Spouse'
]
for indi_id in sorted(indis.keys()):
indi_data = indis[indi_id]
children = []
spouse = None
for fam_id in indi_data['FAMS']:
children += fams[fam_id]['CHIL']
if fams[fam_id]['DIV'] is None:
if indi_data['SEX'] == 'M':
spouse = fams[fam_id]['WIFE']
else:
spouse = fams[fam_id]['HUSB']
age = None
if indi_data['BIRT'] is not None:
birthday = utils.parse_date(indi_data['BIRT'])
age = utils.get_age(birthday)
if age < 0:
age = None
alive = indi_data['DEAT'] is None
indi_table.add_row([
indi_id, indi_data['NAME'] or 'NA', indi_data['SEX'] or 'NA',
indi_data['BIRT'] or 'NA', age or 'NA', alive, indi_data['DEAT']
or 'NA',
utils.format_list(children) if len(children) > 0 else 'NA', spouse
or 'NA'
])
return indi_table
def lunar_phase():
try:
args = parse_url_arguments(request.url)
date = parse_date(args["date"])
return get_lunar_phase(date)
except Exception as ex:
return str(ex)
def map_video(config):
programId = config.get('programId')
kind = config.get('kind')
duration = int(config.get('duration') or 0) * \
60 or config.get('durationSeconds')
airdate = config.get('broadcastBegin')
if airdate is not None:
airdate = str(utils.parse_date(airdate))
return {
'label': utils.format_title_and_subtitle(config.get('title'), config.get('subtitle')),
'path': plugin.url_for('play', kind=kind, program_id=programId),
'thumbnail': config.get('imageUrl'),
'is_playable': True,
'info_type': 'video',
'info': {
'title': config.get('title'),
'duration': duration,
'genre': config.get('genrePresse'),
'plot': config.get('shortDescription') or config.get('fullDescription'),
'plotoutline': config.get('teaserText'),
# year is not correctly used by kodi :(
# the aired year will be used by kodi for production year :(
#'year': int(config.get('productionYear')),
'country': [country.get('label') for country in config.get('productionCountries', [])],
'director': config.get('director'),
'aired': airdate
},
'properties': {
'fanart_image': config.get('imageUrl'),
}
}
def get(self, args):
""" Return the courses matching your query """
# Parameters
date_string = args['date']
department = args['department']
place = args['place']
comment = args['comment']
# Base query
courses = Course.query
# Apply filters
if date_string:
courses = courses.filter_by(date=parse_date(date_string))
if place:
courses = courses.filter_by(place=place)
if department:
courses = courses.filter_by(department=department)
if comment:
# Can be useful to select a group in a course
courses = courses.filter_by(department=comment)
# Do the query
courses = courses.order_by(Course.time_begin).limit(1000).all()
# Format it
course_list = [course.__toJSON__() for course in courses]
return course_list, 200
def validate_marriage_before_child(fams, indis):
ret_data = []
for fid in fams:
if fams[fid]['MARR'] is not None:
marriage = utils.parse_date(fams[fid]['MARR'])
for cid in fams[fid]['CHIL']:
if indis[cid]['BIRT'] is not None:
birth = utils.parse_date(indis[cid]['BIRT'])
if birth < marriage:
ret_data.append((
fid,
f'Child id={cid} has birthdate before marriage of parents'
))
return ret_data
def getData(request):
from wrcc.wea_server.libwea.products.listers import getData
error = require(request, ['stn', 'sD', 'eD'])
if error:
return ErrorResponse(error)
stn = request.args.get('stn')
sD = parse_date(request.args.get('sD'))
eD = parse_date(request.args.get('eD'))
units_system = request.args.get('units', 'N') # N (native) units by default
try:
result = getData(stn, sD, eD, units_system=units_system)
except IOError:
return ErrorResponse("No data available.")
return JsonResponse(result)
def __init__(self, id, name, summary, date, url='', image=DEFAULT_COVER):
self.id = id
self.name = name
self.summary = summary
self.date = parse_date(date)
self.url = url
self.image = image
self.episodes = []
def check_date(d, context):
if not isinstance(d, str):
error(context, str(d) + ": invalid data type")
return None
try:
return utils.parse_date(d)
except Exception as e:
error(context, d + ": " + str(e))
return None
def scrapp_movie_page(movie_link):
movie = {}
driver = utils.generate_webdriver(show=False)
driver.get(config.MAIN_URL + movie_link)
movie.update({'movie_id': int(driver.current_url.split('/')[-1])})
movie.update({'title': driver.find_element_by_class_name('text').text})
movie.update({'year': int(driver.find_element_by_class_name('year').text)})
movie.update({'duration': utils.parse_date(driver.find_element_by_class_name('duration').text)})
movie.update({'synopsis': driver.find_element_by_class_name('synopsis').text})
movie.update({'maturity': 0})
try:
movie['maturity'] = int(driver.find_element_by_class_name('maturity-number').text)
except ValueError:
pass
driver.find_element_by_id('tab-ShowDetails').click()
details = driver.find_element_by_class_name('simpleSlider') \
.find_element_by_class_name('sliderContent')
# cut_indexes saves the initial indexes of director, cast, screenwriter and the last index
movie_cast = details.find_element_by_tag_name('span').find_elements_by_tag_name('li')
cut_indexes = [i for i, m in enumerate(movie_cast) if m.get_attribute('class') == 'listLabel'] + [len(movie_cast)]
cut_map = {0: 'directors', 1: 'cast', 2: 'screenwriters'}
for i in range(len(cut_indexes)-1):
movie.update({
cut_map[i]: [
{
'person_name': movie_cast[j].text,
'person_id': int(movie_cast[j].find_element_by_tag_name('a') \
.get_attribute('href').split('/')[-1])
}
for j in range(cut_indexes[i]+1, cut_indexes[i+1])
]
})
genres = details.find_element_by_class_name('detailsTags') \
.find_elements_by_tag_name('ul')[0] \
.find_elements_by_tag_name('li')
movie.update({
'genres': [
{
'genre': genre.text,
'genre_id': int(genre.find_element_by_tag_name('a') \
.get_attribute('href').split('/')[-1])
}
for genre in genres
]
})
driver.close()
return movie
def check_date(d):
if not isinstance(d, str):
error(str(d) + ": invalid data type")
return None
try:
return utils.parse_date(d)
except Exception as e:
error(d + ": " + str(e))
return None
def make_row(date_string, headline):
date = parse_date(date_string)
L.debug("Parsed date '{}' as {}".format(date_string, date))
return OrderedDict([
('venue', 'LEAF on Bold Street'),
('date', date),
('headline', headline),
('url', 'http://www.thisisleaf.co.uk/#/on-bold-street/events/')
])
def make_row(day, headline):
date = parse_date(day)
L.debug("Parsed date '{}' as {}".format(day, date))
return OrderedDict([
('venue', 'The Caledonia'),
('date', date),
('headline', headline),
('url', 'http://www.thecaledonialiverpool.com/whats-on/'),
])
def parse_disturbances(self):
"""
For a given trendline, for each segment between vertices, determine
the stats for each "disturbance" and return a list of Disturbance
objects
"""
import utils
it = iter(self.points)
left_vertex = it.next()
for p in it:
if not p.vertex:
continue
start_yr = utils.parse_date(left_vertex.index_date).year
end_yr = utils.parse_date(p.index_date).year
yield Disturbance(start_yr, left_vertex.val_fit,
left_vertex.val_fit - p.val_fit,
end_yr - start_yr)
left_vertex = p
def parse_time_frames(self):
modified_tf = []
if self.args.time_frames[0].isdigit():
for chunk in utils.get_chunks(self.args.time_frames, 3):
if re.search(r'[0-9]+-[0-9]+', ''.join(chunk)):
re.compile('[,]')
modified_tf.append([
int(re.compile('[,]').sub('', chunk[0])),
int(re.compile('[,]').sub('', chunk[2]))
])
else:
time_frames = self.parse_date_string(self.args.time_frames)
for tf in time_frames:
modified_tf.append([
utils.parse_date(tf[0]).timestamp(),
utils.parse_date(tf[1]).timestamp()
])
return modified_tf
def get_last_modified(doc):
lm = None
lmi = doc.select_one('.posted_since')
if lmi:
idx = lmi.text.find('ред. ')
if idx > 0:
val = lmi.text[idx + 5:idx + 20]
lm = utils.parse_date(val)
return lm
def make_row(date_string, headline, url):
date = parse_date(date_string)
L.debug("Parsed date '{}' as {}".format(date_string, date))
return OrderedDict([
('organiser', 'The Bluecoat'),
('venue', 'The Bluecoat'),
('date', date),
('headline', headline),
('url', url)
])
def make_row(date_string, earliest_date, headline, url):
L.debug("'{}' : '{}'".format(headline, date_string))
date = parse_date(date_string, not_before=earliest_date)
L.debug("Parsed date '{}' as {}".format(date_string, date))
return OrderedDict([
('venue', "FACT"),
('date', date),
('headline', headline),
('url', url)
])
def make_row(date_string, headline, url):
L.debug("'{}' : '{}'".format(headline, date_string))
date = parse_date(date_string)
L.debug("Parsed date '{}' as {}".format(date_string, date))
return OrderedDict([
('venue', "St George's Hall"),
('date', date),
('headline', headline),
('url', url),
])
def make_row(date_string, headline, url, venue):
date = parse_date(date_string)
L.debug("Parsed date '{}' as {}".format(date_string, date))
return OrderedDict([
('organiser', 'Philosophy In Pubs'),
('venue', venue),
('date', date),
('headline', headline),
('url', url)
])
def post(self, key):
post = self.request.POST
form_data = dict((k, post.get(k, ''))
for k in ('title', 'author', 'date', 'body'))
template_dict = {'form_data': form_data, 'key': key, 'show_form' : True}
if 'delete_article' in post:
try:
NewsArticle.get(Key(key)).delete()
except datastore_errors.Error:
template_dict['message'] = \
'Could not delete article with key %r.' % key
else:
template_dict['message'] = 'Article deleted.'
template_dict['show_form'] = False
else:
try:
date = utils.parse_date(form_data['date'])
except ValueError:
template_dict['message'] = \
'Date is not in the correct format (YYYY-MM-DD).'
else:
if key == 'new':
try:
article = NewsArticle(title=form_data['title'],
author=form_data['author'],
date=date,
body=form_data['body'])
article.put()
except datastore_errors.Error:
template_dict['message'] = \
'Could not create new article.'
else:
template_dict['message'] = 'Article created.'
template_dict['show_form'] = False
else:
try:
article = NewsArticle.get(Key(key))
except BadKeyError:
template_dict['message'] = \
'Could not find article with key %r.' % key
else:
article.title = form_data['title']
article.author = form_data['author']
article.date = date
article.body = form_data['body']
try:
article.put()
except datastore_errors.Error:
template_dict['message'] = \
'Could not save changes to article.'
else:
template_dict['form_data'] = article
template_dict['message'] = 'Changes saved.'
self.render_template('edit', template_dict)
def post(self):
user_id = self.get_secure_cookie("user")
taskname = self.get_argument("taskname", "")
description = self.get_argument("description", "")
date = parse_date(self.get_argument("datetime", ""))
if taskname.strip():
newitem = TaskItem(user_id=user_id,
taskname=taskname,
datetime_due=date,
description=description)
self.db.create(newitem)
self.redirect("/list")
def parse_disturbances(self):
"""
For a given trendline, for each segment between vertices, determine
the stats for each "disturbance" and return a list of Disturbance
objects
"""
import utils
it = iter(self.points)
left_vertex = it.next()
for p in it:
if not p.vertex:
continue
start_yr = utils.parse_date(left_vertex.index_date).year
end_yr = utils.parse_date(p.index_date).year
yield Disturbance(
start_yr,
left_vertex.val_fit,
left_vertex.val_fit - p.val_fit, # TODO %?
end_yr - start_yr
)
left_vertex = p
def run():
if len(sys.argv) != 3:
print("Usage:")
print("python retire.py bioguideID termEndDate")
sys.exit()
try:
utils.parse_date(sys.argv[2])
except:
print("Invalid date: ", sys.argv[2])
sys.exit()
print("Loading current YAML...")
y = utils.load_data("legislators-current.yaml")
print("Loading historical YAML...")
y1 = utils.load_data("legislators-historical.yaml")
for moc in y:
if moc["id"].get("bioguide", None) != sys.argv[1]: continue
print("Updating:")
rtyaml.pprint(moc["id"])
print()
rtyaml.pprint(moc["name"])
print()
rtyaml.pprint(moc["terms"][-1])
moc["terms"][-1]["end"] = sys.argv[2]
y.remove(moc)
y1.append(moc)
break
print("Saving changes...")
utils.save_data(y, "legislators-current.yaml")
utils.save_data(y1, "legislators-historical.yaml")
def food(date, breakfast=None, lunch=None, dinner=None, snacks=None):
try:
date = utils.parse_date(date)
except ValueError:
return 'Error: unable to parse date %s' % date
db = utils.read_db()
initial = utils.daily_points(date, db)
messages = ['%s had %s points' % (date, initial)]
messages.extend(process_meal(breakfast, db, date, 'breakfast'))
messages.extend(process_meal(lunch, db, date, 'lunch'))
messages.extend(process_meal(dinner, db, date, 'dinner'))
messages.extend(process_meal(snacks, db, date, 'snacks'))
final = utils.daily_points(date, db)
diff = final - initial
messages.append('%s added %s points (now has %s)' % (date, diff, final))
utils.write_db(db)
return '\n'.join(messages)
def prehab(date=None, home=None, work=None, travel=None, other=None):
try:
date = utils.parse_date(date)
except ValueError:
return 'Error: unable to parse date %s' % date
db = utils.read_db()
initial = utils.daily_points(date, db)
messages = ['%s had %s points' % (date, initial)]
messages.extend(process_loc(home, db, date, 'home'))
messages.extend(process_loc(work, db, date, 'work'))
messages.extend(process_loc(travel, db, date, 'travel'))
messages.extend(process_loc(other, db, date, 'other'))
final = utils.daily_points(date, db)
diff = final - initial
messages.append('%s added %s points (now has %s)' % (date, diff, final))
utils.write_db(db)
return '\n'.join(messages)
def get_all_listings(listings_fobj):
lxml_root = lxml.html.fromstring(listings_fobj.read())
divs = lxml_root.xpath('//div[@id="events_list"]/div[@class="item"]')
assert len(divs) > 0
earliest_date = None
for div in divs:
headline = clean_headline(
div.xpath('./h4/a')[0].text_content().strip())
url = BASE_URL + div.xpath('./h4/a/@href')[0]
date_tag = div.xpath('./span[@class="from"]')[0]
date_string = extract_date(date_tag.text_content())
if not earliest_date:
earliest_date = parse_date(date_string)
yield make_row(date_string, earliest_date, headline, url)
def analysis_reducer(self, point_wkt, pix_datas):
"""
Given a point wkt and a list of pix datas in the format:
[
{'date': '2011-09-01', 'val': 160.0},
{'date': '2012-09-01', 'val': 180.0},
...
]
perform the landtrendr analysis and change labeling.
Yields out the change labels and trendline data for the given point
"""
sys.stdout.write('.') # for viewing progress
sys.stdout.flush()
job = os.environ.get('LT_JOB')
settings = utils.get_settings(job)
pix_datas = list(pix_datas) # save iterator to a list
pix_trendline = utils.analyze(
pix_datas,
settings['line_cost'],
utils.parse_date(settings['target_date'])
)
# write out pix trendline
for label, val in pix_trendline.mr_label_output().iteritems():
# prepend 'aux/' to label name so written to sub folder
yield (
'trendline/%s' % label,
{'pix_ctr_wkt': point_wkt, 'value': val}
)
label_rules = [
classes.LabelRule(lr) for lr in settings['label_rules']
]
change_labels = utils.change_labeling(pix_trendline, label_rules)
# write out change labels
for label_name, data in change_labels.iteritems():
for key in ['class_val', 'onset_year', 'magnitude', 'duration']:
label_key = '%s_%s' % (label_name, key)
yield label_key, {'pix_ctr_wkt': point_wkt, 'value': data[key]}
def parse_template_6(self, element):
workshop = {}
title = rex.rex(element[1], r'(.*)Edited\s*by.*', re.I | re.S).group(1)
workshop['volume_number'] = WorkshopSummaryParser.extract_volume_number(element[0].get('href'))
workshop['label'] = element[0].text.replace('.', '')
workshop['url'] = element[0].get('href')
workshop['time'] = utils.parse_date(title)
try:
workshop['edition'] = tonumber(
rex.rex(title,
r'.*Proceedings(\s*of)?(\s*the)?\s*(\d{1,}|first|second|third|forth|fourth|fifth)[thrd]*'
r'.*Workshop.*',
re.I, default=None).group(3))
except:
#'edition' property is optional
pass
self.add_workshop(workshop)
def post(self):
post = self.request.POST
if post['kind'] == 'badge':
badge = Badge(
name=post['name'],
description=post['description'],
category=post['category'],
image=post['image'],
value=int(post['value'])
)
badge.save()
elif post['kind'] == 'award':
badge = Badge.get_by_id(int(post['badge']))
for member in post.getall('members'):
member = Member.get_by_id(int(member))
award = Award(
member=member,
badge=badge,
date=datetime.date.today(),
proof=post['proof']
)
award.save()
member.score += badge.value
member.save()
elif post['kind'] == 'talk':
talk = Talk(
title=post['title'],
date=utils.parse_date(post['date']),
description=post['description'],
member=Member.get_by_id(int(post['member'])),
video=post['video']
)
talk.put()
elif post['kind'] == 'taglineform':
properties = GeneralSiteProperties.all().get()
if properties == None:
properties = GeneralSiteProperties(tag_line=post['tagline'])
properties.put()
else:
properties.tag_line = post['tagline']
properties.put()
self.get()
def parse_template_4(self, element):
"""
A template for joint proceedings of three workshops.
Examples:
- http://ceur-ws.org/Vol-981/
- http://ceur-ws.org/Vol-862/
- http://ceur-ws.org/Vol-853/
"""
workshop_1 = {'id': 1}
workshop_2 = {'id': 2}
workshop_3 = {'id': 3}
summary = self.rex(element[1], [
r'(joint\s+proceedings\s+of\s+[the]*.*workshops:\s*([\s\w]+)\(([a-zA-Z]+)\d+\)'
r'[and,\s]+([\s\w]+)\(([a-zA-Z]+)\d+\)[and,\s]+([\s\w]+)\(([a-zA-Z]+)\d+\)[,\s]+.*)Edited by.*',
r"(joint\s+proceedings\s+of\s+([\s\w,]+)\(([a-zA-Z]+)['\s]?\d+\)[and,\s]+([\s\w-]+)\(([a-zA-Z]+)['\s]?\d+\)"
r"[and,\s]+([\s\w]+)\(([a-zA-Z]+)['\s]?\d+\)[,\s]+.*)Edited by.*"
],
re.I | re.S)
if len(summary.groups()) != 7:
raise DataNotFound()
title = summary.group(1)
workshop_1['volume_number'] = workshop_2['volume_number'] = workshop_3['volume_number'] = \
WorkshopSummaryParser.extract_volume_number(element[0].get('href'))
workshop_1['url'] = workshop_2['url'] = workshop_3['url'] = element[0].get('href')
workshop_1['time'] = workshop_2['time'] = workshop_3['time'] = utils.parse_date(title)
workshop_1['label'] = summary.group(2)
workshop_1['short_label'] = summary.group(3)
workshop_2['label'] = summary.group(4)
workshop_2['short_label'] = summary.group(5)
workshop_3['label'] = summary.group(6)
workshop_3['short_label'] = summary.group(7)
self.add_workshop(workshop_1)
self.add_workshop(workshop_2)
self.add_workshop(workshop_3)
def process(html_fobj):
lxml_root = lxml.html.fromstring(html_fobj.read())
divs = lxml_root.xpath('//div[@class="gif_box"]')
assert len(divs) > 0
earliest_date = None
for div in divs:
headline = clean_headline(div.xpath(
'./div[@class="gif_txt"]/a')[0].text_content().strip())
url = div.xpath(
'./div/a/@href')[0]
date_string = div.xpath(
'./div[@class="listdate"]')[0].text_content()
if not earliest_date:
earliest_date = parse_date(date_string)
yield make_row(date_string, earliest_date, headline, url)
def command_received(self, cmd_data):
LOG_MSG('Poll command got response. Response: {0}.'.format(cmd_data))
for cmd in cmd_data:
# Obtain only new commands next time
cmd_date = parse_date(cmd['timestamp'])
if self.owner.timestamp is not None:
self.owner.timestamp = max(self.owner.timestamp, cmd_date)
else:
self.owner.timestamp = cmd_date
# device-application will use this deferred object to notify me about the command progress.
thiscmd = cmd
def ok(result):
self.command_done(thiscmd, result)
def err(reason):
self.command_failed(thiscmd, reason)
defer = Deferred()
defer.addCallbacks(ok, err)
try:
LOG_MSG('Executing command {0} handler.'.format(cmd))
self.owner.run_command(cmd, defer)
except Exception, err:
LOG_ERR('Failed to execute device-delegate on_command. Reason: <{0}>.'.format(err))
self.command_failed(cmd, err)