Пример #1
0
    def process(self):
        """process meetings"""
        parser = etree.XMLParser(recover=True)
        r = requests.get(self.timerange_url)
        xml = r.text.encode('ascii','xmlcharrefreplace') 
        root = etree.fromstring(xml, parser=parser)

        for item in root[1].iterchildren():
            meeting = {}
            for e in item.iterchildren():
                meeting[e.tag] = e.text

            meeting['start_date'] = parse_date(meeting['sisbvcs'])
            meeting['end_date'] = parse_date(meeting['sisevcs'])
            meeting['tz_start_date'] = parse_date(meeting['sisbvcs'], tzinfo=utc).astimezone(self.tzinfo)
            meeting['tz_end_date'] = parse_date(meeting['sisevcs'], tzinfo=utc).astimezone(self.tzinfo)
            silfdnr = meeting['silfdnr']
            try:
                result = self.process_agenda(silfdnr)
                meeting.update(result)
            except Exception, e:
                meeting['ERROR'] = True
                print e
            meeting['_id'] = int(meeting['silfdnr'])
            self.db.meetings.save(meeting)
def validate_no_bigamy(fams, indis):
    ret_data = []

    for iid in indis:
        marriages = []
        for fid in indis[iid]['FAMS']:
            if any(fams[fid][tag] is None for tag in ['HUSB', 'WIFE', 'MARR']):
                continue

            begin_date = utils.parse_date(fams[fid]['MARR'])
            end_date = None
            if indis[fams[fid]['HUSB']]['DEAT'] is not None:
                deat = utils.parse_date(indis[fams[fid]['HUSB']]['DEAT'])
                end_date = (deat if end_date is None else min(end_date, deat))
            if indis[fams[fid]['WIFE']]['DEAT'] is not None:
                deat = utils.parse_date(indis[fams[fid]['WIFE']]['DEAT'])
                end_date = (deat if end_date is None else min(end_date, deat))
            if fams[fid]['DIV'] is not None:
                div = utils.parse_date(fams[fid]['DIV'])
                end_date = (div if end_date is None else min(end_date, div))

            marriages.append((begin_date, end_date))
        marriages.sort()
        for i in range(len(marriages) - 1):
            int1 = marriages[i]
            int2 = marriages[i + 1]

            if utils.interval_intersect(int1, int2):
                ret_data.append((
                    iid,
                    f'Individual id={iid} was in two or more marriages at the same time'
                ))
                break

    return ret_data
Пример #3
0
def history(begin=None, end=None, week=False, month=False):
    if week or month:
        end = datetime.datetime.now()
        if week:
            begin = end - datetime.timedelta(days=7)
        if month:
            begin = end - datetime.timedelta(days=28)
    else:
        try:
            begin = utils.parse_date(begin, dtime=True)
        except ValueError:
            return 'Error: unable to parse date %s' % begin
        try:
            end = utils.parse_date(end, dtime=True)
        except ValueError:
            return 'Error: unable to parse date %s' % end
    cd = utils.compose_date
    if not begin <= end:
        return 'Error: %s is not before %s' % (cd(begin), cd(end))
    db = utils.read_db()
    days = [
        utils.compose_date(begin + datetime.timedelta(days=x))
        for x in xrange(0, (end - begin).days)
    ]
    matches = ['from: %s to: %s' % (cd(begin), cd(end))]
    for day in days:
        matches.append('%s: %s' % (day, utils.daily_points(day, db)))

    return '\n'.join(matches)
def validate_birth_before_marriage(fams, indis):
    return_data = []

    for fid in fams:
        if fams[fid]['MARR'] is not None:
            marriage_day = utils.parse_date(fams[fid]['MARR'])

            husband_id = fams[fid]['HUSB']
            wife_id = fams[fid]['WIFE']

            husband_birth = indis[husband_id]['BIRT']
            wife_birth = indis[wife_id]['BIRT']

            if husband_birth is not None:
                husband_birthday = utils.parse_date(husband_birth)

            if wife_birth is not None:
                wife_birthday = utils.parse_date(wife_birth)

            if marriage_day < husband_birthday:
                return_data.append((
                    husband_id,
                    f'Husband id={husband_id} in family id={fid} has marriage before birth.'
                ))

            if marriage_day < wife_birthday:
                return_data.append((
                    wife_id,
                    f'Wife id={wife_id} in family id={fid} has marriage before birth.'
                ))

    return return_data
def validate_marriage_after_fourteen(fams, indis):
    invalid_marriages = []

    for fid in fams:
        # if married
        if fams[fid]['MARR'] is not None:
            marriage_date = utils.parse_date(fams[fid]['MARR'])

            husbandID, wifeID = fams[fid]['HUSB'], fams[fid]['WIFE']
            husband_birth, wife_birth = indis[husbandID]['BIRT'], indis[
                wifeID]['BIRT']

            ages = []
            if husband_birth is not None:
                husband_birth = utils.parse_date(husband_birth)
                husband_marriage_age = utils.get_age(husband_birth,
                                                     marriage_date)
                ages.append(husband_marriage_age)

            if wife_birth is not None:
                wife_birth = utils.parse_date(wife_birth)
                wife_marriage_age = utils.get_age(wife_birth, marriage_date)
                ages.append(wife_marriage_age)

            if len(ages) > 0 and min(ages) < 14:
                invalid_marriages.append(
                    (fid, f'Family id={fid} has marriage before age 14'))

        return invalid_marriages
def validate_divorce_before_death(fams, indis):
    ret_data = []

    for fid in fams:
        if fams[fid]['DIV'] is not None:
            divorce_date = utils.parse_date(fams[fid]['DIV'])
            husband_id = fams[fid]['HUSB']
            wife_id = fams[fid]['WIFE']

            if indis[husband_id]['DEAT'] is not None:
                death_date = utils.parse_date(indis[husband_id]['DEAT'])
                if death_date < divorce_date:
                    ret_data.append((
                        husband_id,
                        f'Individual id={husband_id} has a divorce after his death'
                    ))
            if indis[wife_id]['DEAT'] is not None:
                death_date = utils.parse_date(indis[wife_id]['DEAT'])
                if death_date < divorce_date:
                    ret_data.append((
                        wife_id,
                        f'Individual id={wife_id} has a divorce after her death'
                    ))

    return ret_data
Пример #7
0
    def process(self):
        """process meetings"""
        parser = etree.XMLParser(recover=True)
        r = requests.get(self.timerange_url)
        xml = r.text.encode('ascii','xmlcharrefreplace') 
        root = etree.fromstring(xml, parser=parser)

        start_at = root.find("list")
        for item in start_at.iterchildren():
            meeting = {}
            for e in item.iterchildren():
                meeting[e.tag] = e.text

            meeting['start_date'] = parse_date(meeting['sisbvcs'])
            meeting['end_date'] = parse_date(meeting['sisevcs'])
            meeting['tz_start_date'] = parse_date(meeting['sisbvcs'], tzinfo=utc).astimezone(self.tzinfo)
            meeting['tz_end_date'] = parse_date(meeting['sisevcs'], tzinfo=utc).astimezone(self.tzinfo)
            silfdnr = meeting['silfdnr']
            print "processing meeting %s" %silfdnr
            try:
                result = self.process_agenda(silfdnr)
                meeting.update(result)
            except Exception, e:
                meeting['ERROR'] = True
                print "exception when trying to parse meeting %s" %silfdnr
                print e
            meeting['meeting_id'] = str(meeting['silfdnr'])
            meeting['_id'] = "%s:%s" %(self.city, silfdnr)
            meeting['city'] = self.city
            self.db.meetings.save(meeting)
Пример #8
0
def history(begin=None, end=None, week=False, month=False):
    if week or month:
        end = datetime.datetime.now()
        if week:
            begin = end - datetime.timedelta(days=7)
        if month:
            begin = end - datetime.timedelta(days=28)
    else:
        try:
            begin = utils.parse_date(begin, dtime=True)
        except ValueError:
            return 'Error: unable to parse date %s' % begin
        try:
            end = utils.parse_date(end, dtime=True)
        except ValueError:
            return 'Error: unable to parse date %s' % end
    cd = utils.compose_date
    if not begin <= end:
        return 'Error: %s is not before %s' % (cd(begin), cd(end))
    db = utils.read_db()
    days = [utils.compose_date(begin + datetime.timedelta(days=x)) for x in xrange(0,
        (end-begin).days)]
    matches = ['from: %s to: %s' % (cd(begin), cd(end))]
    for day in days:
        matches.append('%s: %s' % (day, utils.daily_points(day, db)))

    return '\n'.join(matches)
Пример #9
0
def events():
    try:
        args = parse_url_arguments(request.url)
        start_date = parse_date(args["start_date"])
        end_date = parse_date(args["end_date"])
        classification = args["classification"]
        return get_events(start_date, end_date, classification)
    except Exception as ex:
        return str(ex)
Пример #10
0
def tweets(uid,min_date=None,max_date=None,limit=100):
    "returns all the tweets for a user between min_date and max_date"
    start = parse_date(min_date)
    end = parse_date(max_date)
    limit = int(limit) if limit else None
    tweets = Tweet.find(
        (Tweet.user_id==int(uid)) &
        (Tweet.created_at.range(start,end)),
        limit=limit,
        sort=Tweet._id)
    return [t.to_d() for t in tweets]
Пример #11
0
def trim_and_sort_events(events):
    """Sorts given events by event date, and
  trims those that are past today.
  """
    events = [
        e for e in events \
          if parse_date(e['event_date']) \
            >= utc.localize(datetime.now())
      ]

    events = sorted(events, key=lambda x: parse_date(x['event_date']))
    return events
Пример #12
0
 def __init__(self, timestamp, kwargs):
     if isinstance(timestamp, str):
         self.timestamp = parse_date(timestamp)
     elif isinstance(timestamp, unicode):
         self.timestamp = parse_date(timestamp)
     elif isinstance(timestamp, datetime):
         self.timestamp = timestamp
     self.id = base64.b64decode(self.timestamp)
     self.urls = {}
     self.report_name, self.report_number = get_report_period(
         self.timestamp)
     for name, url in kwargs.items():
         if url:
             self.urls[name] = URL(self.id, name, url)
Пример #13
0
def validate_birth_before_death(fams, indis):
    return_data = []

    for cid in indis:
        if indis[cid]['BIRT'] is not None:
            birthday = utils.parse_date(indis[cid]['BIRT'])

            if indis[cid]['DEAT'] is not None:
                death_day = utils.parse_date(indis[cid]['DEAT'])

                if death_day < birthday:
                    return_data.append(
                        (cid, f'Person id={cid} has death before birth.'))

    return return_data
    def parse_template_5(self, element):
        """
        A template for a workshop with the conference acronym and year in the name

        Examples:
            - http://ceur-ws.org/Vol-958/
        """
        workshop = {}
        title = rex.rex(element[1], r'(.*)Edited\s*by.*', re.I | re.S).group(1)

        workshop['volume_number'] = WorkshopSummaryParser.extract_volume_number(element[0].get('href'))
        label_part = rex.rex(element[0].text, r'(.*)\sat\s(\w{2,})\s(\d{4})[\s\.]*', re.I | re.S)
        workshop['label'] = label_part.group(1)
        workshop['conf_acronym'] = label_part.group(2)
        workshop['conf_year'] = label_part.group(3)
        workshop['url'] = element[0].get('href')
        workshop['time'] = utils.parse_date(title)
        try:
            workshop['edition'] = tonumber(
                rex.rex(title,
                        r'.*Proceedings(\s*of)?(\s*the)?\s*(\d{1,}|first|second|third|forth|fourth|fifth)[thrd]*'
                        r'.*Workshop.*',
                        re.I, default=None).group(3))
        except:
            #'edition' property is optional
            pass

        self.add_workshop(workshop)
Пример #15
0
def validate_no_sextuples(fams, indis):
    one_day = timedelta(days=1)

    ret_data = []
    for fid in fams:
        child_births = [
            indis[cid]['BIRT'] for cid in fams[fid]['CHIL']
            if indis[cid]['BIRT'] is not None
        ]
        if len(child_births) <= 5:
            continue

        birth_freq = {}
        for birth in child_births:
            if birth not in birth_freq:
                birth_freq[birth] = 0
            birth_freq[birth] += 1

        dates = sorted(birth_freq.keys())
        parsed_dates = [utils.parse_date(d) for d in dates]
        for i in range(len(dates)):
            count = birth_freq[dates[i]]
            if i + 1 < len(dates) and parsed_dates[
                    i + 1] - parsed_dates[i] == one_day:
                count = birth_freq[dates[i]] + birth_freq[dates[i + 1]]

            if count > 5:
                ret_data.append((
                    fid,
                    f'Family id={fid} has more than 5 children born together'))
                break

    return ret_data
    def parse_template_2(self, element):
        """
        A template for joint proceedings of two workshops:

        Examples:
            - http://ceur-ws.org/Vol-776/
        """
        workshop_1 = {'id': 1}
        workshop_2 = {'id': 2}
        summary = rex.rex(element[1], r'^\s*(proceedings\s+of\s+joint.*on.*\((\w+)\-(\w+)\s+\d+\).*)Edited by.*',
                          re.I | re.S)

        if len(summary.groups()) != 3:
            raise DataNotFound()

        title = summary.group(1)

        workshop_1['volume_number'] = workshop_2['volume_number'] = \
            WorkshopSummaryParser.extract_volume_number(element[0].get('href'))
        workshop_1['url'] = workshop_2['url'] = element[0].get('href')
        workshop_1['time'] = workshop_2['time'] = utils.parse_date(title)

        workshop_1['short_label'] = summary.group(2)
        workshop_2['short_label'] = summary.group(3)

        self.add_workshop(workshop_1)
        self.add_workshop(workshop_2)
Пример #17
0
    def parse_template_3(self, element):
        """
        A template for joint proceedings of two workshops.

        Examples:
            - http://ceur-ws.org/Vol-1098/
            - http://ceur-ws.org/Vol-989/
        """
        workshop_1 = {'id': 1}
        workshop_2 = {'id': 2}
        summary = self.rex(element[1], [
            r"(joint\s+proceedings\s+of\s+([\s\w,]+)\(([a-zA-Z]+)['\s]?\d+\)[and,\s]+"
            r"([:\s\w-]+)\(([a-zA-Z]+)['\s]?\d+\)([\w\s\-.,^\(]*|[,\s]+workshops\s+of.*|[,\s]+co-located.*))Edited by.*",

            r"(proceedings\s+of\s+joint([\s\w,]+)\(([a-zA-Z]+)['\s]?\d{0,4}\)[and,\s]+"
            r"([:,\s\w-]+)\(([a-zA-Z]+)['\s]?\d{0,4}\)([\w\s\-.,^\(]*|[,\s]+workshops\s+of.*|[,\s]+co-located.*))Edited by.*"
        ], re.I | re.S)

        if len(summary.groups()) != 6:
            raise DataNotFound()

        title = summary.group(1)

        workshop_1['volume_number'] = workshop_2['volume_number'] = \
            WorkshopSummaryParser.extract_volume_number(element[0].get('href'))
        workshop_1['url'] = workshop_2['url'] = element[0].get('href')
        workshop_1['time'] = workshop_2['time'] = utils.parse_date(title)

        workshop_1['label'] = summary.group(2)
        workshop_1['short_label'] = summary.group(3)
        workshop_2['label'] = summary.group(4)
        workshop_2['short_label'] = summary.group(5)

        self.add_workshop(workshop_1)
        self.add_workshop(workshop_2)
Пример #18
0
    def __init__(self, logger, sequences, reference, dateFormat):
        super(sequence_set, self).__init__()
        self.log = logger

        # load sequences from the (parsed) JSON - don't forget to sort out dates
        self.seqs = {}
        for name, data in sequences.iteritems():
            self.seqs[name] = SeqRecord(Seq(data["seq"], generic_dna),
                   id=name, name=name, description=name)
            self.seqs[name].attributes = data["attributes"]
            # tidy up dates
            date_struc = parse_date(self.seqs[name].attributes["raw_date"], dateFormat)
            self.seqs[name].attributes["num_date"] = date_struc[1]
            self.seqs[name].attributes["date"] = date_struc[2]

        # if the reference is to be analysed it'll already be in the (filtered & subsampled)
        # sequences, so no need to add it here, and no need to care about attributes etc
        # we do, however, need it for alignment
        self.reference_in_dataset = reference["included"]
        name = reference["strain"]
        self.reference_seq = SeqRecord(Seq(reference["seq"], generic_dna),
               id=name, name=name, description=name)
        if "genes" in reference and len(reference["genes"]):
            self.proteins = {k:FeatureLocation(start=v["start"], end=v["end"], strand=v["strand"]) for k, v in reference["genes"].iteritems()}
        else:
            self.proteins = None

        # other things:
        self.run_dir = '_'.join(['temp', time.strftime('%Y%m%d-%H%M%S',time.gmtime()), str(random.randint(0,1000000))])
        self.nthreads = 2 # should load from config file
Пример #19
0
 def convert_trait_to_numerical_date(self, trait, dateFormat):
     for name, seq in self.seqs.iteritems():
         try:
             date_struc = parse_date(seq.attributes[trait], dateFormat)
             seq.attributes[trait] = date_struc[1]
         except KeyError:
             self.log.warn("Attribute {} not found for sequence {}. Ignoring".format(trait, seq.name))
    def parse_template_3(self, element):
        """
        A template for joint proceedings of two workshops.

        Examples:
            - http://ceur-ws.org/Vol-1098/
            - http://ceur-ws.org/Vol-989/
        """
        workshop_1 = {'id': 1}
        workshop_2 = {'id': 2}
        summary = self.rex(element[1], [
            r"(joint\s+proceedings\s+of\s+([\s\w,]+)\(([a-zA-Z]+)['\s]?\d+\)[and,\s]+"
            r"([:\s\w-]+)\(([a-zA-Z]+)['\s]?\d+\)([\w\s\-.,^\(]*|[,\s]+workshops\s+of.*|[,\s]+co-located.*))Edited by.*",

            r"(proceedings\s+of\s+joint([\s\w,]+)\(([a-zA-Z]+)['\s]?\d{0,4}\)[and,\s]+"
            r"([:,\s\w-]+)\(([a-zA-Z]+)['\s]?\d{0,4}\)([\w\s\-.,^\(]*|[,\s]+workshops\s+of.*|[,\s]+co-located.*))Edited by.*"
        ], re.I | re.S)

        if len(summary.groups()) != 6:
            raise DataNotFound()

        title = summary.group(1)

        workshop_1['volume_number'] = workshop_2['volume_number'] = \
            WorkshopSummaryParser.extract_volume_number(element[0].get('href'))
        workshop_1['url'] = workshop_2['url'] = element[0].get('href')
        workshop_1['time'] = workshop_2['time'] = utils.parse_date(title)

        workshop_1['label'] = summary.group(2)
        workshop_1['short_label'] = summary.group(3)
        workshop_2['label'] = summary.group(4)
        workshop_2['short_label'] = summary.group(5)

        self.add_workshop(workshop_1)
        self.add_workshop(workshop_2)
def map_video(config):
    programId = config.get('programId')
    kind = config.get('kind')
    duration = int(config.get('duration') or 0) * \
        60 or config.get('durationSeconds')
    airdate = config.get('broadcastBegin')
    if airdate is not None:
        airdate = str(utils.parse_date(airdate))

    return {
        'label': utils.format_title_and_subtitle(config.get('title'), config.get('subtitle')),
        'path': plugin.url_for('play', kind=kind, program_id=programId),
        'thumbnail': config.get('imageUrl'),
        'is_playable': True,
        'info_type': 'video',
        'info': {
            'title': config.get('title'),
            'duration': duration,
            'genre': config.get('genrePresse'),
            'plot': config.get('shortDescription') or config.get('fullDescription'),
            'plotoutline': config.get('teaserText'),
            # year is not correctly used by kodi :(
            # the aired year will be used by kodi for production year :(
            # 'year': int(config.get('productionYear')),
            'country': [country.get('label') for country in config.get('productionCountries', [])],
            'director': config.get('director'),
            'aired': airdate
        },
        'properties': {
            'fanart_image': config.get('imageUrl'),
        }
    }
Пример #22
0
    def parse_standard_rss(self, url):
        headers = self.set_headers()

        req = urllib.request.Request(url, headers=headers)
        parse_xml_url = urllib.request.urlopen(req)

        xml_page = parse_xml_url.read()
        parse_xml_url.close()

        soup_page = BeautifulSoup(xml_page, "lxml")
        channel = soup_page.find("channel")
        news_list = channel.findAll("item")

        links = []
        for getfeed in news_list:
            titolo = getfeed.title.text

            description = ""
            if getfeed.description.text:
                description = getfeed.description.text

            link_id = generate_link_id(titolo)

            links.append({
                'id': link_id,
                'titolo': titolo,
                'text': description,
                'url': getfeed.link.nextSibling.rstrip(),
                'data': parse_date(getfeed.pubdate.text)
            })

            if JUST_ONE_LINK:
                break

        return links
Пример #23
0
    def parse_template_5(self, element):
        """
        A template for a workshop with the conference acronym and year in the name

        Examples:
            - http://ceur-ws.org/Vol-958/
        """
        workshop = {}
        title = rex.rex(element[1], r'(.*)Edited\s*by.*', re.I | re.S).group(1)

        workshop['volume_number'] = WorkshopSummaryParser.extract_volume_number(element[0].get('href'))
        label_part = rex.rex(element[0].text, r'(.*)\sat\s(\w{2,})\s(\d{4})[\s\.]*', re.I | re.S)
        workshop['label'] = label_part.group(1)
        workshop['conf_acronym'] = label_part.group(2)
        workshop['conf_year'] = label_part.group(3)
        workshop['url'] = element[0].get('href')
        workshop['time'] = utils.parse_date(title)
        try:
            workshop['edition'] = tonumber(
                rex.rex(title,
                        r'.*Proceedings(\s*of)?(\s*the)?\s*(\d{1,}|first|second|third|forth|fourth|fifth)[thrd]*'
                        r'.*Workshop.*',
                        re.I, default=None).group(3))
        except:
            #'edition' property is optional
            pass

        self.add_workshop(workshop)
Пример #24
0
    def parse_template_2(self, element):
        """
        A template for joint proceedings of two workshops:

        Examples:
            - http://ceur-ws.org/Vol-776/
        """
        workshop_1 = {'id': 1}
        workshop_2 = {'id': 2}
        summary = rex.rex(element[1], r'^\s*(proceedings\s+of\s+joint.*on.*\((\w+)\-(\w+)\s+\d+\).*)Edited by.*',
                          re.I | re.S)

        if len(summary.groups()) != 3:
            raise DataNotFound()

        title = summary.group(1)

        workshop_1['volume_number'] = workshop_2['volume_number'] = \
            WorkshopSummaryParser.extract_volume_number(element[0].get('href'))
        workshop_1['url'] = workshop_2['url'] = element[0].get('href')
        workshop_1['time'] = workshop_2['time'] = utils.parse_date(title)

        workshop_1['short_label'] = summary.group(2)
        workshop_2['short_label'] = summary.group(3)

        self.add_workshop(workshop_1)
        self.add_workshop(workshop_2)
Пример #25
0
    def command_received(self, cmd_data):
        LOG_MSG('Poll command got response. Response: {0}.'.format(cmd_data))
        for cmd in cmd_data:
            # Obtain only new commands next time
            cmd_date = parse_date(cmd['timestamp'])
            if self.owner.timestamp is not None:
                self.owner.timestamp = max(self.owner.timestamp, cmd_date)
            else:
                self.owner.timestamp = cmd_date
            # device-application will use this deferred object to notify me about the command progress.
            thiscmd = cmd

            def ok(result):
                self.command_done(thiscmd, result)

            def err(reason):
                self.command_failed(thiscmd, reason)

            defer = Deferred()
            defer.addCallbacks(ok, err)
            try:
                LOG_MSG('Executing command {0} handler.'.format(cmd))
                self.owner.run_command(cmd, defer)
            except Exception, err:
                LOG_ERR(
                    'Failed to execute device-delegate on_command. Reason: <{0}>.'
                    .format(err))
                self.command_failed(cmd, err)
Пример #26
0
def parse_data(data):
    final_data = {}

    for year, call_log in data.items():
        print(f'Getting call data for {year}')
        # UNCOMMENT THIS FOR CREATING A YEAR KEY FOR EACH YEAR (1 OF 3)
        # final_data[year] = {}
        for month, calls in call_log.items():
            # UNCOMMENT THIS FOR CREATING A MONTH KEY FOR EACH MONTH (2 OF 3)
            # month = parse_month(month=month, abbreviated=False)
            # final_data[year][month] = {}
            for call in calls:
                call_data = call.split("-")
                call_date = utils.parse_date(call_data[0].split("_")[0])
                call_time = utils.parse_time(call_data[0].split("_")[1])
                call_number = utils.parse_call_number(call_data[1])
                call_direction = call_data[2].split(".")[0]
                # UNCOMMENT THIS FOR CREATING A MONTH KEY FOR EACH MONTH (3 OF 3)
                # final_data[year][month].setdefault(call_date, {})
                # final_data[year][month][call_date].setdefault(call_direction, [])
                # final_data[year][month][call_date][call_direction].append(f'{call_time} - {call_number}')
                final_data.setdefault(call_date, [])
                final_data[call_date].append(
                    f'{call_direction} - {call_time} - {call_number}')

    return final_data
Пример #27
0
def gen_indi_table(fams, indis):
    indi_table = PrettyTable()
    indi_table.field_names = [
        'ID', 'Name', 'Gender', 'Birthday', 'Age', 'Alive', 'Death', 'Child',
        'Spouse'
    ]
    for indi_id in sorted(indis.keys()):
        indi_data = indis[indi_id]

        children = []
        spouse = None
        for fam_id in indi_data['FAMS']:
            children += fams[fam_id]['CHIL']
            if fams[fam_id]['DIV'] is None:
                if indi_data['SEX'] == 'M':
                    spouse = fams[fam_id]['WIFE']
                else:
                    spouse = fams[fam_id]['HUSB']

        age = None
        if indi_data['BIRT'] is not None:
            birthday = utils.parse_date(indi_data['BIRT'])
            age = utils.get_age(birthday)
            if age < 0:
                age = None
        alive = indi_data['DEAT'] is None
        indi_table.add_row([
            indi_id, indi_data['NAME'] or 'NA', indi_data['SEX'] or 'NA',
            indi_data['BIRT'] or 'NA', age or 'NA', alive, indi_data['DEAT']
            or 'NA',
            utils.format_list(children) if len(children) > 0 else 'NA', spouse
            or 'NA'
        ])
    return indi_table
Пример #28
0
def lunar_phase():
    try:
        args = parse_url_arguments(request.url)
        date = parse_date(args["date"])
        return get_lunar_phase(date)
    except Exception as ex:
        return str(ex)
Пример #29
0
def map_video(config):
    programId = config.get('programId')
    kind = config.get('kind')
    duration = int(config.get('duration') or 0) * \
        60 or config.get('durationSeconds')
    airdate = config.get('broadcastBegin')
    if airdate is not None:
        airdate = str(utils.parse_date(airdate))

    return {
        'label': utils.format_title_and_subtitle(config.get('title'), config.get('subtitle')),
        'path': plugin.url_for('play', kind=kind, program_id=programId),
        'thumbnail': config.get('imageUrl'),
        'is_playable': True,
        'info_type': 'video',
        'info': {
            'title': config.get('title'),
            'duration': duration,
            'genre': config.get('genrePresse'),
            'plot': config.get('shortDescription') or config.get('fullDescription'),
            'plotoutline': config.get('teaserText'),
            # year is not correctly used by kodi :(
            # the aired year will be used by kodi for production year :(
            #'year': int(config.get('productionYear')),
            'country': [country.get('label') for country in config.get('productionCountries', [])],
            'director': config.get('director'),
            'aired': airdate
        },
        'properties': {
            'fanart_image': config.get('imageUrl'),
        }
    }
Пример #30
0
    def get(self, args):
        """ Return the courses matching your query """

        # Parameters
        date_string = args['date']
        department = args['department']
        place = args['place']
        comment = args['comment']

        # Base query
        courses = Course.query

        # Apply filters
        if date_string:
            courses = courses.filter_by(date=parse_date(date_string))
        if place:
            courses = courses.filter_by(place=place)
        if department:
            courses = courses.filter_by(department=department)
        if comment:
            # Can be useful to select a group in a course
            courses = courses.filter_by(department=comment)

        # Do the query
        courses = courses.order_by(Course.time_begin).limit(1000).all()

        # Format it
        course_list = [course.__toJSON__() for course in courses]

        return course_list, 200
Пример #31
0
def validate_marriage_before_child(fams, indis):
    ret_data = []

    for fid in fams:
        if fams[fid]['MARR'] is not None:
            marriage = utils.parse_date(fams[fid]['MARR'])

            for cid in fams[fid]['CHIL']:
                if indis[cid]['BIRT'] is not None:
                    birth = utils.parse_date(indis[cid]['BIRT'])
                    if birth < marriage:
                        ret_data.append((
                            fid,
                            f'Child id={cid} has birthdate before marriage of parents'
                        ))

    return ret_data
Пример #32
0
def getData(request):
    from wrcc.wea_server.libwea.products.listers import getData
    error = require(request, ['stn', 'sD', 'eD'])
    if error:
        return ErrorResponse(error)

    stn = request.args.get('stn')
    sD = parse_date(request.args.get('sD'))
    eD = parse_date(request.args.get('eD'))
    units_system = request.args.get('units', 'N')  # N (native) units by default

    try:
        result = getData(stn, sD, eD, units_system=units_system)
    except IOError:
        return ErrorResponse("No data available.")

    return JsonResponse(result)
Пример #33
0
 def __init__(self, id, name, summary, date, url='', image=DEFAULT_COVER):
     self.id = id
     self.name = name
     self.summary = summary
     self.date = parse_date(date)
     self.url = url
     self.image = image
     self.episodes = []
Пример #34
0
def check_date(d, context):
    if not isinstance(d, str):
        error(context, str(d) + ": invalid data type")
        return None
    try:
        return utils.parse_date(d)
    except Exception as e:
        error(context, d + ": " + str(e))
        return None
Пример #35
0
def scrapp_movie_page(movie_link):

    movie = {}
    driver = utils.generate_webdriver(show=False)
    driver.get(config.MAIN_URL + movie_link)

    movie.update({'movie_id': int(driver.current_url.split('/')[-1])})
    movie.update({'title': driver.find_element_by_class_name('text').text})
    movie.update({'year': int(driver.find_element_by_class_name('year').text)})
    movie.update({'duration': utils.parse_date(driver.find_element_by_class_name('duration').text)})
    movie.update({'synopsis': driver.find_element_by_class_name('synopsis').text})
    movie.update({'maturity': 0})

    try:
        movie['maturity'] = int(driver.find_element_by_class_name('maturity-number').text)
    except ValueError:
        pass

    driver.find_element_by_id('tab-ShowDetails').click()
    details = driver.find_element_by_class_name('simpleSlider') \
                    .find_element_by_class_name('sliderContent')

    # cut_indexes saves the initial indexes of director, cast, screenwriter and the last index
    movie_cast = details.find_element_by_tag_name('span').find_elements_by_tag_name('li')
    cut_indexes = [i for i, m in enumerate(movie_cast) if m.get_attribute('class') == 'listLabel'] + [len(movie_cast)]
    cut_map = {0: 'directors', 1: 'cast', 2: 'screenwriters'}

    for i in range(len(cut_indexes)-1):
        movie.update({
            cut_map[i]: [
                {
                    'person_name': movie_cast[j].text, 
                    'person_id': int(movie_cast[j].find_element_by_tag_name('a') \
                                                  .get_attribute('href').split('/')[-1])
                }
                for j in range(cut_indexes[i]+1, cut_indexes[i+1])
            ]
        })

    genres = details.find_element_by_class_name('detailsTags') \
                    .find_elements_by_tag_name('ul')[0] \
                    .find_elements_by_tag_name('li')

    movie.update({
        'genres': [
            {
                'genre': genre.text,
                'genre_id': int(genre.find_element_by_tag_name('a') \
                                     .get_attribute('href').split('/')[-1])
            }
            for genre in genres
        ]
    })

    driver.close()

    return movie
Пример #36
0
def check_date(d):
  if not isinstance(d, str):
    error(str(d) + ": invalid data type")
    return None
  try:
    return utils.parse_date(d)
  except Exception as e:
    error(d + ": " + str(e))
    return None
Пример #37
0
def make_row(date_string, headline):
    date = parse_date(date_string)
    L.debug("Parsed date '{}' as {}".format(date_string, date))
    return OrderedDict([
        ('venue', 'LEAF on Bold Street'),
        ('date', date),
        ('headline', headline),
        ('url', 'http://www.thisisleaf.co.uk/#/on-bold-street/events/')
    ])
def make_row(day, headline):
    date = parse_date(day)
    L.debug("Parsed date '{}' as {}".format(day, date))
    return OrderedDict([
        ('venue', 'The Caledonia'),
        ('date', date),
        ('headline', headline),
        ('url', 'http://www.thecaledonialiverpool.com/whats-on/'),
    ])
Пример #39
0
 def parse_disturbances(self):
     """
     For a given trendline, for each segment between vertices, determine
     the stats for each "disturbance" and return a list of Disturbance
     objects
     """
     import utils
     it = iter(self.points)
     left_vertex = it.next()
     for p in it:
         if not p.vertex:
             continue
         start_yr = utils.parse_date(left_vertex.index_date).year
         end_yr = utils.parse_date(p.index_date).year
         yield Disturbance(start_yr, left_vertex.val_fit,
                           left_vertex.val_fit - p.val_fit,
                           end_yr - start_yr)
         left_vertex = p
Пример #40
0
 def parse_time_frames(self):
     modified_tf = []
     if self.args.time_frames[0].isdigit():
         for chunk in utils.get_chunks(self.args.time_frames, 3):
             if re.search(r'[0-9]+-[0-9]+', ''.join(chunk)):
                 re.compile('[,]')
                 modified_tf.append([
                     int(re.compile('[,]').sub('', chunk[0])),
                     int(re.compile('[,]').sub('', chunk[2]))
                 ])
     else:
         time_frames = self.parse_date_string(self.args.time_frames)
         for tf in time_frames:
             modified_tf.append([
                 utils.parse_date(tf[0]).timestamp(),
                 utils.parse_date(tf[1]).timestamp()
             ])
     return modified_tf
Пример #41
0
 def get_last_modified(doc):
     lm = None
     lmi = doc.select_one('.posted_since')
     if lmi:
         idx = lmi.text.find('ред. ')
         if idx > 0:
             val = lmi.text[idx + 5:idx + 20]
             lm = utils.parse_date(val)
     return lm
def make_row(date_string, headline, url):
    date = parse_date(date_string)
    L.debug("Parsed date '{}' as {}".format(date_string, date))
    return OrderedDict([
        ('organiser', 'The Bluecoat'),
        ('venue', 'The Bluecoat'),
        ('date', date),
        ('headline', headline),
        ('url', url)
    ])
Пример #43
0
def make_row(date_string, earliest_date, headline, url):
    L.debug("'{}' : '{}'".format(headline, date_string))
    date = parse_date(date_string, not_before=earliest_date)
    L.debug("Parsed date '{}' as {}".format(date_string, date))
    return OrderedDict([
        ('venue', "FACT"),
        ('date', date),
        ('headline', headline),
        ('url', url)
    ])
def make_row(date_string, headline, url):
    L.debug("'{}' : '{}'".format(headline, date_string))
    date = parse_date(date_string)
    L.debug("Parsed date '{}' as {}".format(date_string, date))
    return OrderedDict([
        ('venue', "St George's Hall"),
        ('date', date),
        ('headline', headline),
        ('url', url),
    ])
def make_row(date_string, headline, url, venue):
    date = parse_date(date_string)
    L.debug("Parsed date '{}' as {}".format(date_string, date))
    return OrderedDict([
        ('organiser', 'Philosophy In Pubs'),
        ('venue', venue),
        ('date', date),
        ('headline', headline),
        ('url', url)
    ])
Пример #46
0
 def post(self, key):
     post = self.request.POST
     form_data = dict((k, post.get(k, ''))
                       for k in ('title', 'author', 'date', 'body'))
     template_dict = {'form_data': form_data, 'key': key, 'show_form' : True}
     if 'delete_article' in post:
         try:
             NewsArticle.get(Key(key)).delete()
         except datastore_errors.Error:
             template_dict['message'] = \
                 'Could not delete article with key %r.' % key
         else:
             template_dict['message'] = 'Article deleted.'
             template_dict['show_form'] = False
     else:
         try:
             date = utils.parse_date(form_data['date'])
         except ValueError:
             template_dict['message'] = \
                 'Date is not in the correct format (YYYY-MM-DD).'
         else:
             if key == 'new':
                 try:
                     article = NewsArticle(title=form_data['title'],
                                           author=form_data['author'],
                                           date=date,
                                           body=form_data['body'])
                     article.put()
                 except datastore_errors.Error:
                     template_dict['message'] = \
                         'Could not create new article.'
                 else:
                     template_dict['message'] = 'Article created.'
                     template_dict['show_form'] = False
             else:
                 try:
                     article = NewsArticle.get(Key(key))
                 except BadKeyError:
                     template_dict['message'] = \
                         'Could not find article with key %r.' % key
                 else:
                     article.title = form_data['title']
                     article.author = form_data['author']
                     article.date = date
                     article.body = form_data['body']
                     try:
                         article.put()
                     except datastore_errors.Error:
                         template_dict['message'] = \
                             'Could not save changes to article.'
                     else:
                         template_dict['form_data'] = article
                         template_dict['message'] = 'Changes saved.'
     self.render_template('edit', template_dict)
Пример #47
0
 def post(self):
     user_id = self.get_secure_cookie("user")
     taskname = self.get_argument("taskname", "")
     description = self.get_argument("description", "")
     date = parse_date(self.get_argument("datetime", ""))
     if taskname.strip():
         newitem = TaskItem(user_id=user_id,
                            taskname=taskname,
                            datetime_due=date,
                            description=description)
         self.db.create(newitem)
     self.redirect("/list")
Пример #48
0
 def parse_disturbances(self):
     """
     For a given trendline, for each segment between vertices, determine
     the stats for each "disturbance" and return a list of Disturbance
     objects
     """
     import utils
     it = iter(self.points)
     left_vertex = it.next()
     for p in it:
         if not p.vertex:
             continue
         start_yr = utils.parse_date(left_vertex.index_date).year
         end_yr = utils.parse_date(p.index_date).year
         yield Disturbance(
             start_yr,
             left_vertex.val_fit,
             left_vertex.val_fit - p.val_fit,  # TODO %?
             end_yr - start_yr
         )
         left_vertex = p
Пример #49
0
def run():
	if len(sys.argv) != 3:
		print("Usage:")
		print("python retire.py bioguideID termEndDate")
		sys.exit()

	try:
		utils.parse_date(sys.argv[2])
	except:
		print("Invalid date: ", sys.argv[2])
		sys.exit()

	print("Loading current YAML...")
	y = utils.load_data("legislators-current.yaml")
	print("Loading historical YAML...")
	y1 = utils.load_data("legislators-historical.yaml")

	for moc in y:
		if moc["id"].get("bioguide", None) != sys.argv[1]: continue

		print("Updating:")
		rtyaml.pprint(moc["id"])
		print()
		rtyaml.pprint(moc["name"])
		print()
		rtyaml.pprint(moc["terms"][-1])

		moc["terms"][-1]["end"] = sys.argv[2]

		y.remove(moc)
		y1.append(moc)

		break

	print("Saving changes...")
	utils.save_data(y, "legislators-current.yaml")
	utils.save_data(y1, "legislators-historical.yaml")
Пример #50
0
def food(date, breakfast=None, lunch=None, dinner=None, snacks=None):
    try:
        date = utils.parse_date(date)
    except ValueError:
        return 'Error: unable to parse date %s' % date
    db = utils.read_db()
    initial = utils.daily_points(date, db)
    messages = ['%s had %s points' % (date, initial)]
    messages.extend(process_meal(breakfast, db, date, 'breakfast'))
    messages.extend(process_meal(lunch, db, date, 'lunch'))
    messages.extend(process_meal(dinner, db, date, 'dinner'))
    messages.extend(process_meal(snacks, db, date, 'snacks'))
    final = utils.daily_points(date, db)
    diff = final - initial
    messages.append('%s added %s points (now has %s)' % (date, diff, final))
    utils.write_db(db)
    return '\n'.join(messages)
Пример #51
0
def prehab(date=None, home=None, work=None, travel=None, other=None):
    try:
        date = utils.parse_date(date)
    except ValueError:
        return 'Error: unable to parse date %s' % date
    db = utils.read_db()
    initial = utils.daily_points(date, db)
    messages = ['%s had %s points' % (date, initial)]
    messages.extend(process_loc(home, db, date, 'home'))
    messages.extend(process_loc(work, db, date, 'work'))
    messages.extend(process_loc(travel, db, date, 'travel'))
    messages.extend(process_loc(other, db, date, 'other'))
    final = utils.daily_points(date, db)
    diff = final - initial
    messages.append('%s added %s points (now has %s)' % (date, diff, final))
    utils.write_db(db)
    return '\n'.join(messages)
Пример #52
0
def get_all_listings(listings_fobj):
    lxml_root = lxml.html.fromstring(listings_fobj.read())
    divs = lxml_root.xpath('//div[@id="events_list"]/div[@class="item"]')
    assert len(divs) > 0

    earliest_date = None

    for div in divs:
        headline = clean_headline(
            div.xpath('./h4/a')[0].text_content().strip())

        url = BASE_URL + div.xpath('./h4/a/@href')[0]
        date_tag = div.xpath('./span[@class="from"]')[0]
        date_string = extract_date(date_tag.text_content())

        if not earliest_date:
            earliest_date = parse_date(date_string)
        yield make_row(date_string, earliest_date, headline, url)
Пример #53
0
    def analysis_reducer(self, point_wkt, pix_datas):
        """
        Given a point wkt and a list of pix datas in the format:
        [
            {'date': '2011-09-01', 'val': 160.0},
            {'date': '2012-09-01', 'val': 180.0},
            ...
        ]
        perform the landtrendr analysis and change labeling.

        Yields out the change labels and trendline data for the given point
        """
        sys.stdout.write('.')  # for viewing progress
        sys.stdout.flush()

        job = os.environ.get('LT_JOB')
        settings = utils.get_settings(job)

        pix_datas = list(pix_datas)  # save iterator to a list
        pix_trendline = utils.analyze(
            pix_datas,
            settings['line_cost'],
            utils.parse_date(settings['target_date'])
        )

        # write out pix trendline
        for label, val in pix_trendline.mr_label_output().iteritems():
            # prepend 'aux/' to label name so written to sub folder
            yield (
                'trendline/%s' % label,
                {'pix_ctr_wkt': point_wkt, 'value': val}
            )

        label_rules = [
            classes.LabelRule(lr) for lr in settings['label_rules']
        ]

        change_labels = utils.change_labeling(pix_trendline, label_rules)

        # write out change labels
        for label_name, data in change_labels.iteritems():
            for key in ['class_val', 'onset_year', 'magnitude', 'duration']:
                label_key = '%s_%s' % (label_name, key)
                yield label_key, {'pix_ctr_wkt': point_wkt, 'value': data[key]}
    def parse_template_6(self, element):
        workshop = {}
        title = rex.rex(element[1], r'(.*)Edited\s*by.*', re.I | re.S).group(1)

        workshop['volume_number'] = WorkshopSummaryParser.extract_volume_number(element[0].get('href'))
        workshop['label'] = element[0].text.replace('.', '')
        workshop['url'] = element[0].get('href')
        workshop['time'] = utils.parse_date(title)
        try:
            workshop['edition'] = tonumber(
                rex.rex(title,
                        r'.*Proceedings(\s*of)?(\s*the)?\s*(\d{1,}|first|second|third|forth|fourth|fifth)[thrd]*'
                        r'.*Workshop.*',
                        re.I, default=None).group(3))
        except:
            #'edition' property is optional
            pass

        self.add_workshop(workshop)
Пример #55
0
 def post(self):
     post = self.request.POST
     if post['kind'] == 'badge':
         badge = Badge(
             name=post['name'],
             description=post['description'],
             category=post['category'],
             image=post['image'],
             value=int(post['value'])
         )
         badge.save()
     elif post['kind'] == 'award':
         badge = Badge.get_by_id(int(post['badge']))
         for member in post.getall('members'):
             member = Member.get_by_id(int(member))
             award = Award(
                 member=member,
                 badge=badge,
                 date=datetime.date.today(),
                 proof=post['proof']
             )
             award.save()
             member.score += badge.value
             member.save()
     elif post['kind'] == 'talk':
         talk = Talk(
             title=post['title'],
             date=utils.parse_date(post['date']),
             description=post['description'],
             member=Member.get_by_id(int(post['member'])),
             video=post['video']
         )
         talk.put()
     elif post['kind'] == 'taglineform':
         properties = GeneralSiteProperties.all().get()
         if properties == None:
             properties = GeneralSiteProperties(tag_line=post['tagline'])
             properties.put()
         else:
             properties.tag_line = post['tagline']
             properties.put()
     self.get()
    def parse_template_4(self, element):
        """
        A template for joint proceedings of three workshops.

        Examples:
            - http://ceur-ws.org/Vol-981/
            - http://ceur-ws.org/Vol-862/
            - http://ceur-ws.org/Vol-853/
        """
        workshop_1 = {'id': 1}
        workshop_2 = {'id': 2}
        workshop_3 = {'id': 3}
        summary = self.rex(element[1], [
            r'(joint\s+proceedings\s+of\s+[the]*.*workshops:\s*([\s\w]+)\(([a-zA-Z]+)\d+\)'
            r'[and,\s]+([\s\w]+)\(([a-zA-Z]+)\d+\)[and,\s]+([\s\w]+)\(([a-zA-Z]+)\d+\)[,\s]+.*)Edited by.*',

            r"(joint\s+proceedings\s+of\s+([\s\w,]+)\(([a-zA-Z]+)['\s]?\d+\)[and,\s]+([\s\w-]+)\(([a-zA-Z]+)['\s]?\d+\)"
            r"[and,\s]+([\s\w]+)\(([a-zA-Z]+)['\s]?\d+\)[,\s]+.*)Edited by.*"
        ],
                           re.I | re.S)

        if len(summary.groups()) != 7:
            raise DataNotFound()

        title = summary.group(1)

        workshop_1['volume_number'] = workshop_2['volume_number'] = workshop_3['volume_number'] = \
            WorkshopSummaryParser.extract_volume_number(element[0].get('href'))
        workshop_1['url'] = workshop_2['url'] = workshop_3['url'] = element[0].get('href')
        workshop_1['time'] = workshop_2['time'] = workshop_3['time'] = utils.parse_date(title)

        workshop_1['label'] = summary.group(2)
        workshop_1['short_label'] = summary.group(3)
        workshop_2['label'] = summary.group(4)
        workshop_2['short_label'] = summary.group(5)
        workshop_3['label'] = summary.group(6)
        workshop_3['short_label'] = summary.group(7)

        self.add_workshop(workshop_1)
        self.add_workshop(workshop_2)
        self.add_workshop(workshop_3)
def process(html_fobj):

    lxml_root = lxml.html.fromstring(html_fobj.read())
    divs = lxml_root.xpath('//div[@class="gif_box"]')
    assert len(divs) > 0

    earliest_date = None

    for div in divs:
        headline = clean_headline(div.xpath(
            './div[@class="gif_txt"]/a')[0].text_content().strip())

        url = div.xpath(
            './div/a/@href')[0]

        date_string = div.xpath(
            './div[@class="listdate"]')[0].text_content()

        if not earliest_date:
            earliest_date = parse_date(date_string)
        yield make_row(date_string, earliest_date, headline, url)
Пример #58
0
 def command_received(self, cmd_data):
     LOG_MSG('Poll command got response. Response: {0}.'.format(cmd_data))
     for cmd in cmd_data:
         # Obtain only new commands next time
         cmd_date = parse_date(cmd['timestamp'])
         if self.owner.timestamp is not None:
             self.owner.timestamp = max(self.owner.timestamp, cmd_date)
         else:
             self.owner.timestamp = cmd_date
         # device-application will use this deferred object to notify me about the command progress.
         thiscmd = cmd
         def ok(result):
             self.command_done(thiscmd, result)
         def err(reason):
             self.command_failed(thiscmd, reason)
         defer = Deferred()
         defer.addCallbacks(ok, err)
         try:
             LOG_MSG('Executing command {0} handler.'.format(cmd))
             self.owner.run_command(cmd, defer)
         except Exception, err:
             LOG_ERR('Failed to execute device-delegate on_command. Reason: <{0}>.'.format(err))
             self.command_failed(cmd, err)