def run(self):
dates = [x for x in self.interval_generator()]
if self.source_definition.get('reverse_chronological', False):
dates.reverse()
meeting_count = 0
vote_count = 0
passed_vote_count = 0
for start_date, end_date in dates:
meetings = self.client.service.GetMeetingsByDateRange(
Sitename=self.source_definition['sitename'],
StartDate=start_date,
EndDate=end_date,
MetaDataOnly=False)
meeting_types = self._meetingtypes_as_dict()
meeting_sorting_key = self.source_definition.get(
'meeting_sorting', 'MeetingDate')
if meetings.Meetings is not None:
sorted_meetings = sorted(
meetings.Meetings[0],
key=lambda m: getattr(m, meeting_sorting_key))
else:
sorted_meetings = []
processed = []
for meeting in sorted_meetings:
meeting_dict = meeting_to_dict(meeting)
# Getting the meeting type as a string is easier this way ...
# log.debug(meeting_dict['Id'])
meeting_dict['Meetingtype'] = meeting_types[
meeting_dict['MeetingtypeId']]
kv = self.client.factory.create('ns0:iBabsKeyValue')
kv.Key = 'IncludeMeetingItems'
kv.Value = True
kv2 = self.client.factory.create('ns0:iBabsKeyValue')
kv2.Key = 'IncludeListEntries'
kv2.Value = True
kv3 = self.client.factory.create('ns0:iBabsKeyValue')
kv3.Key = 'IncludeMeetingItems'
kv3.Value = True
params = self.client.factory.create('ns0:ArrayOfiBabsKeyValue')
params.iBabsKeyValue.append(kv)
params.iBabsKeyValue.append(kv2)
params.iBabsKeyValue.append(kv3)
vote_meeting = self.client.service.GetMeetingWithOptions(
Sitename=self.source_definition['sitename'],
MeetingId=meeting_dict['Id'],
Options=params)
meeting_dict_short = meeting_to_dict(vote_meeting.Meeting)
# log.debug(meeting_dict_short['MeetingDate'])
if meeting_dict_short['MeetingItems'] is None:
continue
for mi in meeting_dict_short['MeetingItems']:
if mi['ListEntries'] is None:
continue
for le in mi['ListEntries']:
votes = self.client.service.GetListEntryVotes(
Sitename=self.source_definition['sitename'],
EntryId=le['EntryId'])
if votes.ListEntryVotes is None:
votes = []
else:
votes = votes_to_dict(votes.ListEntryVotes[0])
# log.debug(votes)
result = {
'meeting': meeting_dict,
'entry': le,
'votes': votes
}
vote_count += 1
if self.valid_meeting(result):
processed += self.process_meeting(result)
meeting_count += 1
# log.debug(processed)
for result in processed:
yield 'application/json', json.dumps(
result), 'entity_placeholder', result
passed_vote_count += 1
log.debug("[%s] Now processing meetings from %s to %s" % (
self.source_definition['index_name'],
start_date,
end_date,
))
log.info(
"[%s] Extracted total of %d ibabs meetings and passed %s out of %d voting rounds."
% (
self.source_definition['index_name'],
meeting_count,
passed_vote_count,
vote_count,
))
def run(self):
dates = [x for x in self.interval_generator()]
if self.source_definition.get('reverse_chronological', False):
dates.reverse()
for start_date, end_date in dates:
meetings = self.client.service.GetMeetingsByDateRange(
Sitename=self.source_definition['sitename'],
StartDate=start_date,
EndDate=end_date,
MetaDataOnly=False)
meeting_types = self._meetingtypes_as_dict()
meeting_count = 0
vote_count = 0
meeting_sorting_key = self.source_definition.get(
'meeting_sorting', 'MeetingDate')
if meetings.Meetings is not None:
sorted_meetings = sorted(
meetings.Meetings[0],
key=lambda m: getattr(m, meeting_sorting_key))
else:
sorted_meetings = []
processed = []
for meeting in sorted_meetings:
meeting_dict = meeting_to_dict(meeting)
# Getting the meeting type as a string is easier this way ...
log.debug(meeting_dict['Id'])
meeting_dict['Meetingtype'] = meeting_types[
meeting_dict['MeetingtypeId']]
kv = self.client.factory.create('ns0:iBabsKeyValue')
kv.Key = 'IncludeMeetingItems'
kv.Value = True
kv2 = self.client.factory.create('ns0:iBabsKeyValue')
kv2.Key = 'IncludeListEntries'
kv2.Value = True
kv3 = self.client.factory.create('ns0:iBabsKeyValue')
kv3.Key = 'IncludeMeetingItems'
kv3.Value = True
params = self.client.factory.create('ns0:ArrayOfiBabsKeyValue')
params.iBabsKeyValue.append(kv)
params.iBabsKeyValue.append(kv2)
params.iBabsKeyValue.append(kv3)
vote_meeting = self.client.service.GetMeetingWithOptions(
Sitename=self.source_definition['sitename'],
MeetingId=meeting_dict['Id'],
Options=params)
meeting_dict_short = meeting_to_dict(vote_meeting.Meeting)
log.debug(meeting_dict_short['MeetingDate'])
if meeting_dict_short['MeetingItems'] is None:
continue
for mi in meeting_dict_short['MeetingItems']:
if mi['ListEntries'] is None:
continue
for le in mi['ListEntries']:
votes = self.client.service.GetListEntryVotes(
Sitename=self.source_definition['sitename'],
EntryId=le['EntryId'])
if votes.ListEntryVotes is None:
votes = []
else:
votes = votes_to_dict(votes.ListEntryVotes[0])
# log.debug(votes)
result = {
'meeting': meeting_dict,
'entry': le,
'votes': votes
}
vote_count += 1
if self.valid_meeting(result):
processed += self.process_meeting(result)
meeting_count += 1
# log.debug(processed)
passed_vote_count = 0
for result in processed:
yield 'application/json', json.dumps(result)
passed_vote_count += 1
log.info("Now processing meetings from %s to %s" % (start_date, end_date,))
log.info("Extracted %d meetings and passed %s out of %d voting rounds." % (
meeting_count, passed_vote_count, vote_count,))
def run(self):
for start_date, end_date in self.interval_generator():
meetings = self.client.service.GetMeetingsByDateRange(
Sitename=self.source_definition['sitename'],
StartDate=start_date,
EndDate=end_date,
MetaDataOnly=False)
meeting_types = self._meetingtypes_as_dict()
meeting_count = 0
vote_count = 0
meeting_sorting_key = self.source_definition.get(
'meeting_sorting', 'MeetingDate')
sorted_meetings = sorted(
meetings.Meetings[0],
key=lambda m: getattr(m, meeting_sorting_key))
processed = []
for meeting in sorted_meetings:
meeting_dict = meeting_to_dict(meeting)
# Getting the meeting type as a string is easier this way ...
pprint(meeting_dict['Id'])
meeting_dict['Meetingtype'] = meeting_types[
meeting_dict['MeetingtypeId']]
kv = self.client.factory.create('ns0:iBabsKeyValue')
kv.Key = 'IncludeMeetingItems'
kv.Value = True
kv2 = self.client.factory.create('ns0:iBabsKeyValue')
kv2.Key = 'IncludeListEntries'
kv2.Value = True
params = self.client.factory.create('ns0:ArrayOfiBabsKeyValue')
params.iBabsKeyValue.append(kv)
params.iBabsKeyValue.append(kv2)
vote_meeting = self.client.service.GetMeetingWithOptions(
Sitename=self.source_definition['sitename'],
MeetingId=meeting_dict['Id'],
Options=params)
meeting_dict_short = meeting_to_dict(vote_meeting.Meeting)
if meeting_dict_short['MeetingItems'] is None:
continue
for mi in meeting_dict_short['MeetingItems']:
if mi['ListEntries'] is None:
continue
for le in mi['ListEntries']:
log.debug("Motie id : %s" % le['EntryId'])
hash_content = u'motion-%s' % (le['EntryId'])
hashed_motion_id = unicode(
sha1(hash_content.decode('utf8')).hexdigest())
log.debug("Hashedotie id : %s" %
(hashed_motion_id.strip(), ))
votes = self.client.service.GetListEntryVotes(
Sitename=self.source_definition['sitename'],
EntryId=le['EntryId'])
if votes.ListEntryVotes is None:
votes = []
else:
votes = votes_to_dict(votes.ListEntryVotes[0])
result = {
'motion_id': hashed_motion_id,
'meeting': meeting_dict,
'entry': le,
'votes': votes
}
vote_count += 1
if self.valid_meeting(result):
processed += self.process_meeting(result)
meeting_count += 1
#pprint(processed)
passed_vote_count = 0
for result in processed:
yield 'application/json', json.dumps(result)
passed_vote_count += 1
log.info("Now processing meetings from %s to %s" % (
start_date,
end_date,
))
log.info(
"Extracted %d meetings and passed %s out of %d voting rounds."
% (
meeting_count,
passed_vote_count,
vote_count,
))
def run(self):
start_date = self.source_definition.get('start_date',
'2015-06-01T00:00:00')
end_date = self.source_definition.get('end_date',
'2016-07-01T00:00:00')
print "Getting meetings for %s to %s" % (
start_date,
end_date,
)
meetings = self.client.service.GetMeetingsByDateRange(
Sitename=self.source_definition['sitename'],
StartDate=start_date,
EndDate=end_date,
MetaDataOnly=False)
meeting_types = self._meetingtypes_as_dict()
meeting_count = 0
vote_count = 0
meeting_sorting_key = self.source_definition.get(
'meeting_sorting', 'MeetingDate')
sorted_meetings = sorted(meetings.Meetings[0],
lambda m: getaatr(m, meeting_sorting_key))
for meeting in sorted_meetings:
meeting_dict = meeting_to_dict(meeting)
# Getting the meeting type as a string is easier this way ...
pprint(meeting_dict['Id'])
meeting_dict['Meetingtype'] = meeting_types[
meeting_dict['MeetingtypeId']]
kv = self.client.factory.create('ns0:iBabsKeyValue')
kv.Key = 'IncludeMeetingItems'
kv.Value = True
kv2 = self.client.factory.create('ns0:iBabsKeyValue')
kv2.Key = 'IncludeListEntries'
kv2.Value = True
params = self.client.factory.create('ns0:ArrayOfiBabsKeyValue')
params.iBabsKeyValue.append(kv)
params.iBabsKeyValue.append(kv2)
vote_meeting = self.client.service.GetMeetingWithOptions(
Sitename=self.source_definition['sitename'],
MeetingId=meeting_dict['Id'],
Options=params)
meeting_dict_short = meeting_to_dict(vote_meeting.Meeting)
processed = []
for mi in meeting_dict_short['MeetingItems']:
if mi['ListEntries'] is None:
continue
for le in mi['ListEntries']:
# motion's unique identifier
print le['EntryTitle'].split(' ')[0][1:]
motion_id = normalize_motion_id(
le['EntryTitle'].split(' ')[0][1:])
print "Motie id : %s" % (motion_id.strip(), )
hash_content = u'motion-%s' % (motion_id.strip())
hashed_motion_id = unicode(
sha1(hash_content.decode('utf8')).hexdigest())
print "Hashedotie id : %s" % (hashed_motion_id.strip(), )
votes = self.client.service.GetListEntryVotes(
Sitename=self.source_definition['sitename'],
EntryId=le['EntryId'])
if votes.ListEntryVotes is None:
votes = []
else:
votes = votes_to_dict(votes.ListEntryVotes[0])
result = {
'motion_id': hashed_motion_id,
'meeting': meeting_dict,
'entry': le,
'votes': votes
}
vote_count += 1
if self.valid_meeting(result):
processed += self.process_meeting(result)
meeting_count += 1
#pprint(processed)
passed_vote_count = 0
for result in processed:
print "%s - %s" % (
result.get('id', '-'),
result.get('name', '-'),
)
yield 'application/json', json.dumps(result)
passed_vote_count += 1
print "Extracted %d meetings and passed %s out of %d voting rounds." % (
meeting_count,
passed_vote_count,
vote_count,
)