def run(self):
start_date = self.source_definition.get(
'start_date', '2015-06-01T00:00:00')
end_date = self.source_definition.get(
'end_date', '2016-07-01T00:00:00')
print "Getting meetings for %s to %s" % (start_date, end_date,)
meetings = self.client.service.GetMeetingsByDateRange(
Sitename=self.source_definition['sitename'],
StartDate=start_date,
EndDate=end_date,
MetaDataOnly=False)
meeting_types = self._meetingtypes_as_dict()
meeting_count = 0
meeting_item_count = 0
for meeting in meetings.Meetings[0]:
meeting_dict = meeting_to_dict(meeting)
# Getting the meeting type as a string is easier this way ...
meeting_dict['Meetingtype'] = meeting_types[
meeting_dict['MeetingtypeId']]
yield 'application/json', json.dumps(meeting_dict)
if meeting.MeetingItems is not None:
for meeting_item in meeting.MeetingItems[0]:
meeting_item_dict = meeting_item_to_dict(meeting_item)
# This is a bit hacky, but we need to know this
meeting_item_dict['MeetingId'] = meeting_dict['Id']
meeting_item_dict['Meeting'] = meeting_dict
yield 'application/json', json.dumps(meeting_item_dict)
meeting_item_count += 1
meeting_count += 1
print "Extracted %d meetings and %d meeting items." % (
meeting_count, meeting_item_count,)
def run(self):
meeting_count = 0
meetings_skipped = 0
meeting_item_count = 0
for start_date, end_date in self.interval_generator():
log.info("Now processing meetings from %s to %s" % (
start_date,
end_date,
))
meetings = self.client.service.GetMeetingsByDateRange(
Sitename=self.source_definition['sitename'],
StartDate=start_date.strftime('%Y-%m-%dT%H:%M:%S'),
EndDate=end_date.strftime('%Y-%m-%dT%H:%M:%S'),
MetaDataOnly=False)
meeting_types = self._meetingtypes_as_dict()
if meetings.Meetings:
for meeting in meetings.Meetings[0]:
meeting_dict = meeting_to_dict(meeting)
# sometimes a meetingtype is actualy a meeting for some
# reason. Let's ignore these for now
if meeting_dict['MeetingtypeId'] not in meeting_types:
meetings_skipped += 1
continue
meeting_dict['Meetingtype'] = meeting_types[
meeting_dict['MeetingtypeId']]
yield 'application/json', json.dumps(meeting_dict)
if meeting.MeetingItems is not None:
for meeting_item in meeting.MeetingItems[0]:
meeting_item_dict = meeting_item_to_dict(
meeting_item)
# This is a bit hacky, but we need to know this
meeting_item_dict['MeetingId'] = meeting_dict['Id']
meeting_item_dict['Meeting'] = meeting_dict
yield 'application/json', json.dumps(
meeting_item_dict)
meeting_item_count += 1
meeting_count += 1
log.info(
"Extracted total of %d meetings and %d meeting items. Also "
"Skipped %d meetings in total." % (
meeting_count,
meeting_item_count,
meetings_skipped,
))
def run(self):
meeting_count = 0
meetings_skipped = 0
for start_date, end_date in self.interval_generator():
log.debug("[%s] Now processing meetings from %s to %s" % (
self.source_definition['sitename'],
start_date,
end_date,
))
meetings = self.client.service.GetMeetingsByDateRange(
Sitename=self.source_definition['sitename'],
StartDate=start_date.strftime('%Y-%m-%dT%H:%M:%S'),
EndDate=end_date.strftime('%Y-%m-%dT%H:%M:%S'),
MetaDataOnly=False)
meeting_types = self._meetingtypes_as_dict()
if meetings.Meetings:
for meeting in meetings.Meetings[0]:
meeting_dict = meeting_to_dict(meeting)
# sometimes a meetingtype is actually a meeting for some
# reason. Let's ignore these for now
if meeting_dict['MeetingtypeId'] not in meeting_types:
meetings_skipped += 1
continue
meeting_dict['Meetingtype'] = meeting_types[
meeting_dict['MeetingtypeId']]
yield 'application/json', \
json.dumps(meeting_dict), \
meeting_dict['Id'], \
meeting_dict
meeting_count += 1
log.info(
"[%s] Extracted total of %d ibabs meetings. Also skipped %d meetings in total."
% (
self.source_definition['sitename'],
meeting_count,
meetings_skipped,
))
def run(self):
meeting_count = 0
meetings_skipped = 0
meeting_item_count = 0
for start_date, end_date in self.interval_generator():
log.info("%s: Now processing meetings from %s to %s" % (
self.source_definition['sitename'], start_date, end_date,))
meetings = self.client.service.GetMeetingsByDateRange(
Sitename=self.source_definition['sitename'],
StartDate=start_date.strftime('%Y-%m-%dT%H:%M:%S'),
EndDate=end_date.strftime('%Y-%m-%dT%H:%M:%S'),
MetaDataOnly=False)
meeting_types = self._meetingtypes_as_dict()
if meetings.Meetings:
for meeting in meetings.Meetings[0]:
meeting_dict = meeting_to_dict(meeting)
# sometimes a meetingtype is actualy a meeting for some
# reason. Let's ignore these for now
if meeting_dict['MeetingtypeId'] not in meeting_types:
meetings_skipped += 1
continue
meeting_dict['Meetingtype'] = meeting_types[
meeting_dict['MeetingtypeId']]
yield 'application/json', json.dumps(meeting_dict)
if meeting.MeetingItems is not None:
for meeting_item in meeting.MeetingItems[0]:
meeting_item_dict = meeting_item_to_dict(
meeting_item)
# This is a bit hacky, but we need to know this
meeting_item_dict['MeetingId'] = meeting_dict['Id']
meeting_item_dict['Meeting'] = meeting_dict
yield 'application/json', json.dumps(
meeting_item_dict)
meeting_item_count += 1
meeting_count += 1
log.info("Extracted total of %d meetings and %d meeting items. Also "
"Skipped %d meetings in total." % (meeting_count, meeting_item_count, meetings_skipped,))
def run(self):
dates = [x for x in self.interval_generator()]
if self.source_definition.get('reverse_chronological', False):
dates.reverse()
meeting_count = 0
vote_count = 0
passed_vote_count = 0
for start_date, end_date in dates:
meetings = self.client.service.GetMeetingsByDateRange(
Sitename=self.source_definition['sitename'],
StartDate=start_date,
EndDate=end_date,
MetaDataOnly=False)
meeting_types = self._meetingtypes_as_dict()
meeting_sorting_key = self.source_definition.get(
'meeting_sorting', 'MeetingDate')
if meetings.Meetings is not None:
sorted_meetings = sorted(
meetings.Meetings[0],
key=lambda m: getattr(m, meeting_sorting_key))
else:
sorted_meetings = []
processed = []
for meeting in sorted_meetings:
meeting_dict = meeting_to_dict(meeting)
# Getting the meeting type as a string is easier this way ...
# log.debug(meeting_dict['Id'])
meeting_dict['Meetingtype'] = meeting_types[
meeting_dict['MeetingtypeId']]
kv = self.client.factory.create('ns0:iBabsKeyValue')
kv.Key = 'IncludeMeetingItems'
kv.Value = True
kv2 = self.client.factory.create('ns0:iBabsKeyValue')
kv2.Key = 'IncludeListEntries'
kv2.Value = True
kv3 = self.client.factory.create('ns0:iBabsKeyValue')
kv3.Key = 'IncludeMeetingItems'
kv3.Value = True
params = self.client.factory.create('ns0:ArrayOfiBabsKeyValue')
params.iBabsKeyValue.append(kv)
params.iBabsKeyValue.append(kv2)
params.iBabsKeyValue.append(kv3)
vote_meeting = self.client.service.GetMeetingWithOptions(
Sitename=self.source_definition['sitename'],
MeetingId=meeting_dict['Id'],
Options=params)
meeting_dict_short = meeting_to_dict(vote_meeting.Meeting)
# log.debug(meeting_dict_short['MeetingDate'])
if meeting_dict_short['MeetingItems'] is None:
continue
for mi in meeting_dict_short['MeetingItems']:
if mi['ListEntries'] is None:
continue
for le in mi['ListEntries']:
votes = self.client.service.GetListEntryVotes(
Sitename=self.source_definition['sitename'],
EntryId=le['EntryId'])
if votes.ListEntryVotes is None:
votes = []
else:
votes = votes_to_dict(votes.ListEntryVotes[0])
# log.debug(votes)
result = {
'meeting': meeting_dict,
'entry': le,
'votes': votes
}
vote_count += 1
if self.valid_meeting(result):
processed += self.process_meeting(result)
meeting_count += 1
# log.debug(processed)
for result in processed:
yield 'application/json', json.dumps(
result), 'entity_placeholder', result
passed_vote_count += 1
log.debug("[%s] Now processing meetings from %s to %s" % (
self.source_definition['index_name'],
start_date,
end_date,
))
log.info(
"[%s] Extracted total of %d ibabs meetings and passed %s out of %d voting rounds."
% (
self.source_definition['index_name'],
meeting_count,
passed_vote_count,
vote_count,
))
def run(self):
for start_date, end_date in self.interval_generator():
meetings = self.client.service.GetMeetingsByDateRange(
Sitename=self.source_definition['sitename'],
StartDate=start_date,
EndDate=end_date,
MetaDataOnly=False)
meeting_types = self._meetingtypes_as_dict()
meeting_count = 0
vote_count = 0
meeting_sorting_key = self.source_definition.get(
'meeting_sorting', 'MeetingDate')
sorted_meetings = sorted(
meetings.Meetings[0],
key=lambda m: getattr(m, meeting_sorting_key))
processed = []
for meeting in sorted_meetings:
meeting_dict = meeting_to_dict(meeting)
# Getting the meeting type as a string is easier this way ...
pprint(meeting_dict['Id'])
meeting_dict['Meetingtype'] = meeting_types[
meeting_dict['MeetingtypeId']]
kv = self.client.factory.create('ns0:iBabsKeyValue')
kv.Key = 'IncludeMeetingItems'
kv.Value = True
kv2 = self.client.factory.create('ns0:iBabsKeyValue')
kv2.Key = 'IncludeListEntries'
kv2.Value = True
params = self.client.factory.create('ns0:ArrayOfiBabsKeyValue')
params.iBabsKeyValue.append(kv)
params.iBabsKeyValue.append(kv2)
vote_meeting = self.client.service.GetMeetingWithOptions(
Sitename=self.source_definition['sitename'],
MeetingId=meeting_dict['Id'],
Options=params)
meeting_dict_short = meeting_to_dict(vote_meeting.Meeting)
if meeting_dict_short['MeetingItems'] is None:
continue
for mi in meeting_dict_short['MeetingItems']:
if mi['ListEntries'] is None:
continue
for le in mi['ListEntries']:
log.debug("Motie id : %s" % le['EntryId'])
hash_content = u'motion-%s' % (le['EntryId'])
hashed_motion_id = unicode(
sha1(hash_content.decode('utf8')).hexdigest())
log.debug("Hashedotie id : %s" %
(hashed_motion_id.strip(), ))
votes = self.client.service.GetListEntryVotes(
Sitename=self.source_definition['sitename'],
EntryId=le['EntryId'])
if votes.ListEntryVotes is None:
votes = []
else:
votes = votes_to_dict(votes.ListEntryVotes[0])
result = {
'motion_id': hashed_motion_id,
'meeting': meeting_dict,
'entry': le,
'votes': votes
}
vote_count += 1
if self.valid_meeting(result):
processed += self.process_meeting(result)
meeting_count += 1
#pprint(processed)
passed_vote_count = 0
for result in processed:
yield 'application/json', json.dumps(result)
passed_vote_count += 1
log.info("Now processing meetings from %s to %s" % (
start_date,
end_date,
))
log.info(
"Extracted %d meetings and passed %s out of %d voting rounds."
% (
meeting_count,
passed_vote_count,
vote_count,
))
def run(self):
dates = [x for x in self.interval_generator()]
if self.source_definition.get('reverse_chronological', False):
dates.reverse()
for start_date, end_date in dates:
meetings = self.client.service.GetMeetingsByDateRange(
Sitename=self.source_definition['sitename'],
StartDate=start_date,
EndDate=end_date,
MetaDataOnly=False)
meeting_types = self._meetingtypes_as_dict()
meeting_count = 0
vote_count = 0
meeting_sorting_key = self.source_definition.get(
'meeting_sorting', 'MeetingDate')
if meetings.Meetings is not None:
sorted_meetings = sorted(
meetings.Meetings[0],
key=lambda m: getattr(m, meeting_sorting_key))
else:
sorted_meetings = []
processed = []
for meeting in sorted_meetings:
meeting_dict = meeting_to_dict(meeting)
# Getting the meeting type as a string is easier this way ...
log.debug(meeting_dict['Id'])
meeting_dict['Meetingtype'] = meeting_types[
meeting_dict['MeetingtypeId']]
kv = self.client.factory.create('ns0:iBabsKeyValue')
kv.Key = 'IncludeMeetingItems'
kv.Value = True
kv2 = self.client.factory.create('ns0:iBabsKeyValue')
kv2.Key = 'IncludeListEntries'
kv2.Value = True
kv3 = self.client.factory.create('ns0:iBabsKeyValue')
kv3.Key = 'IncludeMeetingItems'
kv3.Value = True
params = self.client.factory.create('ns0:ArrayOfiBabsKeyValue')
params.iBabsKeyValue.append(kv)
params.iBabsKeyValue.append(kv2)
params.iBabsKeyValue.append(kv3)
vote_meeting = self.client.service.GetMeetingWithOptions(
Sitename=self.source_definition['sitename'],
MeetingId=meeting_dict['Id'],
Options=params)
meeting_dict_short = meeting_to_dict(vote_meeting.Meeting)
log.debug(meeting_dict_short['MeetingDate'])
if meeting_dict_short['MeetingItems'] is None:
continue
for mi in meeting_dict_short['MeetingItems']:
if mi['ListEntries'] is None:
continue
for le in mi['ListEntries']:
votes = self.client.service.GetListEntryVotes(
Sitename=self.source_definition['sitename'],
EntryId=le['EntryId'])
if votes.ListEntryVotes is None:
votes = []
else:
votes = votes_to_dict(votes.ListEntryVotes[0])
# log.debug(votes)
result = {
'meeting': meeting_dict,
'entry': le,
'votes': votes
}
vote_count += 1
if self.valid_meeting(result):
processed += self.process_meeting(result)
meeting_count += 1
# log.debug(processed)
passed_vote_count = 0
for result in processed:
yield 'application/json', json.dumps(result)
passed_vote_count += 1
log.info("Now processing meetings from %s to %s" % (start_date, end_date,))
log.info("Extracted %d meetings and passed %s out of %d voting rounds." % (
meeting_count, passed_vote_count, vote_count,))
def run(self):
current_start = datetime(2000, 1, 1) # Start somewhere by default
if 'start_date' in self.source_definition:
current_start = parse(self.source_definition['start_date'])
end_date = datetime.today() # End today by default
if 'end_date' in self.source_definition:
end_date = parse(self.source_definition['end_date'])
months = 6 # Max 6 months intervals by default
if 'months_interval' in self.source_definition:
months = self.source_definition['months_interval']
print "Getting all meetings for %s to %s" % (
current_start,
end_date,
)
meeting_count = 0
meeting_item_count = 0
while True:
current_end = current_start + relativedelta(months=months)
meetings = self.client.service.GetMeetingsByDateRange(
Sitename=self.source_definition['sitename'],
StartDate=current_start.strftime('%Y-%m-%dT%H:%M:%S'),
EndDate=current_end.strftime('%Y-%m-%dT%H:%M:%S'),
MetaDataOnly=False)
meeting_types = self._meetingtypes_as_dict()
if meetings.Meetings:
for meeting in meetings.Meetings[0]:
meeting_dict = meeting_to_dict(meeting)
#print meeting_types
# Getting the meeting type as a string is easier this way ...
meeting_dict['Meetingtype'] = meeting_types[
meeting_dict['MeetingtypeId']]
yield 'application/json', json.dumps(meeting_dict)
if meeting.MeetingItems is not None:
for meeting_item in meeting.MeetingItems[0]:
meeting_item_dict = meeting_item_to_dict(
meeting_item)
# This is a bit hacky, but we need to know this
meeting_item_dict['MeetingId'] = meeting_dict['Id']
meeting_item_dict['Meeting'] = meeting_dict
yield 'application/json', json.dumps(
meeting_item_dict)
meeting_item_count += 1
meeting_count += 1
print "Now processing meetings %s months from %s to %s" % (
months,
current_start,
current_end,
)
print "Extracted total of %d meetings and %d meeting items." % (
meeting_count,
meeting_item_count,
)
if current_end > end_date: # Stop while loop if exceeded end_date
break
current_start = current_end + relativedelta(days=1)
def run(self):
start_date = self.source_definition.get('start_date',
'2015-06-01T00:00:00')
end_date = self.source_definition.get('end_date',
'2016-07-01T00:00:00')
print "Getting meetings for %s to %s" % (
start_date,
end_date,
)
meetings = self.client.service.GetMeetingsByDateRange(
Sitename=self.source_definition['sitename'],
StartDate=start_date,
EndDate=end_date,
MetaDataOnly=False)
meeting_types = self._meetingtypes_as_dict()
meeting_count = 0
vote_count = 0
meeting_sorting_key = self.source_definition.get(
'meeting_sorting', 'MeetingDate')
sorted_meetings = sorted(meetings.Meetings[0],
lambda m: getaatr(m, meeting_sorting_key))
for meeting in sorted_meetings:
meeting_dict = meeting_to_dict(meeting)
# Getting the meeting type as a string is easier this way ...
pprint(meeting_dict['Id'])
meeting_dict['Meetingtype'] = meeting_types[
meeting_dict['MeetingtypeId']]
kv = self.client.factory.create('ns0:iBabsKeyValue')
kv.Key = 'IncludeMeetingItems'
kv.Value = True
kv2 = self.client.factory.create('ns0:iBabsKeyValue')
kv2.Key = 'IncludeListEntries'
kv2.Value = True
params = self.client.factory.create('ns0:ArrayOfiBabsKeyValue')
params.iBabsKeyValue.append(kv)
params.iBabsKeyValue.append(kv2)
vote_meeting = self.client.service.GetMeetingWithOptions(
Sitename=self.source_definition['sitename'],
MeetingId=meeting_dict['Id'],
Options=params)
meeting_dict_short = meeting_to_dict(vote_meeting.Meeting)
processed = []
for mi in meeting_dict_short['MeetingItems']:
if mi['ListEntries'] is None:
continue
for le in mi['ListEntries']:
# motion's unique identifier
print le['EntryTitle'].split(' ')[0][1:]
motion_id = normalize_motion_id(
le['EntryTitle'].split(' ')[0][1:])
print "Motie id : %s" % (motion_id.strip(), )
hash_content = u'motion-%s' % (motion_id.strip())
hashed_motion_id = unicode(
sha1(hash_content.decode('utf8')).hexdigest())
print "Hashedotie id : %s" % (hashed_motion_id.strip(), )
votes = self.client.service.GetListEntryVotes(
Sitename=self.source_definition['sitename'],
EntryId=le['EntryId'])
if votes.ListEntryVotes is None:
votes = []
else:
votes = votes_to_dict(votes.ListEntryVotes[0])
result = {
'motion_id': hashed_motion_id,
'meeting': meeting_dict,
'entry': le,
'votes': votes
}
vote_count += 1
if self.valid_meeting(result):
processed += self.process_meeting(result)
meeting_count += 1
#pprint(processed)
passed_vote_count = 0
for result in processed:
print "%s - %s" % (
result.get('id', '-'),
result.get('name', '-'),
)
yield 'application/json', json.dumps(result)
passed_vote_count += 1
print "Extracted %d meetings and passed %s out of %d voting rounds." % (
meeting_count,
passed_vote_count,
vote_count,
)