def test_local_to_utc(self):
# with day light saving on
local_tz = pytz.timezone('Australia/Sydney')
local_dt = datetime(2015, 2, 8, 8, 0, 0, 0, local_tz)
utc_dt = local_to_utc('Australia/Sydney', local_dt)
self.assertEqual(utc_dt.hour - local_dt.hour, 13)
# without day light saving
local_dt = local_tz.normalize(datetime(2015, 2, 8, 8, 0, 0, 0).replace(tzinfo=local_tz))
utc_dt = local_to_utc('Australia/Sydney', local_dt)
self.assertEqual(utc_dt.hour - local_dt.hour, 14)
def get_next_run(schedule, now_utc=None):
"""Get next run time based on schedule.
Schedule is day of week and time.
:param dict schedule: dict with `day_of_week` and `create_at` params
:param now_utc
:return datetime
"""
if not schedule.get('is_active', False):
return None
if now_utc is None:
now_utc = utcnow()
now_utc = now_utc.replace(second=0)
# Derive the first cron_list entry from the create_at and day_of_week
if 'create_at' in schedule and 'cron_list' not in schedule:
time = schedule.get('create_at').split(':')
cron_days = ','.join(schedule.get('day_of_week', '*')) if len(schedule.get('day_of_week')) else '*'
cron_entry = '{} {} * * {}'.format(time[1], time[0], cron_days)
schedule['cron_list'] = [cron_entry]
schedule.pop('create_at', None)
# adjust current time to the schedule's timezone
tz_name = schedule.get('time_zone', 'UTC')
if tz_name != 'UTC':
current_local_datetime = utc_to_local(tz_name, now_utc) # convert utc to local time
cron = croniter(schedule.get('cron_list')[0], current_local_datetime)
next_run = local_to_utc(tz_name, cron.get_next(datetime))
for cron_entry in schedule.get('cron_list'):
next_candidate = local_to_utc(tz_name, croniter(cron_entry, current_local_datetime).get_next(datetime))
if next_candidate < next_run:
next_run = next_candidate
else:
cron = croniter(schedule.get('cron_list')[0], now_utc)
next_run = cron.get_next(datetime)
for cron_entry in schedule.get('cron_list'):
next_candidate = croniter(cron_entry, now_utc).get_next(datetime)
if next_candidate < next_run:
next_run = next_candidate
return next_run
def parse(self, filename, provider=None):
default_item = self._set_default_item()
items = []
with open(filename, 'r', encoding='UTF-8') as f:
csv_reader = csv.reader(f)
for row in list(csv_reader)[1:]:
if not len(row):
continue
item = deepcopy(default_item)
item[GUID_FIELD] = ('urn:www.abs.gov.au:' + row[0].split(' ')[0] +
row[0].split(',')[-1]).replace('/', '-').replace(' ', '-')
if row[5] == 'true':
start = datetime.strptime('{} 11:30'.format(row[1]), '%d/%m/%Y %H:%M')
end = datetime.strptime('{} 11:30'.format(row[1]), '%d/%m/%Y %H:%M')
item['dates'] = {
'start': local_to_utc(config.DEFAULT_TIMEZONE, start),
'end': local_to_utc(config.DEFAULT_TIMEZONE, end),
'tz': config.DEFAULT_TIMEZONE,
}
item['name'] = ' '.join(row[0].split(' ')[1:])
item['definition_short'] = row[0]
items.append(item)
return items
def _parse_matches(self, xml, items):
for match in xml.findall('.//Matches/Match'):
try:
date = match.attrib.get('Date')
time = match.attrib.get('Start_Time')
match_id = match.attrib.get('Match_ID')
try:
when = datetime.strptime('{} {}'.format(date, time), '%Y-%m-%d %H:%M')
except Exception:
continue
if self._isDomestic(match_id):
match_id = match.attrib.get('Fixture_ID') # fixture id should be fine for domestic
if datetime.now() < when:
teamA = self.teams.get(match.attrib.get('TeamA_ID')).get('name')
teamB = self.teams.get(match.attrib.get('TeamB_ID')).get('name')
item = self._set_default_item(self.fixture.get('sport_id'), self.fixture.get('comp_id'), match_id)
if self._can_ingest_item(item):
item['name'] = '{} - {} v {}'.format(
self.sport_map.get(self.fixture.get('sport_id', {}), {}).get('name', ''), teamA, teamB)
item['definition_short'] = '{}/{}/{} {} v {}'.format(self.sport,
self.series,
self.round,
teamA,
teamB)
item['dates'] = {
'start': local_to_utc(config.DEFAULT_TIMEZONE, when),
'end': local_to_utc(config.DEFAULT_TIMEZONE, when) + timedelta(hours=2),
'tz': config.DEFAULT_TIMEZONE,
}
# add location
self._set_location(item, '{}, {}'.format(
self.venues.get(match.attrib.get('Venue_ID')).get('name'),
self.venues.get(match.attrib.get('Venue_ID')).get('location')))
items.append(item)
except Exception:
logger.exception('Failed to parse series fixtures.')
def update_schedule_settings(updates, field_name, value):
"""Calculates and sets the utc schedule for the given field.
:param updates: Where the time_zone information will be read and the updated
schedule_settings will be recorded
:param field_name: Name of he field: either publish_schedule or embargo
:param value: The original value
"""
schedule_settings = updates.get(SCHEDULE_SETTINGS, {}) or {}
utc_field_name = 'utc_{}'.format(field_name)
if field_name:
tz_name = schedule_settings.get('time_zone')
if tz_name:
schedule_settings[utc_field_name] = local_to_utc(tz_name, value)
else:
schedule_settings[utc_field_name] = value
schedule_settings['time_zone'] = None
updates[SCHEDULE_SETTINGS] = schedule_settings
def get_next_run(schedule, now=None):
"""Get next run time based on schedule.
Schedule is day of week and time.
:param dict schedule: dict with `day_of_week` and `create_at` params
:param datetime now
:return datetime
"""
if not schedule.get('is_active', False):
return None
allowed_days = [
Weekdays[day.upper()].value for day in schedule.get('day_of_week', [])
]
if not allowed_days:
return None
if now is None:
now = utcnow()
now = now.replace(second=0)
# adjust current time to the schedule's timezone
tz_name = schedule.get('time_zone', 'UTC')
if tz_name != 'UTC':
next_run = local_to_utc(tz_name,
set_time(now, schedule.get('create_at')))
else:
next_run = set_time(now, schedule.get('create_at'))
# if the time passed already today do it tomorrow earliest
if next_run <= now:
next_run += timedelta(days=1)
while next_run.weekday() not in allowed_days:
next_run += timedelta(days=1)
return next_run
def _parse_iso_date(date_str, timezone=None):
""" Create a date object from the given string in ISO 8601 format.
:param date_str:
:type date_str: str or None
:return: resulting date object or None if None is given
:rtype: datetime.date
:raises ValueError: if `date_str` is not in the ISO 8601 date format
"""
if date_str is None:
return None
else:
dt = parser.parse(date_str)
if dt.tzinfo is None:
if timezone:
if timezone not in pytz.all_timezones:
raise BadParameterValueError("Bad parameter value for Parameter (timezone)")
dt = local_to_utc(timezone, dt)
else:
dt = pytz.timezone('UTC').localize(dt)
return dt
def run(self, now=None):
lock_name = get_lock_id('planning', 'export_scheduled_filters')
if not lock(lock_name, expire=600):
logger.info('Export scheduled filters task is already running')
return
if now:
now_utc = now if isinstance(now, datetime) else local_to_utc(
app.config['DEFAULT_TIMEZONE'],
datetime.strptime(now, '%Y-%m-%dT%H'))
else:
now_utc = utcnow()
now_local = utc_to_local(app.config['DEFAULT_TIMEZONE'], now_utc)
# Set now to the beginning of the hour (in local time)
now_local = now_local.replace(minute=0, second=0, microsecond=0)
logger.info(f'Starting to export scheduled filters: {now_utc}')
self.process_filters(self.get_filters_with_schedules(), now_local,
now_utc)
unlock(lock_name)
logger.info(f'Completed sending scheduled exports: {now_utc}')
def get_next_run(schedule, now=None):
"""Get next run time based on schedule.
Schedule is day of week and time.
:param dict schedule: dict with `day_of_week` and `create_at` params
:param datetime now
:return datetime
"""
if not schedule.get('is_active', False):
return None
allowed_days = [Weekdays[day.upper()].value for day in schedule.get('day_of_week', [])]
if not allowed_days:
return None
if now is None:
now = utcnow()
now = now.replace(second=0)
# adjust current time to the schedule's timezone
tz_name = schedule.get('time_zone', 'UTC')
if tz_name != 'UTC':
next_run = local_to_utc(tz_name, set_time(now, schedule.get('create_at')))
else:
next_run = set_time(now, schedule.get('create_at'))
# if the time passed already today do it tomorrow earliest
if next_run <= now:
next_run += timedelta(days=1)
while next_run.weekday() not in allowed_days:
next_run += timedelta(days=1)
return next_run
def parse(self, cal, provider=None):
try:
items = []
for component in cal.walk():
if component.name == "VEVENT":
item = {
ITEM_TYPE: CONTENT_TYPE.EVENT,
GUID_FIELD: generate_guid(type=GUID_NEWSML),
FORMAT: FORMATS.PRESERVED
}
item['name'] = component.get('summary')
item['definition_short'] = component.get('summary')
item['definition_long'] = component.get('description')
item['original_source'] = component.get('uid')
item['state'] = CONTENT_STATE.INGESTED
item['pubstatus'] = None
eocstat_map = get_resource_service(
'vocabularies').find_one(req=None,
_id='eventoccurstatus')
if eocstat_map:
item['occur_status'] = [
x for x in eocstat_map.get('items', [])
if x['qcode'] == 'eocstat:eos5'
and x.get('is_active', True)
][0]
item['occur_status'].pop('is_active', None)
# add dates
# check if component .dt return date instead of datetime, if so, convert to datetime
dtstart = component.get('dtstart').dt
dates_start = dtstart if isinstance(dtstart, datetime.datetime) \
else datetime.datetime.combine(dtstart, datetime.datetime.min.time())
if not dates_start.tzinfo:
dates_start = local_to_utc(config.DEFAULT_TIMEZONE,
dates_start)
try:
dtend = component.get('dtend').dt
if isinstance(dtend, datetime.datetime):
dates_end = dtend
else: # Date only is non inclusive
dates_end = \
(datetime.datetime.combine(dtend, datetime.datetime.max.time()) -
datetime.timedelta(days=1)).replace(microsecond=0)
if not dates_end.tzinfo:
dates_end = local_to_utc(config.DEFAULT_TIMEZONE,
dates_end)
except AttributeError:
dates_end = None
item['dates'] = {
'start': dates_start,
'end': dates_end,
'tz': ''
}
# parse ics RRULE to fit eventsML recurring_rule
r_rule = component.get('rrule')
if isinstance(r_rule, vRecur):
r_rule_dict = vRecur.from_ical(r_rule)
if 'FREQ' in r_rule_dict.keys():
item['dates'].setdefault(
'recurring_rule', {})['frequency'] = ''.join(
r_rule_dict.get('FREQ'))
if 'INTERVAL' in r_rule_dict.keys():
item['dates'].setdefault(
'recurring_rule',
{})['interval'] = r_rule_dict.get(
'INTERVAL')[0]
if 'UNTIL' in r_rule_dict.keys():
item['dates'].setdefault(
'recurring_rule',
{})['until'] = r_rule_dict.get('UNTIL')[0]
if 'COUNT' in r_rule_dict.keys():
item['dates'].setdefault(
'recurring_rule',
{})['count'] = r_rule_dict.get('COUNT')
if 'BYMONTH' in r_rule_dict.keys():
item['dates'].setdefault(
'recurring_rule', {})['bymonth'] = ' '.join(
r_rule_dict.get('BYMONTH'))
if 'BYDAY' in r_rule_dict.keys():
item['dates'].setdefault(
'recurring_rule', {})['byday'] = ' '.join(
r_rule_dict.get('BYDAY'))
if 'BYHOUR' in r_rule_dict.keys():
item['dates'].setdefault(
'recurring_rule', {})['byhour'] = ' '.join(
r_rule_dict.get('BYHOUR'))
if 'BYMIN' in r_rule_dict.keys():
item['dates'].setdefault(
'recurring_rule', {})['bymin'] = ' '.join(
r_rule_dict.get('BYMIN'))
# set timezone info if date is a datetime
if isinstance(
component.get('dtstart').dt, datetime.datetime):
item['dates']['tz'] = tzid_from_dt(
component.get('dtstart').dt)
# add participants
item['participant'] = []
if component.get('attendee'):
for attendee in component.get('attendee'):
if isinstance(attendee, vCalAddress):
item['participant'].append({
'name':
vCalAddress.from_ical(attendee),
'qcode':
''
})
# add organizers
item['organizer'] = [{
'name': component.get('organizer', ''),
'qcode': ''
}]
# add location
item['location'] = [{
'name': component.get('location', ''),
'qcode': '',
'geo': ''
}]
if component.get('geo'):
item['location'][0]['geo'] = vGeo.from_ical(
component.get('geo').to_ical())
# IMPORTANT: firstcreated must be less than 2 days past
# we must preserve the original event created and updated in some other fields
if component.get('created'):
item['event_created'] = component.get('created').dt
if component.get('last-modified'):
item['event_lastmodified'] = component.get(
'last-modified').dt
item['firstcreated'] = utcnow()
item['versioncreated'] = utcnow()
items.append(item)
original_source_ids = [
_['original_source'] for _ in items
if _.get('original_source', None)
]
existing_items = list(
get_resource_service('events').get_from_mongo(
req=None,
lookup={'original_source': {
'$in': original_source_ids
}}))
def original_source_exists(item):
"""Return true if the item exists in `existing_items`"""
for c in existing_items:
if c['original_source'] == item['original_source']:
if c['dates']['start'] == item['dates']['start']:
return True
return False
def is_future(item):
"""Return true if the item is reccuring or in the future"""
if not item['dates'].get('recurring_rule'):
if item['dates']['start'] < utcnow() - datetime.timedelta(
days=1):
return False
return True
items = [_ for _ in items if is_future(_)]
items = [_ for _ in items if not original_source_exists(_)]
return items
except Exception as ex:
raise ParserError.parseMessageError(ex, provider)
def send_alerts(self, monitoring_list, created_from, created_from_time,
now):
general_settings = get_settings_collection().find_one(
GENERAL_SETTINGS_LOOKUP)
error_recipients = []
if general_settings and general_settings['values'].get(
'system_alerts_recipients'):
error_recipients = general_settings['values'][
'system_alerts_recipients'].split(',')
from newsroom.email import send_email
for m in monitoring_list:
if m.get('users'):
internal_req = ParsedRequest()
internal_req.args = {
'navigation': str(m['_id']),
'created_from': created_from,
'created_from_time': created_from_time,
'skip_user_validation': True
}
items = list(
get_resource_service('monitoring_search').get(
req=internal_req, lookup=None))
template_kwargs = {'profile': m}
if items:
company = get_entity_or_404(m['company'], 'companies')
try:
template_kwargs.update({
'items': items,
'section': 'wire',
})
truncate_article_body(items, m)
_file = get_monitoring_file(m, items)
attachment = base64.b64encode(_file.read())
formatter = app.download_formatters[
m['format_type']]['formatter']
# If there is only one story to send and the headline is to be used as the subject
if m.get('headline_subject',
False) and len(items) == 1:
subject = items[0].get(
'headline',
m.get('subject') or m['name'])
else:
subject = m.get('subject') or m['name']
send_email(
[
u['email'] for u in get_items_by_id(
[ObjectId(u) for u in m['users']], 'users')
],
subject,
text_body=render_template('monitoring_email.txt',
**template_kwargs),
html_body=render_template('monitoring_email.html',
**template_kwargs),
attachments_info=[{
'file':
attachment,
'file_name':
formatter.format_filename(None),
'content_type':
'application/{}'.format(
formatter.FILE_EXTENSION),
'file_desc':
'Monitoring Report for Celery monitoring alerts for profile: {}'
.format(m['name'])
}])
except Exception:
logger.exception(
'{0} Error processing monitoring profile {1} for company {2}.'
.format(self.log_msg, m['name'], company['name']))
if error_recipients:
# Send an email to admin
template_kwargs = {
'name': m['name'],
'company': company['name'],
'run_time': now,
}
send_email(
error_recipients,
gettext(
'Error sending alerts for monitoring: {0}'.
format(m['name'])),
text_body=render_template(
'monitoring_error.txt', **template_kwargs),
html_body=render_template(
'monitoring_error.html',
**template_kwargs),
)
elif m['schedule'].get('interval') != 'immediate' and m.get(
'always_send'):
send_email(
[
u['email'] for u in get_items_by_id(
[ObjectId(u) for u in m['users']], 'users')
],
m.get('subject') or m['name'],
text_body=render_template(
'monitoring_email_no_updates.txt',
**template_kwargs),
html_body=render_template(
'monitoring_email_no_updates.html',
**template_kwargs),
)
get_resource_service('monitoring').patch(m['_id'], {
'last_run_time':
local_to_utc(app.config['DEFAULT_TIMEZONE'], now)
})
def _parse_fixture_list(self, fixture, items):
xml = fixture.get('fixture_xml')
competition_detail = xml.find(
'.//Fixture_List/Competition/Competition_Details')
start_date = competition_detail.attrib.get('Start_Date', '')
end_date = competition_detail.attrib.get('End_Date', '')
comp_type = competition_detail.attrib.get('Comp_Type', '')
# A fixture list that the competition details provide the start and end date
if start_date != '' and end_date != '':
try:
self.season = competition_detail.attrib.get('Season', '')
start = datetime.strptime('{} 00:00'.format(start_date),
'%Y-%m-%d %H:%M')
end = datetime.strptime('{} 23:59'.format(end_date),
'%Y-%m-%d %H:%M')
event = xml.find(
'.//Fixture_List/Competition/Comp_Fixtures/Event')
match_id = None
if event is not None and event.attrib.get(
'Event_ID') is not None:
match_id = event.attrib.get('Event_ID')
if datetime.now() < end and match_id:
item = self._set_default_item(fixture.get('sport_id'),
fixture.get('comp_id'),
match_id)
if self._can_ingest_item(item):
item['name'] = '{} - {}'.format(
self.sport_map.get(fixture.get('sport_id', {}),
{}).get('name', ''),
fixture.get('comp_name'))
item[
'definition_short'] = competition_detail.attrib.get(
'Gender', '')
item['dates'] = {
'start': local_to_utc(config.DEFAULT_TIMEZONE,
start),
'end': local_to_utc(config.DEFAULT_TIMEZONE, end),
'tz': config.DEFAULT_TIMEZONE,
}
items.append(item)
except Exception as ex:
logger.exception('Failed to parse event fixtures.')
else:
# A fixture list that we need to pull out each match to determine the match date etc.
for match in xml.find(
'.//Fixture_List/Competition/Competition_Fixtures'):
start_date = match.find('.//Match_Details').attrib.get(
'Match_Date', '')
start_time = match.find('.//Match_Details').attrib.get(
'Match_Time', '')
self.season = match.find('.//Match_Details').attrib.get(
'Season', '')
try:
when = datetime.strptime(
'{} {}'.format(start_date, start_time),
'%Y-%m-%d %H:%M:%S')
except Exception as ex:
continue
if datetime.now() < when:
try:
match_id = match.find('.//Match_Details').attrib.get(
'Match_ID', '')
# Some match id's are not yet available
if match_id[-1] == '-':
continue
match_no = match.find('.//Match_Details').attrib.get(
'Match_No', '')
teamA_name = match.findall(
'.//Teams/Team_Details')[0].attrib.get(
'Team_Name', '')
teamB_name = match.findall(
'.//Teams/Team_Details')[1].attrib.get(
'Team_Name', '')
venue_name = match.find('.//Venue').attrib.get(
'Venue_Name', '')
venue_location = match.find('.//Venue').attrib.get(
'Venue_Location', '')
item = self._set_default_item(fixture.get('sport_id'),
fixture.get('comp_id'),
match_id)
if self._can_ingest_item(item):
item['name'] = '{} - {} v {}'.format(
self.sport_map.get(fixture.get('sport_id', {}),
{}).get('name', ''),
teamA_name, teamB_name)
item[
'definition_short'] = '{} match {} {} v {}'.format(
fixture.get('comp_name', ''), match_no,
teamA_name, teamB_name)
# kludge for cricket
if fixture.get('sport_id') == '3':
if 'test' in comp_type.lower():
delta = timedelta(days=5)
elif 'shef' in comp_type.lower():
delta = timedelta(days=4)
elif 't20' in comp_type.lower():
delta = timedelta(hours=4)
elif 'odi' in comp_type.lower(
) or 'odd' in comp_type.lower():
delta = timedelta(hours=8)
else:
delta = timedelta(hours=8)
else:
delta = timedelta(hours=2)
item['dates'] = {
'start':
local_to_utc(config.DEFAULT_TIMEZONE, when),
'end':
local_to_utc(config.DEFAULT_TIMEZONE, when) +
delta,
'tz':
config.DEFAULT_TIMEZONE,
}
# add location
self._set_location(
item, '{}, {}'.format(venue_name,
venue_location))
items.append(item)
except Exception as ex:
logger.exception(
'Failed to parse competition fixtures.')
def parse(self, data, provider=None):
index = self.parse_titles(data[0])
items = []
cells_list = [] # use for patch update to reduce write requests usage
# skip first two title rows
for row in range(3, len(data) + 1):
if not row:
break
item = {}
error_message = None
values = data[row - 1]
is_updated = values[index['_STATUS']].strip().upper(
) if len(values) - 1 > index['_STATUS'] else None
try:
# only insert item if _STATUS is empty
if is_updated in ('UPDATED', 'ERROR'):
guid = values[index['_GUID']]
# check if it's exists and guid is valid
if not superdesk.get_resource_service('events').find_one(
guid=guid, req=None):
raise KeyError('GUID is not exists')
else:
guid = generate_guid(type=GUID_NEWSML)
# avoid momentsJS throw null timezone value error
tzone = values[index['Timezone']] if values[
index['Timezone']] != 'none' else 'UTC'
start_datetime = parse(values[index['Start date']] + ' ' +
values[index['Start time']])
end_datetime = parse(values[index['End date']] + ' ' +
values[index['End time']])
if values[index['All day']] == 'TRUE':
start_datetime = parse(values[index['Start date']])
end_datetime = parse(
values[index['End date']]) + timedelta(days=1,
seconds=-1)
if end_datetime < start_datetime:
raise ValueError(
'End datetime is smaller than Start datetime')
item = {
'type': 'event',
'name': values[index['Event name']],
'slugline': values[index['Slugline']],
'dates': {
'start': local_to_utc(tzone, start_datetime),
'end': local_to_utc(tzone, end_datetime),
'tz': tzone,
},
'definition_short': values[index['Description']],
'definition_long': values[index['Long description']],
'internal_note': values[index['Internal note']],
'ednote': values[index['Ed note']],
'links': [values[index['External links']]],
'guid': guid,
'status': is_updated,
}
item.setdefault(ITEM_STATE, CONTENT_STATE.DRAFT)
occur_status = values[index['Occurence status']]
if occur_status and occur_status in self.occur_status_qcode_mapping:
item['occur_status'] = {
'qcode':
self.occur_status_qcode_mapping.get(
values[index['Occurence status']]),
'name':
values[index['Occurence status']],
'label':
values[index['Occurence status']].lower(),
}
calendars = values[index['Calendars']]
if calendars:
item['calendars'] = [{
'is_active': True,
'name': calendars,
'qcode': calendars.lower(),
}]
if all(values[index[field]]
for field in self.required_location_field):
item['location'] = [{
'name': values[index['Location Name']],
'address': {
'line': [values[index['Location Address']]],
'locality': values[index['Location City/Town']],
'area':
values[index['Location State/Province/Region']],
'country': values[index['Location Country']],
}
}]
if all(values[index[field]] for field in self.required_contact_field) \
and (all(values[index[field]] for field in ['Contact First name', 'Contact Last name'])
or values[index['Contact Organisation']]):
is_public = values[index['Contact Phone Public']] == 'TRUE'
if values[index['Contact Phone Usage']] == 'Confidential':
is_public = False
item['contact'] = {
'honorific':
values[index['Contact Honorific']],
'first_name':
values[index['Contact First name']],
'last_name':
values[index['Contact Last name']],
'organisation':
values[index['Contact Organisation']],
'contact_email': [values[index['Contact Email']]],
'contact_address':
[values[index['Contact Point of Contact']]],
'contact_phone': [{
'number':
values[index['Contact Phone Number']],
'public':
is_public,
'usage':
values[index['Contact Phone Usage']],
}]
}
# ignore invalid item
missing_fields = [
field for field in self.required_field
if not item.get(field)
]
if missing_fields:
missing_fields = ', '.join(missing_fields)
logger.error(
'Provider %s: Event "%s". Missing %s fields',
provider.get('name'),
item.get('name'),
missing_fields,
)
error_message = 'Missing ' + missing_fields + ' fields'
except UnknownTimeZoneError:
error_message = 'Invalid timezone'
logger.error('Provider %s: Event "%s": Invalid timezone %s',
provider.get('name'), values[index['Event name']],
tzone)
except (TypeError, ValueError, KeyError) as e:
error_message = e.args[0]
logger.error('Provider %s: Event "%s": %s',
provider.get('name'), item.get('name'),
error_message)
if error_message:
cells_list.extend([
Cell(row, index['_STATUS'] + 1, 'ERROR'),
Cell(row, index['_ERR_MESSAGE'] + 1, error_message)
])
elif not is_updated or is_updated == 'UPDATED':
cells_list.extend([
Cell(row, index['_STATUS'] + 1, 'DONE'),
Cell(row, index['_ERR_MESSAGE'] + 1, ''),
])
if not is_updated:
# only update _GUID when status is empty
cells_list.append(Cell(row, index['_GUID'] + 1, guid))
items.append(item)
return items, cells_list
def test_featured(client, app):
app.data.insert('products', [{
'_id': 12,
'name': 'product test',
'query': '_featured',
'companies': ['1'],
'navigations': ['51'],
'is_enabled': True,
'product_type': 'agenda',
}, {
'_id': 13,
'name': 'all items',
'query': '*:*',
'companies': ['1'],
'navigations': ['51'],
'is_enabled': True,
'product_type': 'agenda',
}])
_items = []
for i in range(5):
item = agenda_items[0].copy()
item['_id'] = 'urn:item:%d' % i
item['dates'] = item['dates'].copy()
item['dates']['start'] += timedelta(hours=1)
_items.append(item)
app.data.insert('agenda', _items)
# post first 2 items
date = utc_to_local('Australia/Sydney',
datetime.utcnow().replace(microsecond=0))
_id = date.strftime('%Y%m%d')
featured = {
'_id': _id,
'type': 'planning_featured',
'item_id': _id,
'items': [item['_id'] for item in _items[:2]],
'tz': 'Australia/Sydney',
}
resp = post_json(client, 'push', featured)
assert 200 == resp.status_code
# public user
with client.session_transaction() as session:
session['user'] = PUBLIC_USER_ID
session['user_type'] = 'public'
data = get_json(client, '/agenda/search?navigation=51')
assert 2 == data['_meta']['total']
assert _items[0]['_id'] == data['_items'][0]['_id']
assert _items[1]['_id'] == data['_items'][1]['_id']
assert '_aggregations' in data
assert data['_items'][0]['_display_from'].replace('+0000', '+00:00') == \
local_to_utc('Australia/Sydney', date.replace(hour=0, minute=0, second=0)).isoformat()
assert data['_items'][0]['_display_to'].replace('+0000', '+00:00') == \
local_to_utc('Australia/Sydney', date.replace(hour=23, minute=59, second=59)).isoformat()
# post first 3 items in reverse order
featured['items'] = [item['_id'] for item in _items[:3]]
featured['items'].reverse()
resp = post_json(client, 'push', featured)
assert 200 == resp.status_code
data = get_json(client, '/agenda/search?navigation=51')
assert 3 == data['_meta']['total']
assert _items[2]['_id'] == data['_items'][0]['_id']
assert _items[1]['_id'] == data['_items'][1]['_id']
assert _items[0]['_id'] == data['_items'][2]['_id']
data = get_json(client, '/agenda/search?navigation=51&q=slugline:nonsense')
assert 0 == data['_meta']['total']
# search with no nav - featured disabled
data = get_json(client, '/agenda/search')
assert len(_items) <= data['_meta']['total']
def _publish_date_filter(self, date_string):
local = dateutil.parser.parse(date_string)
return local_to_utc(self.TIMEZONE, local)
def _parse_doc(self, doc):
new_doc = {}
new_doc['_id'] = doc['refPtr']
new_doc['guid'] = doc['refPtr']
try:
new_doc['description_text'] = doc['caption']
except KeyError:
pass
try:
new_doc['headline'] = doc['headline']
except KeyError:
pass
try:
new_doc['original_source'] = new_doc['source'] = doc['credit']
except KeyError:
pass
new_doc['versioncreated'] = new_doc['firstcreated'] = self._datetime(
local_to_utc(SCANPIX_TZ, get_date(doc['archivedTime'])))
new_doc['pubstatus'] = 'usable'
# This must match the action
new_doc['_type'] = 'externalsource'
# entry that the client can use to identify the fetch endpoint
new_doc['fetch_endpoint'] = 'scanpix'
# mimetype is not directly found in Scanpix API
# so we use original filename to guess it
mimetype = mimetypes.guess_type("_{}".format(
splitext(doc.get('originalFileName', ''))[1]))[0]
if mimetype is None:
# nothing found with filename, we try out luck with fileFormat
try:
format_ = doc['fileFormat'].split()[0]
except (KeyError, IndexError):
mimetype = None
else:
mimetype = mimetypes.guess_type('_.{}'.format(format_))[0]
if mimetype is not None:
new_doc['mimetype'] = mimetype
main_group = doc['mainGroup']
if main_group == 'video':
new_doc[ITEM_TYPE] = CONTENT_TYPE.VIDEO
elif main_group == 'graphic':
new_doc[ITEM_TYPE] = CONTENT_TYPE.GRAPHIC
new_doc['mimetype'] = 'image/jpeg'
else:
new_doc[ITEM_TYPE] = CONTENT_TYPE.PICTURE
try:
doc_previews = doc['previews']
except KeyError:
logger.warning('no preview found for item {}'.format(
new_doc['_id']))
else:
# we look for best available scanpix preview
available_previews = [p['type'] for p in doc_previews]
renditions = new_doc['renditions'] = {}
for rend, previews in REND2PREV.items():
for prev in previews:
if prev in available_previews:
idx = available_previews.index(prev)
renditions[rend] = {"href": doc_previews[idx]['url']}
break
new_doc['byline'] = doc['byline']
doc.clear()
doc.update(new_doc)
def test_run_daily_jpeg(self, mocked):
with self.app.app_context():
self.app.data.insert('users', mock_users)
self.app.data.insert('vocabularies', mock_vocabs)
self.app.data.insert('saved_reports', mock_saved_reports)
self.app.data.insert('scheduled_reports',
[{
'_id': 'sched1',
'name': 'Scheduled Report',
'saved_report': 'srep1',
'schedule': {
'frequency': 'daily',
'hour': 1
},
'transmitter': 'email',
'mimetype': MIME_TYPES.JPEG,
'extra': {
'body': 'This is a test email'
},
'recipients': ['*****@*****.**'],
'report_width': 1200,
'active': True
}])
scheduled_service = get_resource_service('scheduled_reports')
report = scheduled_service.find_one(req=None, _id='sched1')
self.assertNotIn('_last_sent', report)
# Simulate running every hour for a few hours
start_date = to_naive('2018-06-30T00')
end_date = to_naive('2018-06-30T03')
should_have_updated = False
with self.app.mail.record_messages() as outbox:
for now in rrule(HOURLY, dtstart=start_date, until=end_date):
local_tz = pytz.timezone(app.config['DEFAULT_TIMEZONE'])
now_local = local_tz.localize(now)
now_utc = local_to_utc(app.config['DEFAULT_TIMEZONE'],
now_local)
SendScheduledReports().run(now_utc)
# _last sent is updated
report = scheduled_service.find_one(req=None, _id='sched1')
if not should_have_updated:
should_have_updated = True
self.assertNotIn('_last_sent', report)
else:
self.assertEqual(report.get('_last_sent'),
to_utc('2018-06-30T01'))
# Test that the command sent only 1 email
self.assertEqual(len(outbox), 1)
# Test attachment
self.assertEqual(len(outbox[0].attachments), 1)
self.assertEqual(
outbox[0].attachments[0].content_type,
'{}; name="chart_1.jpeg"'.format(MIME_TYPES.JPEG))
self.assertEqual(outbox[0].attachments[0].filename,
'chart_1.jpeg')
report = scheduled_service.find_one(req=None, _id='sched1')
self.assertEqual(report.get('_last_sent'), to_utc('2018-06-30T01'))
def parse_datetime(self, value):
if not value:
return None
local = datetime.strptime(value, '%m/%d/%Y %H:%M:%S %p')
return local_to_utc(self.TZ, local)
def test_local_to_utc_europe(self):
utc_dt = local_to_utc('Europe/Prague', datetime(2016, 4, 19, 15, 8, 0))
self.assertEqual('2016-04-19T13:08:00+00:00', utc_dt.isoformat())
def update_recurring_events(self, updates, original, update_method):
historic, past, future = self.get_recurring_timeline(original)
# Determine if the selected event is the first one, if so then
# act as if we're changing future events
if len(historic) == 0 and len(past) == 0:
update_method = UPDATE_FUTURE
if update_method == UPDATE_FUTURE:
new_series = [original] + future
else:
new_series = past + [original] + future
# Release the Lock on the selected Event
remove_lock_information(updates)
# Get the timezone from the original Event (as the series was created with that timezone in mind)
timezone = original['dates']['tz']
# First find the hour and minute of the start date in local time
start_time = utc_to_local(timezone, updates['dates']['start']).time()
# Next convert that to seconds since midnight (which gives us a timedelta instance)
delta_since_midnight = datetime.combine(date.min,
start_time) - datetime.min
# And calculate the new duration of the events
duration = updates['dates']['end'] - updates['dates']['start']
for event in new_series:
if not event.get(config.ID_FIELD):
continue
new_updates = {'dates': deepcopy(event['dates'])} \
if event.get(config.ID_FIELD) != original.get(config.ID_FIELD) else updates
# Calculate midnight in local time for this occurrence
start_of_day_local = utc_to_local(timezone, event['dates']['start'])\
.replace(hour=0, minute=0, second=0)
# Then convert midnight in local time to UTC
start_date_time = local_to_utc(timezone, start_of_day_local)
# Finally add the delta since midnight
start_date_time += delta_since_midnight
# Set the new start and end times
new_updates['dates']['start'] = start_date_time
new_updates['dates']['end'] = start_date_time + duration
if event.get(TO_BE_CONFIRMED_FIELD):
new_updates[TO_BE_CONFIRMED_FIELD] = False
# Set '_planning_schedule' on the Event item
self.set_planning_schedule(new_updates)
if event.get(config.ID_FIELD) != original.get(config.ID_FIELD):
new_updates['skip_on_update'] = True
self.patch(event[config.ID_FIELD], new_updates)
app.on_updated_events_update_time(
new_updates, {'_id': event[config.ID_FIELD]})
def to_utc(date_str):
return local_to_utc(
app.config['DEFAULT_TIMEZONE'],
datetime.strptime(date_str, '%Y-%m-%dT%H')
)
def _process_sheet(self, title, dates, values, _id):
i = 0
for date in dates:
start_date = self.epoch + timedelta(days=date)
dt = start_date
end_dt = None
if values[i][1] and values[i][1].lower() != 'tbc':
try:
hours = values[i][1].split(':')[0]
minutes = values[i][1].split(':')[1]
dt = dt + timedelta(hours=int(hours), minutes=int(minutes))
except:
pass
if values[i][2] and values[i][2].lower() != 'tbc':
try:
hours = values[i][2].split(':')[0]
minutes = values[i][2].split(':')[1]
end_dt = start_date + timedelta(hours=int(hours), minutes=int(minutes))
except:
pass
v = values[i]
item = self._set_default_item(title, _id, hashlib.sha1('-'.join(v).encode('utf8')).hexdigest(),
v[7] if v[7] else 'australia')
self._set_location(item, '{} {} {} {}'.format(v[4], v[5], v[6], v[7]))
sheet_country = self._set_country(v[7])
sheet_city = v[5].lower()
loc = sheet_city + '/' + sheet_country
tz = self.tz_map.get(loc, config.DEFAULT_TIMEZONE)
item['name'] = v[3]
item['definition_short'] = v[3]
item['source'] = 'AAP Sports Sheet'
if end_dt:
item['dates'] = {
'start': local_to_utc(tz, dt),
'end': local_to_utc(tz, end_dt),
'tz': tz,
}
else:
item['dates'] = {
'start': local_to_utc(tz, dt),
'end': local_to_utc(tz, dt) + timedelta(hours=23, minutes=59, seconds=59),
'tz': tz,
}
# print('{} {} {} {} {}'.format(title, item['name'], item.get('dates').get('start'),
# item.get('dates').get('end'), tz))
# print('{}-{} city:[{}] state:[{}] county[{}] ---> {}'.format(title, item['name'], v[5], v[6], v[7], tz))
print('{},{},{},{},{},{}'.format(title.replace(',', ' '), item['name'].replace(',', ' '), v[5], v[6], v[7],
tz))
old = superdesk.get_resource_service('events').find_one(req=None, guid=item['guid'])
if old:
superdesk.get_resource_service('events').patch(old.get('_id'), item)
else:
superdesk.get_resource_service('events').post([item])
i += 1
def _parse_fixture_list(self, fixture, items):
xml = fixture.get('fixture_xml')
competition_detail = xml.find('.//Fixture_List/Competition/Competition_Details')
start_date = competition_detail.attrib.get('Start_Date', '')
end_date = competition_detail.attrib.get('End_Date', '')
comp_type = competition_detail.attrib.get('Comp_Type', '')
if comp_type == '':
if 'Test' in fixture.get('comp_name'):
comp_type = 'test'
elif 'One Day' in fixture.get('comp_name'):
comp_type = 'odi'
# A fixture list that the competition details provide the start and end date
if start_date != '' and end_date != '':
try:
self.season = competition_detail.attrib.get('Season', '')
start = datetime.strptime('{} 00:00'.format(start_date), '%Y-%m-%d %H:%M')
end = datetime.strptime('{} 23:59'.format(end_date), '%Y-%m-%d %H:%M')
event = xml.find('.//Fixture_List/Competition/Comp_Fixtures/Event')
match_id = None
if event is not None and event.attrib.get('Event_ID') is not None:
match_id = event.attrib.get('Event_ID')
if datetime.now() < end and match_id:
item = self._set_default_item(fixture.get('sport_id'), fixture.get('comp_id'), match_id)
if self._can_ingest_item(item):
item['name'] = '{} - {}'.format(
self.sport_map.get(fixture.get('sport_id', {}), {}).get('name', ''),
fixture.get('comp_name')
)
item['definition_short'] = competition_detail.attrib.get('Gender', '')
item['dates'] = {
'start': local_to_utc(config.DEFAULT_TIMEZONE, start),
'end': local_to_utc(config.DEFAULT_TIMEZONE, end),
'tz': config.DEFAULT_TIMEZONE,
}
items.append(item)
except Exception:
logger.exception('Failed to parse event fixtures.')
else:
# A fixture list that we need to pull out each match to determine the match date etc.
for match in xml.find('.//Fixture_List/Competition/Competition_Fixtures'):
start_date = match.find('.//Match_Details').attrib.get('Match_Date', '')
start_time = match.find('.//Match_Details').attrib.get('Match_Time', '')
self.season = match.find('.//Match_Details').attrib.get('Season', '')
try:
when = datetime.strptime('{} {}'.format(start_date, start_time), '%Y-%m-%d %H:%M:%S')
except Exception:
continue
if datetime.now() < when:
try:
match_id = match.find('.//Match_Details').attrib.get('Match_ID', '')
# Some match id's are not yet available
if match_id[-1] == '-':
continue
match_no = match.find('.//Match_Details').attrib.get('Match_No', '')
teamA_name = match.findall('.//Teams/Team_Details')[0].attrib.get('Team_Name', '')
teamB_name = match.findall('.//Teams/Team_Details')[1].attrib.get('Team_Name', '')
venue_name = match.find('.//Venue').attrib.get('Venue_Name', '')
venue_location = match.find('.//Venue').attrib.get('Venue_Location', '')
item = self._set_default_item(fixture.get('sport_id'), fixture.get('comp_id'), match_id)
if self._can_ingest_item(item):
item['name'] = '{} - {} v {}'.format(
self.sport_map.get(fixture.get('sport_id', {}), {}).get('name', ''), teamA_name,
teamB_name)
item['definition_short'] = '{} match {} {} v {}'.format(fixture.get('comp_name', ''),
match_no,
teamA_name,
teamB_name)
# kludge for cricket
if fixture.get('sport_id') == '3':
if 'test' in comp_type.lower():
delta = timedelta(days=5)
elif 'shef' in comp_type.lower():
delta = timedelta(days=4)
elif 't20' in comp_type.lower():
delta = timedelta(hours=4)
elif 'odi' in comp_type.lower() or 'odd' in comp_type.lower():
delta = timedelta(hours=8)
else:
delta = timedelta(hours=8)
else:
delta = timedelta(hours=2)
item['dates'] = {
'start': local_to_utc(config.DEFAULT_TIMEZONE, when),
'end': local_to_utc(config.DEFAULT_TIMEZONE, when) + delta,
'tz': config.DEFAULT_TIMEZONE,
}
# add location
self._set_location(item, '{}, {}'.format(venue_name, venue_location))
items.append(item)
except Exception:
logger.exception('Failed to parse competition fixtures.')
def _parse_date(self, value):
dt = datetime.strptime(value, DATETIME_FORMAT)
return local_to_utc(TIMEZONE, dt)
def datetime(self, value):
"""When there is no timezone info, assume it's Helsinki timezone."""
parsed = super().datetime(value)
if '+' not in value:
return local_to_utc(TIMEZONE, parsed)
return parsed
def test_run_hourly_png(self, mocked):
with self.app.app_context():
self.app.data.insert('users', mock_users)
self.app.data.insert('vocabularies', mock_vocabs)
self.app.data.insert('saved_reports', mock_saved_reports)
self.app.data.insert('scheduled_reports',
[{
'_id': 'sched1',
'name': 'Scheduled Report',
'saved_report': 'srep1',
'schedule': {
'frequency': 'hourly'
},
'transmitter': 'email',
'mimetype': MIME_TYPES.PNG,
'extra': {
'body': 'This is a test email'
},
'recipients': ['*****@*****.**'],
'report_width': 1200,
'active': True
}])
scheduled_service = get_resource_service('scheduled_reports')
report = scheduled_service.find_one(req=None, _id='sched1')
self.assertNotIn('_last_sent', report)
# Simulate running every hour for a few hours
start_date = to_naive('2018-06-30T00')
end_date = to_naive('2018-06-30T03')
local_tz = pytz.timezone(app.config['DEFAULT_TIMEZONE'])
with self.app.mail.record_messages() as outbox:
for now in rrule(HOURLY, dtstart=start_date, until=end_date):
now_local = local_tz.localize(now)
now_utc = local_to_utc(app.config['DEFAULT_TIMEZONE'],
now_local)
SendScheduledReports().run(now_utc)
# _last sent is updated
report = scheduled_service.find_one(req=None, _id='sched1')
self.assertEqual(report.get('_last_sent'), now_utc)
# Test that the command sent emails across the 4 iterations
self.assertEqual(len(outbox), 4)
# Test the first email
self.assertEqual(outbox[0].sender, '*****@*****.**')
self.assertEqual(outbox[0].subject,
'Superdesk Analytics - Scheduled Report')
self.assertTrue(
outbox[0].body.startswith('\nThis is a test email'))
self.assertTrue('This is a test email' in outbox[0].html)
self.assertTrue('<img src="cid:' in outbox[0].html)
self.assertEqual(outbox[0].recipients,
['*****@*****.**'])
# Test attachment
self.assertEqual(len(outbox[0].attachments), 1)
self.assertEqual(outbox[0].attachments[0].data,
b64decode(mock_file))
self.assertEqual(
outbox[0].attachments[0].content_type,
'{}; name="chart_1.png"'.format(MIME_TYPES.PNG))
self.assertEqual(outbox[0].attachments[0].filename,
'chart_1.png')
report = scheduled_service.find_one(req=None, _id='sched1')
self.assertEqual(report.get('_last_sent'), to_utc('2018-06-30T03'))
def test_local_to_utc_europe(self):
utc_dt = local_to_utc('Europe/Prague', datetime(2016, 4, 19, 15, 8, 0))
self.assertEqual('2016-04-19T13:08:00+00:00', utc_dt.isoformat())
def _parse_date(self, string):
local = datetime.strptime(string, '%Y%m%d %H:%M:%S')
return local_to_utc(TZ, local)
def parse_newsmanagement(self, item, manage_el):
super().parse_newsmanagement(item, manage_el)
tz = 'Europe/Moscow'
item['firstcreated'] = local_to_utc(tz, item['firstcreated'])
item['versioncreated'] = local_to_utc(tz, item['firstcreated'])
def send_alerts(self, monitoring_list, created_from, created_from_time,
now):
general_settings = get_settings_collection().find_one(
GENERAL_SETTINGS_LOOKUP)
error_recipients = []
if general_settings and general_settings['values'].get(
'system_alerts_recipients'):
error_recipients = general_settings['values'][
'system_alerts_recipients'].split(',')
from newsroom.email import send_email
for m in monitoring_list:
# if immediate set the created from to the time of the last item sent, if available
if m.get('schedule', {}).get('interval') == 'immediate' and m.get(
'last_run_time', False):
created_from = m.get('last_run_time').strftime('%Y-%m-%d')
created_from_time = m.get('last_run_time').strftime('%H:%M:%S')
last_run_time = local_to_utc(app.config['DEFAULT_TIMEZONE'], now)
company = get_entity_or_404(m['company'], 'companies')
user_list = self.filter_users(m, company)
if len(user_list) and company.get('is_enabled'):
internal_req = ParsedRequest()
internal_req.args = {
'navigation': str(m['_id']),
'created_from': created_from,
'created_from_time': created_from_time,
'skip_user_validation': True
}
items = list(
get_resource_service('monitoring_search').get(
req=internal_req, lookup=None))
items[:] = [
item for item in items if not self.already_sent(item, m)
]
template_kwargs = {'profile': m}
if items:
try:
template_kwargs.update({
'items': items,
'section': 'wire',
})
truncate_article_body(items, m)
# If there is only one story to send and the headline is to be used as the subject
if m.get('headline_subject',
False) and len(items) == 1:
subject = items[0].get(
'headline',
m.get('subject') or m['name'])
else:
subject = m.get('subject') or m['name']
if m.get('format_type') == 'monitoring_email':
self.send_email_alert(items, subject, m, user_list)
else:
_file = get_monitoring_file(m, items)
attachment = base64.b64encode(_file.read())
formatter = app.download_formatters[
m['format_type']]['formatter']
send_email(
user_list,
subject,
text_body=render_template(
'monitoring_email.txt', **template_kwargs),
html_body=render_template(
'monitoring_email.html',
**template_kwargs),
attachments_info=[{
'file':
attachment,
'file_name':
formatter.format_filename(None),
'content_type':
'application/{}'.format(
formatter.FILE_EXTENSION),
'file_desc':
'Monitoring Report for Celery monitoring alerts for profile: {}'
.format(m['name'])
}])
get_resource_service('history').create_history_record(
items,
action='email',
user={
'_id': None,
'company': m['company']
},
section='monitoring',
monitoring=m['_id'])
except Exception:
logger.exception(
'{0} Error processing monitoring profile {1} for company {2}.'
.format(self.log_msg, m['name'], company['name']))
if error_recipients:
# Send an email to admin
template_kwargs = {
'name': m['name'],
'company': company['name'],
'run_time': now,
}
send_email(
error_recipients,
gettext(
'Error sending alerts for monitoring: {0}'.
format(m['name'])),
text_body=render_template(
'monitoring_error.txt', **template_kwargs),
html_body=render_template(
'monitoring_error.html',
**template_kwargs),
)
elif m['schedule'].get('interval') != 'immediate' and m.get(
'always_send'):
send_email(
user_list,
m.get('subject') or m['name'],
text_body=render_template(
'monitoring_email_no_updates.txt',
**template_kwargs),
html_body=render_template(
'monitoring_email_no_updates.html',
**template_kwargs),
)
if m.get('schedule',
{}).get('interval') == 'immediate' and len(items):
# determine the version created time for the most recent item
last_article_created = max(
item.get('versioncreated') for item in items)
# last run time is either the time of the last sent item or the last run time.
last_run_time = last_article_created if last_article_created else last_run_time
get_resource_service('monitoring').patch(
m['_id'], {'last_run_time': last_run_time})
def convert_date(epoch):
dt = local_to_utc(config.DEFAULT_TIMEZONE, datetime.fromtimestamp(int(str(epoch)[:10])))
return dt
def _publish_date_filter(self, date_string):
local = dateutil.parser.parse(date_string)
return local_to_utc(self.TIMEZONE, local)
def get_datetime(value):
dt = arrow.get(value).datetime
return local_to_utc(BELGA_TZ, dt)
def datetime(self, string):
dt = parse(string).replace(tzinfo=utc)
return local_to_utc(TIMEZONE, dt)
