def test_local_to_utc(self): # with day light saving on local_tz = pytz.timezone('Australia/Sydney') local_dt = datetime(2015, 2, 8, 8, 0, 0, 0, local_tz) utc_dt = local_to_utc('Australia/Sydney', local_dt) self.assertEqual(utc_dt.hour - local_dt.hour, 13) # without day light saving local_dt = local_tz.normalize(datetime(2015, 2, 8, 8, 0, 0, 0).replace(tzinfo=local_tz)) utc_dt = local_to_utc('Australia/Sydney', local_dt) self.assertEqual(utc_dt.hour - local_dt.hour, 14)
def get_next_run(schedule, now_utc=None): """Get next run time based on schedule. Schedule is day of week and time. :param dict schedule: dict with `day_of_week` and `create_at` params :param now_utc :return datetime """ if not schedule.get('is_active', False): return None if now_utc is None: now_utc = utcnow() now_utc = now_utc.replace(second=0) # Derive the first cron_list entry from the create_at and day_of_week if 'create_at' in schedule and 'cron_list' not in schedule: time = schedule.get('create_at').split(':') cron_days = ','.join(schedule.get('day_of_week', '*')) if len(schedule.get('day_of_week')) else '*' cron_entry = '{} {} * * {}'.format(time[1], time[0], cron_days) schedule['cron_list'] = [cron_entry] schedule.pop('create_at', None) # adjust current time to the schedule's timezone tz_name = schedule.get('time_zone', 'UTC') if tz_name != 'UTC': current_local_datetime = utc_to_local(tz_name, now_utc) # convert utc to local time cron = croniter(schedule.get('cron_list')[0], current_local_datetime) next_run = local_to_utc(tz_name, cron.get_next(datetime)) for cron_entry in schedule.get('cron_list'): next_candidate = local_to_utc(tz_name, croniter(cron_entry, current_local_datetime).get_next(datetime)) if next_candidate < next_run: next_run = next_candidate else: cron = croniter(schedule.get('cron_list')[0], now_utc) next_run = cron.get_next(datetime) for cron_entry in schedule.get('cron_list'): next_candidate = croniter(cron_entry, now_utc).get_next(datetime) if next_candidate < next_run: next_run = next_candidate return next_run
def parse(self, filename, provider=None): default_item = self._set_default_item() items = [] with open(filename, 'r', encoding='UTF-8') as f: csv_reader = csv.reader(f) for row in list(csv_reader)[1:]: if not len(row): continue item = deepcopy(default_item) item[GUID_FIELD] = ('urn:www.abs.gov.au:' + row[0].split(' ')[0] + row[0].split(',')[-1]).replace('/', '-').replace(' ', '-') if row[5] == 'true': start = datetime.strptime('{} 11:30'.format(row[1]), '%d/%m/%Y %H:%M') end = datetime.strptime('{} 11:30'.format(row[1]), '%d/%m/%Y %H:%M') item['dates'] = { 'start': local_to_utc(config.DEFAULT_TIMEZONE, start), 'end': local_to_utc(config.DEFAULT_TIMEZONE, end), 'tz': config.DEFAULT_TIMEZONE, } item['name'] = ' '.join(row[0].split(' ')[1:]) item['definition_short'] = row[0] items.append(item) return items
def _parse_matches(self, xml, items): for match in xml.findall('.//Matches/Match'): try: date = match.attrib.get('Date') time = match.attrib.get('Start_Time') match_id = match.attrib.get('Match_ID') try: when = datetime.strptime('{} {}'.format(date, time), '%Y-%m-%d %H:%M') except Exception: continue if self._isDomestic(match_id): match_id = match.attrib.get('Fixture_ID') # fixture id should be fine for domestic if datetime.now() < when: teamA = self.teams.get(match.attrib.get('TeamA_ID')).get('name') teamB = self.teams.get(match.attrib.get('TeamB_ID')).get('name') item = self._set_default_item(self.fixture.get('sport_id'), self.fixture.get('comp_id'), match_id) if self._can_ingest_item(item): item['name'] = '{} - {} v {}'.format( self.sport_map.get(self.fixture.get('sport_id', {}), {}).get('name', ''), teamA, teamB) item['definition_short'] = '{}/{}/{} {} v {}'.format(self.sport, self.series, self.round, teamA, teamB) item['dates'] = { 'start': local_to_utc(config.DEFAULT_TIMEZONE, when), 'end': local_to_utc(config.DEFAULT_TIMEZONE, when) + timedelta(hours=2), 'tz': config.DEFAULT_TIMEZONE, } # add location self._set_location(item, '{}, {}'.format( self.venues.get(match.attrib.get('Venue_ID')).get('name'), self.venues.get(match.attrib.get('Venue_ID')).get('location'))) items.append(item) except Exception: logger.exception('Failed to parse series fixtures.')
def update_schedule_settings(updates, field_name, value): """Calculates and sets the utc schedule for the given field. :param updates: Where the time_zone information will be read and the updated schedule_settings will be recorded :param field_name: Name of he field: either publish_schedule or embargo :param value: The original value """ schedule_settings = updates.get(SCHEDULE_SETTINGS, {}) or {} utc_field_name = 'utc_{}'.format(field_name) if field_name: tz_name = schedule_settings.get('time_zone') if tz_name: schedule_settings[utc_field_name] = local_to_utc(tz_name, value) else: schedule_settings[utc_field_name] = value schedule_settings['time_zone'] = None updates[SCHEDULE_SETTINGS] = schedule_settings
def get_next_run(schedule, now=None): """Get next run time based on schedule. Schedule is day of week and time. :param dict schedule: dict with `day_of_week` and `create_at` params :param datetime now :return datetime """ if not schedule.get('is_active', False): return None allowed_days = [ Weekdays[day.upper()].value for day in schedule.get('day_of_week', []) ] if not allowed_days: return None if now is None: now = utcnow() now = now.replace(second=0) # adjust current time to the schedule's timezone tz_name = schedule.get('time_zone', 'UTC') if tz_name != 'UTC': next_run = local_to_utc(tz_name, set_time(now, schedule.get('create_at'))) else: next_run = set_time(now, schedule.get('create_at')) # if the time passed already today do it tomorrow earliest if next_run <= now: next_run += timedelta(days=1) while next_run.weekday() not in allowed_days: next_run += timedelta(days=1) return next_run
def _parse_iso_date(date_str, timezone=None): """ Create a date object from the given string in ISO 8601 format. :param date_str: :type date_str: str or None :return: resulting date object or None if None is given :rtype: datetime.date :raises ValueError: if `date_str` is not in the ISO 8601 date format """ if date_str is None: return None else: dt = parser.parse(date_str) if dt.tzinfo is None: if timezone: if timezone not in pytz.all_timezones: raise BadParameterValueError("Bad parameter value for Parameter (timezone)") dt = local_to_utc(timezone, dt) else: dt = pytz.timezone('UTC').localize(dt) return dt
def run(self, now=None): lock_name = get_lock_id('planning', 'export_scheduled_filters') if not lock(lock_name, expire=600): logger.info('Export scheduled filters task is already running') return if now: now_utc = now if isinstance(now, datetime) else local_to_utc( app.config['DEFAULT_TIMEZONE'], datetime.strptime(now, '%Y-%m-%dT%H')) else: now_utc = utcnow() now_local = utc_to_local(app.config['DEFAULT_TIMEZONE'], now_utc) # Set now to the beginning of the hour (in local time) now_local = now_local.replace(minute=0, second=0, microsecond=0) logger.info(f'Starting to export scheduled filters: {now_utc}') self.process_filters(self.get_filters_with_schedules(), now_local, now_utc) unlock(lock_name) logger.info(f'Completed sending scheduled exports: {now_utc}')
def get_next_run(schedule, now=None): """Get next run time based on schedule. Schedule is day of week and time. :param dict schedule: dict with `day_of_week` and `create_at` params :param datetime now :return datetime """ if not schedule.get('is_active', False): return None allowed_days = [Weekdays[day.upper()].value for day in schedule.get('day_of_week', [])] if not allowed_days: return None if now is None: now = utcnow() now = now.replace(second=0) # adjust current time to the schedule's timezone tz_name = schedule.get('time_zone', 'UTC') if tz_name != 'UTC': next_run = local_to_utc(tz_name, set_time(now, schedule.get('create_at'))) else: next_run = set_time(now, schedule.get('create_at')) # if the time passed already today do it tomorrow earliest if next_run <= now: next_run += timedelta(days=1) while next_run.weekday() not in allowed_days: next_run += timedelta(days=1) return next_run
def parse(self, cal, provider=None): try: items = [] for component in cal.walk(): if component.name == "VEVENT": item = { ITEM_TYPE: CONTENT_TYPE.EVENT, GUID_FIELD: generate_guid(type=GUID_NEWSML), FORMAT: FORMATS.PRESERVED } item['name'] = component.get('summary') item['definition_short'] = component.get('summary') item['definition_long'] = component.get('description') item['original_source'] = component.get('uid') item['state'] = CONTENT_STATE.INGESTED item['pubstatus'] = None eocstat_map = get_resource_service( 'vocabularies').find_one(req=None, _id='eventoccurstatus') if eocstat_map: item['occur_status'] = [ x for x in eocstat_map.get('items', []) if x['qcode'] == 'eocstat:eos5' and x.get('is_active', True) ][0] item['occur_status'].pop('is_active', None) # add dates # check if component .dt return date instead of datetime, if so, convert to datetime dtstart = component.get('dtstart').dt dates_start = dtstart if isinstance(dtstart, datetime.datetime) \ else datetime.datetime.combine(dtstart, datetime.datetime.min.time()) if not dates_start.tzinfo: dates_start = local_to_utc(config.DEFAULT_TIMEZONE, dates_start) try: dtend = component.get('dtend').dt if isinstance(dtend, datetime.datetime): dates_end = dtend else: # Date only is non inclusive dates_end = \ (datetime.datetime.combine(dtend, datetime.datetime.max.time()) - datetime.timedelta(days=1)).replace(microsecond=0) if not dates_end.tzinfo: dates_end = local_to_utc(config.DEFAULT_TIMEZONE, dates_end) except AttributeError: dates_end = None item['dates'] = { 'start': dates_start, 'end': dates_end, 'tz': '' } # parse ics RRULE to fit eventsML recurring_rule r_rule = component.get('rrule') if isinstance(r_rule, vRecur): r_rule_dict = vRecur.from_ical(r_rule) if 'FREQ' in r_rule_dict.keys(): item['dates'].setdefault( 'recurring_rule', {})['frequency'] = ''.join( r_rule_dict.get('FREQ')) if 'INTERVAL' in r_rule_dict.keys(): item['dates'].setdefault( 'recurring_rule', {})['interval'] = r_rule_dict.get( 'INTERVAL')[0] if 'UNTIL' in r_rule_dict.keys(): item['dates'].setdefault( 'recurring_rule', {})['until'] = r_rule_dict.get('UNTIL')[0] if 'COUNT' in r_rule_dict.keys(): item['dates'].setdefault( 'recurring_rule', {})['count'] = r_rule_dict.get('COUNT') if 'BYMONTH' in r_rule_dict.keys(): item['dates'].setdefault( 'recurring_rule', {})['bymonth'] = ' '.join( r_rule_dict.get('BYMONTH')) if 'BYDAY' in r_rule_dict.keys(): item['dates'].setdefault( 'recurring_rule', {})['byday'] = ' '.join( r_rule_dict.get('BYDAY')) if 'BYHOUR' in r_rule_dict.keys(): item['dates'].setdefault( 'recurring_rule', {})['byhour'] = ' '.join( r_rule_dict.get('BYHOUR')) if 'BYMIN' in r_rule_dict.keys(): item['dates'].setdefault( 'recurring_rule', {})['bymin'] = ' '.join( r_rule_dict.get('BYMIN')) # set timezone info if date is a datetime if isinstance( component.get('dtstart').dt, datetime.datetime): item['dates']['tz'] = tzid_from_dt( component.get('dtstart').dt) # add participants item['participant'] = [] if component.get('attendee'): for attendee in component.get('attendee'): if isinstance(attendee, vCalAddress): item['participant'].append({ 'name': vCalAddress.from_ical(attendee), 'qcode': '' }) # add organizers item['organizer'] = [{ 'name': component.get('organizer', ''), 'qcode': '' }] # add location item['location'] = [{ 'name': component.get('location', ''), 'qcode': '', 'geo': '' }] if component.get('geo'): item['location'][0]['geo'] = vGeo.from_ical( component.get('geo').to_ical()) # IMPORTANT: firstcreated must be less than 2 days past # we must preserve the original event created and updated in some other fields if component.get('created'): item['event_created'] = component.get('created').dt if component.get('last-modified'): item['event_lastmodified'] = component.get( 'last-modified').dt item['firstcreated'] = utcnow() item['versioncreated'] = utcnow() items.append(item) original_source_ids = [ _['original_source'] for _ in items if _.get('original_source', None) ] existing_items = list( get_resource_service('events').get_from_mongo( req=None, lookup={'original_source': { '$in': original_source_ids }})) def original_source_exists(item): """Return true if the item exists in `existing_items`""" for c in existing_items: if c['original_source'] == item['original_source']: if c['dates']['start'] == item['dates']['start']: return True return False def is_future(item): """Return true if the item is reccuring or in the future""" if not item['dates'].get('recurring_rule'): if item['dates']['start'] < utcnow() - datetime.timedelta( days=1): return False return True items = [_ for _ in items if is_future(_)] items = [_ for _ in items if not original_source_exists(_)] return items except Exception as ex: raise ParserError.parseMessageError(ex, provider)
def send_alerts(self, monitoring_list, created_from, created_from_time, now): general_settings = get_settings_collection().find_one( GENERAL_SETTINGS_LOOKUP) error_recipients = [] if general_settings and general_settings['values'].get( 'system_alerts_recipients'): error_recipients = general_settings['values'][ 'system_alerts_recipients'].split(',') from newsroom.email import send_email for m in monitoring_list: if m.get('users'): internal_req = ParsedRequest() internal_req.args = { 'navigation': str(m['_id']), 'created_from': created_from, 'created_from_time': created_from_time, 'skip_user_validation': True } items = list( get_resource_service('monitoring_search').get( req=internal_req, lookup=None)) template_kwargs = {'profile': m} if items: company = get_entity_or_404(m['company'], 'companies') try: template_kwargs.update({ 'items': items, 'section': 'wire', }) truncate_article_body(items, m) _file = get_monitoring_file(m, items) attachment = base64.b64encode(_file.read()) formatter = app.download_formatters[ m['format_type']]['formatter'] # If there is only one story to send and the headline is to be used as the subject if m.get('headline_subject', False) and len(items) == 1: subject = items[0].get( 'headline', m.get('subject') or m['name']) else: subject = m.get('subject') or m['name'] send_email( [ u['email'] for u in get_items_by_id( [ObjectId(u) for u in m['users']], 'users') ], subject, text_body=render_template('monitoring_email.txt', **template_kwargs), html_body=render_template('monitoring_email.html', **template_kwargs), attachments_info=[{ 'file': attachment, 'file_name': formatter.format_filename(None), 'content_type': 'application/{}'.format( formatter.FILE_EXTENSION), 'file_desc': 'Monitoring Report for Celery monitoring alerts for profile: {}' .format(m['name']) }]) except Exception: logger.exception( '{0} Error processing monitoring profile {1} for company {2}.' .format(self.log_msg, m['name'], company['name'])) if error_recipients: # Send an email to admin template_kwargs = { 'name': m['name'], 'company': company['name'], 'run_time': now, } send_email( error_recipients, gettext( 'Error sending alerts for monitoring: {0}'. format(m['name'])), text_body=render_template( 'monitoring_error.txt', **template_kwargs), html_body=render_template( 'monitoring_error.html', **template_kwargs), ) elif m['schedule'].get('interval') != 'immediate' and m.get( 'always_send'): send_email( [ u['email'] for u in get_items_by_id( [ObjectId(u) for u in m['users']], 'users') ], m.get('subject') or m['name'], text_body=render_template( 'monitoring_email_no_updates.txt', **template_kwargs), html_body=render_template( 'monitoring_email_no_updates.html', **template_kwargs), ) get_resource_service('monitoring').patch(m['_id'], { 'last_run_time': local_to_utc(app.config['DEFAULT_TIMEZONE'], now) })
def _parse_fixture_list(self, fixture, items): xml = fixture.get('fixture_xml') competition_detail = xml.find( './/Fixture_List/Competition/Competition_Details') start_date = competition_detail.attrib.get('Start_Date', '') end_date = competition_detail.attrib.get('End_Date', '') comp_type = competition_detail.attrib.get('Comp_Type', '') # A fixture list that the competition details provide the start and end date if start_date != '' and end_date != '': try: self.season = competition_detail.attrib.get('Season', '') start = datetime.strptime('{} 00:00'.format(start_date), '%Y-%m-%d %H:%M') end = datetime.strptime('{} 23:59'.format(end_date), '%Y-%m-%d %H:%M') event = xml.find( './/Fixture_List/Competition/Comp_Fixtures/Event') match_id = None if event is not None and event.attrib.get( 'Event_ID') is not None: match_id = event.attrib.get('Event_ID') if datetime.now() < end and match_id: item = self._set_default_item(fixture.get('sport_id'), fixture.get('comp_id'), match_id) if self._can_ingest_item(item): item['name'] = '{} - {}'.format( self.sport_map.get(fixture.get('sport_id', {}), {}).get('name', ''), fixture.get('comp_name')) item[ 'definition_short'] = competition_detail.attrib.get( 'Gender', '') item['dates'] = { 'start': local_to_utc(config.DEFAULT_TIMEZONE, start), 'end': local_to_utc(config.DEFAULT_TIMEZONE, end), 'tz': config.DEFAULT_TIMEZONE, } items.append(item) except Exception as ex: logger.exception('Failed to parse event fixtures.') else: # A fixture list that we need to pull out each match to determine the match date etc. for match in xml.find( './/Fixture_List/Competition/Competition_Fixtures'): start_date = match.find('.//Match_Details').attrib.get( 'Match_Date', '') start_time = match.find('.//Match_Details').attrib.get( 'Match_Time', '') self.season = match.find('.//Match_Details').attrib.get( 'Season', '') try: when = datetime.strptime( '{} {}'.format(start_date, start_time), '%Y-%m-%d %H:%M:%S') except Exception as ex: continue if datetime.now() < when: try: match_id = match.find('.//Match_Details').attrib.get( 'Match_ID', '') # Some match id's are not yet available if match_id[-1] == '-': continue match_no = match.find('.//Match_Details').attrib.get( 'Match_No', '') teamA_name = match.findall( './/Teams/Team_Details')[0].attrib.get( 'Team_Name', '') teamB_name = match.findall( './/Teams/Team_Details')[1].attrib.get( 'Team_Name', '') venue_name = match.find('.//Venue').attrib.get( 'Venue_Name', '') venue_location = match.find('.//Venue').attrib.get( 'Venue_Location', '') item = self._set_default_item(fixture.get('sport_id'), fixture.get('comp_id'), match_id) if self._can_ingest_item(item): item['name'] = '{} - {} v {}'.format( self.sport_map.get(fixture.get('sport_id', {}), {}).get('name', ''), teamA_name, teamB_name) item[ 'definition_short'] = '{} match {} {} v {}'.format( fixture.get('comp_name', ''), match_no, teamA_name, teamB_name) # kludge for cricket if fixture.get('sport_id') == '3': if 'test' in comp_type.lower(): delta = timedelta(days=5) elif 'shef' in comp_type.lower(): delta = timedelta(days=4) elif 't20' in comp_type.lower(): delta = timedelta(hours=4) elif 'odi' in comp_type.lower( ) or 'odd' in comp_type.lower(): delta = timedelta(hours=8) else: delta = timedelta(hours=8) else: delta = timedelta(hours=2) item['dates'] = { 'start': local_to_utc(config.DEFAULT_TIMEZONE, when), 'end': local_to_utc(config.DEFAULT_TIMEZONE, when) + delta, 'tz': config.DEFAULT_TIMEZONE, } # add location self._set_location( item, '{}, {}'.format(venue_name, venue_location)) items.append(item) except Exception as ex: logger.exception( 'Failed to parse competition fixtures.')
def parse(self, data, provider=None): index = self.parse_titles(data[0]) items = [] cells_list = [] # use for patch update to reduce write requests usage # skip first two title rows for row in range(3, len(data) + 1): if not row: break item = {} error_message = None values = data[row - 1] is_updated = values[index['_STATUS']].strip().upper( ) if len(values) - 1 > index['_STATUS'] else None try: # only insert item if _STATUS is empty if is_updated in ('UPDATED', 'ERROR'): guid = values[index['_GUID']] # check if it's exists and guid is valid if not superdesk.get_resource_service('events').find_one( guid=guid, req=None): raise KeyError('GUID is not exists') else: guid = generate_guid(type=GUID_NEWSML) # avoid momentsJS throw null timezone value error tzone = values[index['Timezone']] if values[ index['Timezone']] != 'none' else 'UTC' start_datetime = parse(values[index['Start date']] + ' ' + values[index['Start time']]) end_datetime = parse(values[index['End date']] + ' ' + values[index['End time']]) if values[index['All day']] == 'TRUE': start_datetime = parse(values[index['Start date']]) end_datetime = parse( values[index['End date']]) + timedelta(days=1, seconds=-1) if end_datetime < start_datetime: raise ValueError( 'End datetime is smaller than Start datetime') item = { 'type': 'event', 'name': values[index['Event name']], 'slugline': values[index['Slugline']], 'dates': { 'start': local_to_utc(tzone, start_datetime), 'end': local_to_utc(tzone, end_datetime), 'tz': tzone, }, 'definition_short': values[index['Description']], 'definition_long': values[index['Long description']], 'internal_note': values[index['Internal note']], 'ednote': values[index['Ed note']], 'links': [values[index['External links']]], 'guid': guid, 'status': is_updated, } item.setdefault(ITEM_STATE, CONTENT_STATE.DRAFT) occur_status = values[index['Occurence status']] if occur_status and occur_status in self.occur_status_qcode_mapping: item['occur_status'] = { 'qcode': self.occur_status_qcode_mapping.get( values[index['Occurence status']]), 'name': values[index['Occurence status']], 'label': values[index['Occurence status']].lower(), } calendars = values[index['Calendars']] if calendars: item['calendars'] = [{ 'is_active': True, 'name': calendars, 'qcode': calendars.lower(), }] if all(values[index[field]] for field in self.required_location_field): item['location'] = [{ 'name': values[index['Location Name']], 'address': { 'line': [values[index['Location Address']]], 'locality': values[index['Location City/Town']], 'area': values[index['Location State/Province/Region']], 'country': values[index['Location Country']], } }] if all(values[index[field]] for field in self.required_contact_field) \ and (all(values[index[field]] for field in ['Contact First name', 'Contact Last name']) or values[index['Contact Organisation']]): is_public = values[index['Contact Phone Public']] == 'TRUE' if values[index['Contact Phone Usage']] == 'Confidential': is_public = False item['contact'] = { 'honorific': values[index['Contact Honorific']], 'first_name': values[index['Contact First name']], 'last_name': values[index['Contact Last name']], 'organisation': values[index['Contact Organisation']], 'contact_email': [values[index['Contact Email']]], 'contact_address': [values[index['Contact Point of Contact']]], 'contact_phone': [{ 'number': values[index['Contact Phone Number']], 'public': is_public, 'usage': values[index['Contact Phone Usage']], }] } # ignore invalid item missing_fields = [ field for field in self.required_field if not item.get(field) ] if missing_fields: missing_fields = ', '.join(missing_fields) logger.error( 'Provider %s: Event "%s". Missing %s fields', provider.get('name'), item.get('name'), missing_fields, ) error_message = 'Missing ' + missing_fields + ' fields' except UnknownTimeZoneError: error_message = 'Invalid timezone' logger.error('Provider %s: Event "%s": Invalid timezone %s', provider.get('name'), values[index['Event name']], tzone) except (TypeError, ValueError, KeyError) as e: error_message = e.args[0] logger.error('Provider %s: Event "%s": %s', provider.get('name'), item.get('name'), error_message) if error_message: cells_list.extend([ Cell(row, index['_STATUS'] + 1, 'ERROR'), Cell(row, index['_ERR_MESSAGE'] + 1, error_message) ]) elif not is_updated or is_updated == 'UPDATED': cells_list.extend([ Cell(row, index['_STATUS'] + 1, 'DONE'), Cell(row, index['_ERR_MESSAGE'] + 1, ''), ]) if not is_updated: # only update _GUID when status is empty cells_list.append(Cell(row, index['_GUID'] + 1, guid)) items.append(item) return items, cells_list
def test_featured(client, app): app.data.insert('products', [{ '_id': 12, 'name': 'product test', 'query': '_featured', 'companies': ['1'], 'navigations': ['51'], 'is_enabled': True, 'product_type': 'agenda', }, { '_id': 13, 'name': 'all items', 'query': '*:*', 'companies': ['1'], 'navigations': ['51'], 'is_enabled': True, 'product_type': 'agenda', }]) _items = [] for i in range(5): item = agenda_items[0].copy() item['_id'] = 'urn:item:%d' % i item['dates'] = item['dates'].copy() item['dates']['start'] += timedelta(hours=1) _items.append(item) app.data.insert('agenda', _items) # post first 2 items date = utc_to_local('Australia/Sydney', datetime.utcnow().replace(microsecond=0)) _id = date.strftime('%Y%m%d') featured = { '_id': _id, 'type': 'planning_featured', 'item_id': _id, 'items': [item['_id'] for item in _items[:2]], 'tz': 'Australia/Sydney', } resp = post_json(client, 'push', featured) assert 200 == resp.status_code # public user with client.session_transaction() as session: session['user'] = PUBLIC_USER_ID session['user_type'] = 'public' data = get_json(client, '/agenda/search?navigation=51') assert 2 == data['_meta']['total'] assert _items[0]['_id'] == data['_items'][0]['_id'] assert _items[1]['_id'] == data['_items'][1]['_id'] assert '_aggregations' in data assert data['_items'][0]['_display_from'].replace('+0000', '+00:00') == \ local_to_utc('Australia/Sydney', date.replace(hour=0, minute=0, second=0)).isoformat() assert data['_items'][0]['_display_to'].replace('+0000', '+00:00') == \ local_to_utc('Australia/Sydney', date.replace(hour=23, minute=59, second=59)).isoformat() # post first 3 items in reverse order featured['items'] = [item['_id'] for item in _items[:3]] featured['items'].reverse() resp = post_json(client, 'push', featured) assert 200 == resp.status_code data = get_json(client, '/agenda/search?navigation=51') assert 3 == data['_meta']['total'] assert _items[2]['_id'] == data['_items'][0]['_id'] assert _items[1]['_id'] == data['_items'][1]['_id'] assert _items[0]['_id'] == data['_items'][2]['_id'] data = get_json(client, '/agenda/search?navigation=51&q=slugline:nonsense') assert 0 == data['_meta']['total'] # search with no nav - featured disabled data = get_json(client, '/agenda/search') assert len(_items) <= data['_meta']['total']
def _publish_date_filter(self, date_string): local = dateutil.parser.parse(date_string) return local_to_utc(self.TIMEZONE, local)
def _parse_doc(self, doc): new_doc = {} new_doc['_id'] = doc['refPtr'] new_doc['guid'] = doc['refPtr'] try: new_doc['description_text'] = doc['caption'] except KeyError: pass try: new_doc['headline'] = doc['headline'] except KeyError: pass try: new_doc['original_source'] = new_doc['source'] = doc['credit'] except KeyError: pass new_doc['versioncreated'] = new_doc['firstcreated'] = self._datetime( local_to_utc(SCANPIX_TZ, get_date(doc['archivedTime']))) new_doc['pubstatus'] = 'usable' # This must match the action new_doc['_type'] = 'externalsource' # entry that the client can use to identify the fetch endpoint new_doc['fetch_endpoint'] = 'scanpix' # mimetype is not directly found in Scanpix API # so we use original filename to guess it mimetype = mimetypes.guess_type("_{}".format( splitext(doc.get('originalFileName', ''))[1]))[0] if mimetype is None: # nothing found with filename, we try out luck with fileFormat try: format_ = doc['fileFormat'].split()[0] except (KeyError, IndexError): mimetype = None else: mimetype = mimetypes.guess_type('_.{}'.format(format_))[0] if mimetype is not None: new_doc['mimetype'] = mimetype main_group = doc['mainGroup'] if main_group == 'video': new_doc[ITEM_TYPE] = CONTENT_TYPE.VIDEO elif main_group == 'graphic': new_doc[ITEM_TYPE] = CONTENT_TYPE.GRAPHIC new_doc['mimetype'] = 'image/jpeg' else: new_doc[ITEM_TYPE] = CONTENT_TYPE.PICTURE try: doc_previews = doc['previews'] except KeyError: logger.warning('no preview found for item {}'.format( new_doc['_id'])) else: # we look for best available scanpix preview available_previews = [p['type'] for p in doc_previews] renditions = new_doc['renditions'] = {} for rend, previews in REND2PREV.items(): for prev in previews: if prev in available_previews: idx = available_previews.index(prev) renditions[rend] = {"href": doc_previews[idx]['url']} break new_doc['byline'] = doc['byline'] doc.clear() doc.update(new_doc)
def test_run_daily_jpeg(self, mocked): with self.app.app_context(): self.app.data.insert('users', mock_users) self.app.data.insert('vocabularies', mock_vocabs) self.app.data.insert('saved_reports', mock_saved_reports) self.app.data.insert('scheduled_reports', [{ '_id': 'sched1', 'name': 'Scheduled Report', 'saved_report': 'srep1', 'schedule': { 'frequency': 'daily', 'hour': 1 }, 'transmitter': 'email', 'mimetype': MIME_TYPES.JPEG, 'extra': { 'body': 'This is a test email' }, 'recipients': ['*****@*****.**'], 'report_width': 1200, 'active': True }]) scheduled_service = get_resource_service('scheduled_reports') report = scheduled_service.find_one(req=None, _id='sched1') self.assertNotIn('_last_sent', report) # Simulate running every hour for a few hours start_date = to_naive('2018-06-30T00') end_date = to_naive('2018-06-30T03') should_have_updated = False with self.app.mail.record_messages() as outbox: for now in rrule(HOURLY, dtstart=start_date, until=end_date): local_tz = pytz.timezone(app.config['DEFAULT_TIMEZONE']) now_local = local_tz.localize(now) now_utc = local_to_utc(app.config['DEFAULT_TIMEZONE'], now_local) SendScheduledReports().run(now_utc) # _last sent is updated report = scheduled_service.find_one(req=None, _id='sched1') if not should_have_updated: should_have_updated = True self.assertNotIn('_last_sent', report) else: self.assertEqual(report.get('_last_sent'), to_utc('2018-06-30T01')) # Test that the command sent only 1 email self.assertEqual(len(outbox), 1) # Test attachment self.assertEqual(len(outbox[0].attachments), 1) self.assertEqual( outbox[0].attachments[0].content_type, '{}; name="chart_1.jpeg"'.format(MIME_TYPES.JPEG)) self.assertEqual(outbox[0].attachments[0].filename, 'chart_1.jpeg') report = scheduled_service.find_one(req=None, _id='sched1') self.assertEqual(report.get('_last_sent'), to_utc('2018-06-30T01'))
def parse_datetime(self, value): if not value: return None local = datetime.strptime(value, '%m/%d/%Y %H:%M:%S %p') return local_to_utc(self.TZ, local)
def test_local_to_utc_europe(self): utc_dt = local_to_utc('Europe/Prague', datetime(2016, 4, 19, 15, 8, 0)) self.assertEqual('2016-04-19T13:08:00+00:00', utc_dt.isoformat())
def update_recurring_events(self, updates, original, update_method): historic, past, future = self.get_recurring_timeline(original) # Determine if the selected event is the first one, if so then # act as if we're changing future events if len(historic) == 0 and len(past) == 0: update_method = UPDATE_FUTURE if update_method == UPDATE_FUTURE: new_series = [original] + future else: new_series = past + [original] + future # Release the Lock on the selected Event remove_lock_information(updates) # Get the timezone from the original Event (as the series was created with that timezone in mind) timezone = original['dates']['tz'] # First find the hour and minute of the start date in local time start_time = utc_to_local(timezone, updates['dates']['start']).time() # Next convert that to seconds since midnight (which gives us a timedelta instance) delta_since_midnight = datetime.combine(date.min, start_time) - datetime.min # And calculate the new duration of the events duration = updates['dates']['end'] - updates['dates']['start'] for event in new_series: if not event.get(config.ID_FIELD): continue new_updates = {'dates': deepcopy(event['dates'])} \ if event.get(config.ID_FIELD) != original.get(config.ID_FIELD) else updates # Calculate midnight in local time for this occurrence start_of_day_local = utc_to_local(timezone, event['dates']['start'])\ .replace(hour=0, minute=0, second=0) # Then convert midnight in local time to UTC start_date_time = local_to_utc(timezone, start_of_day_local) # Finally add the delta since midnight start_date_time += delta_since_midnight # Set the new start and end times new_updates['dates']['start'] = start_date_time new_updates['dates']['end'] = start_date_time + duration if event.get(TO_BE_CONFIRMED_FIELD): new_updates[TO_BE_CONFIRMED_FIELD] = False # Set '_planning_schedule' on the Event item self.set_planning_schedule(new_updates) if event.get(config.ID_FIELD) != original.get(config.ID_FIELD): new_updates['skip_on_update'] = True self.patch(event[config.ID_FIELD], new_updates) app.on_updated_events_update_time( new_updates, {'_id': event[config.ID_FIELD]})
def to_utc(date_str): return local_to_utc( app.config['DEFAULT_TIMEZONE'], datetime.strptime(date_str, '%Y-%m-%dT%H') )
def _process_sheet(self, title, dates, values, _id): i = 0 for date in dates: start_date = self.epoch + timedelta(days=date) dt = start_date end_dt = None if values[i][1] and values[i][1].lower() != 'tbc': try: hours = values[i][1].split(':')[0] minutes = values[i][1].split(':')[1] dt = dt + timedelta(hours=int(hours), minutes=int(minutes)) except: pass if values[i][2] and values[i][2].lower() != 'tbc': try: hours = values[i][2].split(':')[0] minutes = values[i][2].split(':')[1] end_dt = start_date + timedelta(hours=int(hours), minutes=int(minutes)) except: pass v = values[i] item = self._set_default_item(title, _id, hashlib.sha1('-'.join(v).encode('utf8')).hexdigest(), v[7] if v[7] else 'australia') self._set_location(item, '{} {} {} {}'.format(v[4], v[5], v[6], v[7])) sheet_country = self._set_country(v[7]) sheet_city = v[5].lower() loc = sheet_city + '/' + sheet_country tz = self.tz_map.get(loc, config.DEFAULT_TIMEZONE) item['name'] = v[3] item['definition_short'] = v[3] item['source'] = 'AAP Sports Sheet' if end_dt: item['dates'] = { 'start': local_to_utc(tz, dt), 'end': local_to_utc(tz, end_dt), 'tz': tz, } else: item['dates'] = { 'start': local_to_utc(tz, dt), 'end': local_to_utc(tz, dt) + timedelta(hours=23, minutes=59, seconds=59), 'tz': tz, } # print('{} {} {} {} {}'.format(title, item['name'], item.get('dates').get('start'), # item.get('dates').get('end'), tz)) # print('{}-{} city:[{}] state:[{}] county[{}] ---> {}'.format(title, item['name'], v[5], v[6], v[7], tz)) print('{},{},{},{},{},{}'.format(title.replace(',', ' '), item['name'].replace(',', ' '), v[5], v[6], v[7], tz)) old = superdesk.get_resource_service('events').find_one(req=None, guid=item['guid']) if old: superdesk.get_resource_service('events').patch(old.get('_id'), item) else: superdesk.get_resource_service('events').post([item]) i += 1
def _parse_fixture_list(self, fixture, items): xml = fixture.get('fixture_xml') competition_detail = xml.find('.//Fixture_List/Competition/Competition_Details') start_date = competition_detail.attrib.get('Start_Date', '') end_date = competition_detail.attrib.get('End_Date', '') comp_type = competition_detail.attrib.get('Comp_Type', '') if comp_type == '': if 'Test' in fixture.get('comp_name'): comp_type = 'test' elif 'One Day' in fixture.get('comp_name'): comp_type = 'odi' # A fixture list that the competition details provide the start and end date if start_date != '' and end_date != '': try: self.season = competition_detail.attrib.get('Season', '') start = datetime.strptime('{} 00:00'.format(start_date), '%Y-%m-%d %H:%M') end = datetime.strptime('{} 23:59'.format(end_date), '%Y-%m-%d %H:%M') event = xml.find('.//Fixture_List/Competition/Comp_Fixtures/Event') match_id = None if event is not None and event.attrib.get('Event_ID') is not None: match_id = event.attrib.get('Event_ID') if datetime.now() < end and match_id: item = self._set_default_item(fixture.get('sport_id'), fixture.get('comp_id'), match_id) if self._can_ingest_item(item): item['name'] = '{} - {}'.format( self.sport_map.get(fixture.get('sport_id', {}), {}).get('name', ''), fixture.get('comp_name') ) item['definition_short'] = competition_detail.attrib.get('Gender', '') item['dates'] = { 'start': local_to_utc(config.DEFAULT_TIMEZONE, start), 'end': local_to_utc(config.DEFAULT_TIMEZONE, end), 'tz': config.DEFAULT_TIMEZONE, } items.append(item) except Exception: logger.exception('Failed to parse event fixtures.') else: # A fixture list that we need to pull out each match to determine the match date etc. for match in xml.find('.//Fixture_List/Competition/Competition_Fixtures'): start_date = match.find('.//Match_Details').attrib.get('Match_Date', '') start_time = match.find('.//Match_Details').attrib.get('Match_Time', '') self.season = match.find('.//Match_Details').attrib.get('Season', '') try: when = datetime.strptime('{} {}'.format(start_date, start_time), '%Y-%m-%d %H:%M:%S') except Exception: continue if datetime.now() < when: try: match_id = match.find('.//Match_Details').attrib.get('Match_ID', '') # Some match id's are not yet available if match_id[-1] == '-': continue match_no = match.find('.//Match_Details').attrib.get('Match_No', '') teamA_name = match.findall('.//Teams/Team_Details')[0].attrib.get('Team_Name', '') teamB_name = match.findall('.//Teams/Team_Details')[1].attrib.get('Team_Name', '') venue_name = match.find('.//Venue').attrib.get('Venue_Name', '') venue_location = match.find('.//Venue').attrib.get('Venue_Location', '') item = self._set_default_item(fixture.get('sport_id'), fixture.get('comp_id'), match_id) if self._can_ingest_item(item): item['name'] = '{} - {} v {}'.format( self.sport_map.get(fixture.get('sport_id', {}), {}).get('name', ''), teamA_name, teamB_name) item['definition_short'] = '{} match {} {} v {}'.format(fixture.get('comp_name', ''), match_no, teamA_name, teamB_name) # kludge for cricket if fixture.get('sport_id') == '3': if 'test' in comp_type.lower(): delta = timedelta(days=5) elif 'shef' in comp_type.lower(): delta = timedelta(days=4) elif 't20' in comp_type.lower(): delta = timedelta(hours=4) elif 'odi' in comp_type.lower() or 'odd' in comp_type.lower(): delta = timedelta(hours=8) else: delta = timedelta(hours=8) else: delta = timedelta(hours=2) item['dates'] = { 'start': local_to_utc(config.DEFAULT_TIMEZONE, when), 'end': local_to_utc(config.DEFAULT_TIMEZONE, when) + delta, 'tz': config.DEFAULT_TIMEZONE, } # add location self._set_location(item, '{}, {}'.format(venue_name, venue_location)) items.append(item) except Exception: logger.exception('Failed to parse competition fixtures.')
def _parse_date(self, value): dt = datetime.strptime(value, DATETIME_FORMAT) return local_to_utc(TIMEZONE, dt)
def datetime(self, value): """When there is no timezone info, assume it's Helsinki timezone.""" parsed = super().datetime(value) if '+' not in value: return local_to_utc(TIMEZONE, parsed) return parsed
def test_run_hourly_png(self, mocked): with self.app.app_context(): self.app.data.insert('users', mock_users) self.app.data.insert('vocabularies', mock_vocabs) self.app.data.insert('saved_reports', mock_saved_reports) self.app.data.insert('scheduled_reports', [{ '_id': 'sched1', 'name': 'Scheduled Report', 'saved_report': 'srep1', 'schedule': { 'frequency': 'hourly' }, 'transmitter': 'email', 'mimetype': MIME_TYPES.PNG, 'extra': { 'body': 'This is a test email' }, 'recipients': ['*****@*****.**'], 'report_width': 1200, 'active': True }]) scheduled_service = get_resource_service('scheduled_reports') report = scheduled_service.find_one(req=None, _id='sched1') self.assertNotIn('_last_sent', report) # Simulate running every hour for a few hours start_date = to_naive('2018-06-30T00') end_date = to_naive('2018-06-30T03') local_tz = pytz.timezone(app.config['DEFAULT_TIMEZONE']) with self.app.mail.record_messages() as outbox: for now in rrule(HOURLY, dtstart=start_date, until=end_date): now_local = local_tz.localize(now) now_utc = local_to_utc(app.config['DEFAULT_TIMEZONE'], now_local) SendScheduledReports().run(now_utc) # _last sent is updated report = scheduled_service.find_one(req=None, _id='sched1') self.assertEqual(report.get('_last_sent'), now_utc) # Test that the command sent emails across the 4 iterations self.assertEqual(len(outbox), 4) # Test the first email self.assertEqual(outbox[0].sender, '*****@*****.**') self.assertEqual(outbox[0].subject, 'Superdesk Analytics - Scheduled Report') self.assertTrue( outbox[0].body.startswith('\nThis is a test email')) self.assertTrue('This is a test email' in outbox[0].html) self.assertTrue('<img src="cid:' in outbox[0].html) self.assertEqual(outbox[0].recipients, ['*****@*****.**']) # Test attachment self.assertEqual(len(outbox[0].attachments), 1) self.assertEqual(outbox[0].attachments[0].data, b64decode(mock_file)) self.assertEqual( outbox[0].attachments[0].content_type, '{}; name="chart_1.png"'.format(MIME_TYPES.PNG)) self.assertEqual(outbox[0].attachments[0].filename, 'chart_1.png') report = scheduled_service.find_one(req=None, _id='sched1') self.assertEqual(report.get('_last_sent'), to_utc('2018-06-30T03'))
def test_local_to_utc_europe(self): utc_dt = local_to_utc('Europe/Prague', datetime(2016, 4, 19, 15, 8, 0)) self.assertEqual('2016-04-19T13:08:00+00:00', utc_dt.isoformat())
def _parse_date(self, string): local = datetime.strptime(string, '%Y%m%d %H:%M:%S') return local_to_utc(TZ, local)
def parse_newsmanagement(self, item, manage_el): super().parse_newsmanagement(item, manage_el) tz = 'Europe/Moscow' item['firstcreated'] = local_to_utc(tz, item['firstcreated']) item['versioncreated'] = local_to_utc(tz, item['firstcreated'])
def send_alerts(self, monitoring_list, created_from, created_from_time, now): general_settings = get_settings_collection().find_one( GENERAL_SETTINGS_LOOKUP) error_recipients = [] if general_settings and general_settings['values'].get( 'system_alerts_recipients'): error_recipients = general_settings['values'][ 'system_alerts_recipients'].split(',') from newsroom.email import send_email for m in monitoring_list: # if immediate set the created from to the time of the last item sent, if available if m.get('schedule', {}).get('interval') == 'immediate' and m.get( 'last_run_time', False): created_from = m.get('last_run_time').strftime('%Y-%m-%d') created_from_time = m.get('last_run_time').strftime('%H:%M:%S') last_run_time = local_to_utc(app.config['DEFAULT_TIMEZONE'], now) company = get_entity_or_404(m['company'], 'companies') user_list = self.filter_users(m, company) if len(user_list) and company.get('is_enabled'): internal_req = ParsedRequest() internal_req.args = { 'navigation': str(m['_id']), 'created_from': created_from, 'created_from_time': created_from_time, 'skip_user_validation': True } items = list( get_resource_service('monitoring_search').get( req=internal_req, lookup=None)) items[:] = [ item for item in items if not self.already_sent(item, m) ] template_kwargs = {'profile': m} if items: try: template_kwargs.update({ 'items': items, 'section': 'wire', }) truncate_article_body(items, m) # If there is only one story to send and the headline is to be used as the subject if m.get('headline_subject', False) and len(items) == 1: subject = items[0].get( 'headline', m.get('subject') or m['name']) else: subject = m.get('subject') or m['name'] if m.get('format_type') == 'monitoring_email': self.send_email_alert(items, subject, m, user_list) else: _file = get_monitoring_file(m, items) attachment = base64.b64encode(_file.read()) formatter = app.download_formatters[ m['format_type']]['formatter'] send_email( user_list, subject, text_body=render_template( 'monitoring_email.txt', **template_kwargs), html_body=render_template( 'monitoring_email.html', **template_kwargs), attachments_info=[{ 'file': attachment, 'file_name': formatter.format_filename(None), 'content_type': 'application/{}'.format( formatter.FILE_EXTENSION), 'file_desc': 'Monitoring Report for Celery monitoring alerts for profile: {}' .format(m['name']) }]) get_resource_service('history').create_history_record( items, action='email', user={ '_id': None, 'company': m['company'] }, section='monitoring', monitoring=m['_id']) except Exception: logger.exception( '{0} Error processing monitoring profile {1} for company {2}.' .format(self.log_msg, m['name'], company['name'])) if error_recipients: # Send an email to admin template_kwargs = { 'name': m['name'], 'company': company['name'], 'run_time': now, } send_email( error_recipients, gettext( 'Error sending alerts for monitoring: {0}'. format(m['name'])), text_body=render_template( 'monitoring_error.txt', **template_kwargs), html_body=render_template( 'monitoring_error.html', **template_kwargs), ) elif m['schedule'].get('interval') != 'immediate' and m.get( 'always_send'): send_email( user_list, m.get('subject') or m['name'], text_body=render_template( 'monitoring_email_no_updates.txt', **template_kwargs), html_body=render_template( 'monitoring_email_no_updates.html', **template_kwargs), ) if m.get('schedule', {}).get('interval') == 'immediate' and len(items): # determine the version created time for the most recent item last_article_created = max( item.get('versioncreated') for item in items) # last run time is either the time of the last sent item or the last run time. last_run_time = last_article_created if last_article_created else last_run_time get_resource_service('monitoring').patch( m['_id'], {'last_run_time': last_run_time})
def convert_date(epoch): dt = local_to_utc(config.DEFAULT_TIMEZONE, datetime.fromtimestamp(int(str(epoch)[:10]))) return dt
def _publish_date_filter(self, date_string): local = dateutil.parser.parse(date_string) return local_to_utc(self.TIMEZONE, local)
def get_datetime(value): dt = arrow.get(value).datetime return local_to_utc(BELGA_TZ, dt)
def datetime(self, string): dt = parse(string).replace(tzinfo=utc) return local_to_utc(TIMEZONE, dt)