def fetch_records(self, url, end_day):
page, detail = 0, None
while True:
page += 1
resp = self.requests.get(furl(url).add(query_params={
'page': page,
}).url)
if resp.status_code == 422:
# We've asked for too much. Time to readjust date range
# Thanks for leaking variables python
page, url = 0, furl(url).add(query_params={
'modified_date': pendulum.parse(detail['modified_date']).date().isoformat()
})
continue
for item in resp.json():
resp = self.requests.get(item['url'])
detail = resp.json()
if pendulum.parse(detail['modified_date']).date() > end_day:
return
yield item['url'], detail
if len(resp.json()) < self.page_size:
return # We've hit the end of our results
def do_run(self, start: [str, datetime.datetime]=None, end: [str, datetime.datetime]=None, limit: int=None, force=False, **kwargs):
if self.config.disabled and not force:
raise Exception('Harvester {} is disabled. Either enable it or disable its celery beat entry'.format(self.config))
if not start and not end:
start, end = datetime.timedelta(days=-1), datetime.datetime.utcnow()
if type(end) is str:
end = pendulum.parse(end.rstrip('Z')) # TODO Fix me
if type(start) is str:
start = pendulum.parse(start.rstrip('Z')) # TODO Fix Me
harvester = self.config.harvester(self.config)
try:
logger.info('Starting harvester run for %s %s - %s', self.config.label, start, end)
raw_ids = harvester.harvest(start, end, limit=limit, **kwargs)
logger.info('Collected %d data blobs from %s', len(raw_ids), self.config.label)
except Exception as e:
logger.exception('Failed harvester task (%s, %s, %s)', self.config.label, start, end)
raise self.retry(countdown=10, exc=e)
# attach task to each RawData
RawData.tasks.through.objects.bulk_create([
RawData.tasks.through(rawdata_id=raw_id, celeryprovidertask_id=self.task.id)
for raw_id in raw_ids
])
for raw_id in raw_ids:
task = NormalizerTask().apply_async((self.started_by.id, self.config.label, raw_id,))
logger.debug('Started normalizer task %s for %s', task, raw_id)
def widget_search_button_handler(change):
global features, datasets, standard_name, dataset_id, constraints
search_min_time = pendulum.parse(widget_search_min_time.value)
search_max_time = pendulum.parse(widget_search_max_time.value)
# get standard_name from dropdown widget
standard_name = widget_std_names.value
# get updated datsets and map features
features, datasets = stdname2geojson(e, standard_name, cdm_data_type, search_min_time, search_max_time)
# update map
feature_layer = ipyl.GeoJSON(data=features)
feature_layer.on_click(map_click_handler)
map.layers = [map.layers[0], feature_layer]
# widget_plot_start_time.value = widget_search_min_time.value
# widget_plot_stop_time.value = widget_search_max_time.value
# populate datasets widget with new info
dataset_id = datasets[0]
widget_dsnames.options = datasets
widget_dsnames.value = dataset_id
constraints = {
'time>=': search_min_time,
'time<=': search_max_time
}
print(dataset_id, standard_name, constraints)
update_timeseries_plot(dataset=dataset_id, standard_name=standard_name, constraints=constraints)
def test_obsolete(self):
source_config = factories.SourceConfigFactory()
hlv1 = factories.HarvestJobFactory(
harvester_version=source_config.harvester.version,
source_config=source_config,
start_date=pendulum.parse('2017-01-01').date(),
)
old_version = source_config.harvester.get_class().VERSION
source_config.harvester.get_class().VERSION += 1
new_version = source_config.harvester.get_class().VERSION
hlv2 = factories.HarvestJobFactory(
harvester_version=source_config.harvester.version,
source_config=source_config,
start_date=pendulum.parse('2017-01-01').date(),
)
tasks.harvest(job_id=hlv2.id)
tasks.harvest(job_id=hlv1.id)
hlv1.refresh_from_db()
hlv2.refresh_from_db()
assert hlv2.status == HarvestJob.STATUS.succeeded
assert hlv2.harvester_version == new_version
assert hlv1.status == HarvestJob.STATUS.skipped
assert hlv1.harvester_version == old_version
assert hlv1.error_context == HarvestJob.SkipReasons.obsolete.value
def start_harvest(context, label, start=None, end=None):
job = HarvestScheduler(models.SourceConfig.objects.get(label=label), claim_jobs=True).range(
pendulum.parse(start),
pendulum.parse(end),
)[0]
tasks.harvest(job_id=job.id)
def change(self):
changes, relations = {}, {}
extra = copy.deepcopy(self.extra)
if self.namespace:
if self.namespace and getattr(self.instance, 'extra', None):
# NOTE extra changes are only diffed at the top level
self.instance.extra.data.setdefault(self.namespace, {})
changes['extra'] = {
k: v
for k, v in extra.items()
if k not in self.instance.extra.data[self.namespace]
or self.instance.extra.data[self.namespace][k] != v
}
else:
changes['extra'] = extra
if not changes['extra']:
del changes['extra']
for edge in self.related(backward=False):
if edge.field.one_to_many:
relations.setdefault(edge.name, []).append(edge.related.ref)
else:
relations[edge.name] = edge.related.ref
if self.is_blank:
return {**changes, **self.attrs, **relations}
if self.instance and type(self.instance) is not self.model:
changes['type'] = self.model._meta.label_lower
# Hacky fix for SHARE-604
# If the given date_updated is older than the current one, don't accept any changes that would overwrite newer changes
ignore_attrs = set()
if issubclass(self.model, apps.get_model('share', 'creativework')) and 'date_updated' in self.attrs and self.instance.date_updated:
date_updated = pendulum.parse(self.attrs['date_updated'])
if date_updated < self.instance.date_updated:
logger.warning('%s appears to be from the past, change date_updated (%s) is older than the current (%s). Ignoring conflicting changes.', self, self.attrs['date_updated'], self.instance.date_updated)
# Just in case
ignore_attrs.update(self.instance.change.change.keys())
# Go back until we find a change that is older than us
for version in self.instance.versions.select_related('change').all():
if not version.date_updated or date_updated > version.date_updated:
break
ignore_attrs.update(version.change.change.keys())
attrs = {}
for k, v in self.attrs.items():
if k in ignore_attrs:
logger.debug('Ignoring potentially conflicting change to "%s"', k)
continue
old_value = getattr(self.instance, k)
if isinstance(old_value, datetime.datetime):
v = pendulum.parse(v)
if v != old_value:
attrs[k] = v.isoformat() if isinstance(v, datetime.datetime) else v
# TODO Add relationships in. Somehow got ommitted first time around
return {**changes, **attrs}
def do_harvest(self, start_date: pendulum.Pendulum, end_date: pendulum.Pendulum):
url = self.config.base_url
while True:
logger.debug('Fetching page %s', url)
resp = self.requests.get(url)
resp.raise_for_status()
resp_data = resp.json()
for record in resp_data['_items']:
if pendulum.parse(record['date']) < start_date:
logger.info('%s is before %s, ending harvest', record['date'], start_date)
return
if pendulum.parse(record['date']) > end_date:
logger.debug('%s is after %s, skipping', record['date'], end_date)
continue
doc_id = self.kwargs.get('identifier_prefix', '') + record['identifiers']['peerj']
if self.kwargs.get('fetch_xml', False):
logger.debug('Fetching article %s', record['_links']['alternate']['xml']['href'])
details = self.requests.get(record['_links']['alternate']['xml']['href'])
details.raise_for_status()
yield doc_id, details.content
else:
yield doc_id, record
if 'next' not in resp_data['_links']:
logger.info('No "next" key found, ending harvest')
return
url = resp_data['_links']['next']['href']
def test_start_must_be_before_end(self, harvester):
with pytest.raises(ValueError) as e:
list(harvester.fetch_date_range(
pendulum.parse('2016-01-05'),
pendulum.parse('2016-01-04'),
))
assert e.value.args == ('start must be before end. <Pendulum [2016-01-05T00:00:00+00:00]> > <Pendulum [2016-01-04T00:00:00+00:00]>', )
def test_datestamps_within_24_hours(self, source_config):
source_config.harvester.get_class()._do_fetch.clear()
source_config.harvester.get_class()._do_fetch.extend([
('identifier{}'.format(timestamp), 'data{}'.format(timestamp), timestamp)
for timestamp in (pendulum.parse('2016-01-01') - pendulum.parse('2016-01-03')).range('hours')
])
list(source_config.get_harvester().harvest_date(pendulum.parse('2016-01-02')))
def test_fetch_date(self, harvester):
harvester.fetch_date_range = mock.Mock()
harvester.fetch_date(pendulum.parse('2016-01-05'), custom='kwarg')
assert harvester.fetch_date_range.assert_called_once_with(
pendulum.parse('2016-01-04'),
pendulum.parse('2016-01-05'),
custom='kwarg'
) is None
def widget_replot_button_handler(change):
global dataset_id, constraints
plot_start_time = pendulum.parse(widget_plot_start_time.value)
plot_stop_time = pendulum.parse(widget_plot_stop_time.value)
constraints = {
'time>=': plot_start_time,
'time<=': plot_stop_time
}
dataset_id = widget_dsnames.value
update_timeseries_plot(dataset=dataset_id, standard_name=standard_name, constraints=constraints)
def test_datestamps_out_of_range(self, source_config):
source_config.harvester.get_class()._do_fetch.clear()
source_config.harvester.get_class()._do_fetch.extend([
('identifier{}'.format(i), 'data{}'.format(i), pendulum.parse('2016-01-{}'.format(i)))
for i in range(1, 10)
])
with pytest.raises(ValueError) as e:
list(source_config.get_harvester().harvest_date(pendulum.parse('2016-01-01')))
assert e.value.args == ('result.datestamp is outside of the requested date range. 2016-01-03T00:00:00+00:00 from identifier3 is not within [2015-12-31T00:00:00+00:00 - 2016-01-01T00:00:00+00:00]', )
def test_list_services_returns_updated_date(self):
self.setup_dummy_services_including_unpublished(1)
response = self.client.get('/services')
data = json.loads(response.get_data())
assert_equal(response.status_code, 200)
try:
pendulum.parse(
data['services'][0]['updatedAt'])
assert True, "Parsed date"
except ValueError:
assert False, "Should be able to parse date"
def test_fetch_date_range_calls_do_harvest(self, harvester):
harvester.do_harvest = mock.Mock()
BaseHarvester._do_fetch(
harvester,
pendulum.parse('2017-01-01').date(),
pendulum.parse('2017-01-02').date(),
)
assert harvester.do_harvest.assert_called_once_with(
pendulum.parse('2017-01-01').date(),
pendulum.parse('2017-01-02').date(),
) is None
def _merge_node_attrs(self, from_node, into_node):
into_attrs = into_node.attrs()
for k, new_val in from_node.attrs().items():
if k in into_attrs:
old_val = into_attrs[k]
if new_val == old_val:
continue
field = resolve_field(into_node.model, k)
if isinstance(field, DateTimeField):
new_val = max(pendulum.parse(new_val), pendulum.parse(old_val)).isoformat()
else:
new_val = self._merge_value(new_val, old_val)
into_node[k] = new_val
def test_datestamps(self, source_config):
source_config.harvester.get_class()._do_fetch.clear()
source_config.harvester.get_class()._do_fetch.extend([
('identifier{}'.format(i), 'data{}'.format(i), pendulum.parse('2017-01-{}'.format(i)))
for i in range(1, 10)
])
for i, raw in enumerate(source_config.get_harvester().harvest_date_range(
pendulum.parse('2017-01-01'),
pendulum.parse('2017-02-01'),
)):
assert raw.datestamp is not None
assert raw.datestamp.day == (i + 1)
assert raw.datestamp.year == 2017
def test_latest_date(self):
source_config = factories.SourceConfigFactory(
full_harvest=True,
earliest_date=pendulum.parse('2017-01-01').date()
)
# We have a harvest job with start_date equal to earliest_date
# but a different source_config
factories.HarvestJobFactory(
start_date=pendulum.parse('2017-01-01').date(),
end_date=pendulum.parse('2017-01-02').date(),
)
assert len(HarvestScheduler(source_config).all(cutoff=pendulum.parse('2018-01-01').date())) == 365
def resolve_date_range(cls, start, end):
logger.debug('Coercing start and end (%r, %r) into UTC dates', start, end)
if bool(start) ^ bool(end):
raise ValueError('"start" and "end" must either both be supplied or omitted')
if not start and not end:
start, end = datetime.date.today() - datetime.timedelta(days=1), datetime.date.today()
if type(end) is str:
end = pendulum.timezone('utc').convert(pendulum.parse(end.rstrip('Z'))).date()
if type(start) is str:
start = pendulum.timezone('utc').convert(pendulum.parse(start.rstrip('Z'))).date()
logger.debug('Interpretting start and end as %r and %r', start, end)
return start, end
def elig(bot, trigger):
"""Cloak eligibility tool"""
need_verified_email = True
need_edit_count = 250
need_months_registered = 3
if not trigger.group(2):
return bot.say('To be eligible for a cloak you need: ' + ['', 'a verified email, '][need_verified_email] + 'to be registered for ' + str(need_months_registered) + ' months,' + ' and have ' + str(need_edit_count) + ' edits.')
query = trigger.group(3)
r = requests.get('https://en.wikipedia.org/w/api.php?action=query&meta=globaluserinfo&guiuser='******'&guiprop=editcount&format=json')
try:
required_registration_time = pendulum.now().subtract(months=need_months_registered)
actual_registration_time = pendulum.parse(str(r.json()['query']['globaluserinfo']['registration']))
except KeyError:
return bot.say('No such user (please note that usernames are case-sensitive).')
if actual_registration_time > required_registration_time:
return bot.say('User: "******" is ineligible for a cloak because they have not been registered onwiki for long enough (there may be other reasons why this user is ineligible).')
actual_edit_count = r.json()['query']['globaluserinfo']['editcount']
if actual_edit_count < need_edit_count:
return bot.say('User: "******" is ineligible for a cloak because their edit count is not high enough. They need ' + str(need_edit_count - actual_edit_count) + ' more edits to be eligible under this criterion (there may be other reasons why this user is ineligible).')
r2 = requests.get('https://en.wikipedia.org/w/api.php?action=query&list=users&ususers=' + query + '&usprop=emailable&format=json')
try:
r2.json()['query']['users'][0]['emailable']
except KeyError:
return bot.say('User: "******" is ineligible for a cloak because they do not have a verified email address (this problem may be caused be the user disabling Special:EmailUser in their English Wikipedia preferences).')
bot.say('User: "******" is eligible for a cloak. They registered onwiki about ' + actual_registration_time.diff_for_humans() + ', have ' + str(actual_edit_count) + ' edits and have a verified email address.')
async def strawpolls(self, ctx, poll_id: str = None):
"""This command can be used to show a strawpoll setup on this server"""
# Strawpolls cannot be 'deleted' so to handle whether a poll is running or not on a server
# Just save the poll in the config file, which can then be removed when it should not be "running" anymore
all_polls = config.get_content('strawpolls') or {}
server_polls = all_polls.get(ctx.message.server.id) or {}
if not server_polls:
await self.bot.say("There are currently no strawpolls running on this server!")
return
# If no poll_id was provided, print a list of all current running poll's on this server
if not poll_id:
fmt = "\n".join(
"{}: https://strawpoll.me/{}".format(data['title'], _id) for _id, data in server_polls.items())
await self.bot.say("```\n{}```".format(fmt))
# Else if a valid poll_id was provided, print info about that poll
elif poll_id in server_polls.keys():
poll = server_polls[poll_id]
async with self.session.get("{}/{}".format(self.url, poll_id),
headers={'User-Agent': 'Bonfire/1.0.0'}) as response:
data = await response.json()
# The response for votes and options is provided as two separate lists
# We are enumarting the list of options, to print r (the option)
# And the votes to match it, based on the index of the option
# The rest is simple formatting
fmt_options = "\n\t".join(
"{}: {}".format(r, data['votes'][i]) for i, r in enumerate(data['options']))
author = discord.utils.get(ctx.message.server.members, id=poll['author'])
created_ago = (pendulum.utcnow() - pendulum.parse(poll['date'])).in_words()
link = "https://strawpoll.me/{}".format(poll_id)
fmt = "Link: {}\nTitle: {}\nAuthor: {}\nCreated: {} ago\nOptions:\n\t{}".format(link, data['title'],
author.display_name,
created_ago, fmt_options)
await self.bot.say("```\n{}```".format(fmt))
def test_add_to_fixed_timezones():
dt = pendulum.parse("2015-03-08T01:00:00-06:00")
dt = dt.add(weeks=1)
assert_datetime(dt, 2015, 3, 15, 1, 0, 0)
assert dt.timezone_name == "-06:00"
assert dt.offset == -6 * 3600
def get_metadata(self, for_date):
"""
Note that querying for video metadata on a particular date may yield video clips dated for other days.
Their video MMS URLs will match the specified date, but their actual dates can differ.
"""
for mms_url, clips in group_clips(self.get_clips(for_date)).items():
clips = list(clips)
if clips[0].title.startswith('Due to Technical Difficulties'):
continue
# Langley sometimes has meetings that don't have a clip encompassing the entire meeting.
# Create a dummy 'entire meeting' clip that does.
if not any(map(lambda x: is_root_clip(x.title), clips)):
fake_root = InsIncVideoClip(
clips[0].category,
'Entire Meeting',
clips[0].mms_url,
clips[0].for_date,
min(map(lambda x: x.start_time, clips)),
max(map(lambda x: x.start_time, clips)),
)
clips.insert(0, fake_root)
for root, subclips in group_root_and_subclips(clips).items():
start_ts = pendulum.combine(root.for_date, pendulum.parse(root.start_time).time()).tz_(self.tz)
timecodes = [TimeCode(c.start_time, c.title, c.end_time) for c in subclips]
if not timecodes:
timecodes.append(TimeCode(root.start_time, root.title, root.end_time))
yield VideoMetadata(
video_id=os.path.basename(root.mms_url).replace('.wmv', ''),
category=root.category,
title=root.title,
url=root.mms_url,
start_ts=start_ts.isoformat(),
end_ts=root.end_time,
timecodes=timecodes,
)
def get_clips(self, for_date: date):
"""
Get all available video clips for the given local date.
"""
# The '_sl' suffix yields mms:// URLs.
resp = self._search(self.provider_url + '/meeting_search_sl.php',
'search_clips_sl', ['', for_date.strftime('%Y-%m-%d'), ''])
start_bit, end_bit = "+:var res = { \"result\": '", "'}; res;"
body = '"{}"'.format(resp.text[len(start_bit):-1 - len(end_bit)])
body = body.replace("\\n", "\n").replace("\\'", "'").replace('\\"', '"')
parsed_html = BeautifulSoup(body, 'html.parser')
category = None
for element in parsed_html.select('td.gameDate'):
strong, a_link = element.find('strong'), element.find('a')
if strong:
category = str(strong.string).strip()
elif a_link:
# Back up to previous <td> and grab the date.
# Asking for videos on a particular date may yield videos that are for nearby dates,
# but on the same date according to the video URL.
actual_date = str(list(element.previous_siblings)[1].string).strip()
actual_date = pendulum.parse(actual_date).date()
href = a_link['href']
match = re.match(
r"javascript:reload_media_sl\('(mms://[\w\-./]+)', '(\d+:\d+:\d+)', '(\d+:\d+:\d+)'\)", href)
if not match:
continue
mms_url, start_time, end_time = match.group(1), match.group(2), match.group(3)
if start_time == '41:09:00':
start_time = '00:41:09'
title = str(element.string).strip()
yield InsIncVideoClip(category, title, mms_url, actual_date, start_time, end_time)
def postprocess(self, video_metadata, download_dir, destination_dir, **kwargs):
filename_from_video_url = os.path.basename(video_metadata.url)
video_path = os.path.join(download_dir, filename_from_video_url)
if not os.path.exists(video_path):
raise ValueError("{} doesn't exist".format(video_path))
start_timestamp = pendulum.parse(video_metadata.start_ts)
start_timecode = min(chain([m.start_ts for m in video_metadata.timecodes], [start_timestamp.to_time_string()]))
end_timecode = max(chain([m.end_ts for m in video_metadata.timecodes], [video_metadata.end_ts]))
start_timecode = adjust_timecode(start_timecode, -2)
end_timecode = adjust_timecode(end_timecode, 2)
shift_timecodes(video_metadata.timecodes, start_timecode)
filename_parts = filename_from_video_url.split('.')
filename_parts.insert(len(filename_parts)-1, '{}_{}'.format(
start_timecode.replace(':', ''), end_timecode.replace(':', '')))
final_video_filename = '.'.join(filename_parts)
dest_file = os.path.join(destination_dir, final_video_filename)
prepped_video_info = PreparedVideoInfo(video_metadata, final_video_filename)
with open(dest_file + '.yaml', 'w') as outf:
yaml.dump(prepped_video_info, outf)
if os.path.exists(dest_file):
print(dest_file + " already exists")
else:
clip_video(video_path, start_timecode, end_timecode, dest_file)
return prepped_video_info
def test_autoupdate(self, completions, status, new_version, updated):
source_config = factories.SourceConfigFactory()
source_config.harvester.get_class().VERSION = 1
hl = factories.HarvestJobFactory(
status=status,
completions=completions,
harvester_version=source_config.harvester.version,
source_config=source_config,
start_date=pendulum.parse('2017-01-01').date(),
)
source_config.harvester.get_class().VERSION = new_version
tasks.harvest(job_id=hl.id)
hl.refresh_from_db()
if updated:
assert hl.status == HarvestJob.STATUS.succeeded
elif new_version > 1:
assert hl.status == HarvestJob.STATUS.skipped
assert hl.error_context == HarvestJob.SkipReasons.obsolete.value
assert (hl.harvester_version == new_version) == updated
def parse(self, value, instance): # type: (str, base.Entity) -> pendulum.DateTime
config = getattr(instance, '_config', None) or utils.Config.factory()
if isinstance(value, datetime.datetime):
if self._is_naive(value):
return pendulum.instance(value, config.timezone)
return pendulum.instance(value)
elif isinstance(value, pendulum.DateTime):
return value
try:
return pendulum.parse(value, strict=False, dayfirst=config.day_first,
yearfirst=config.year_first)
except AttributeError:
return pendulum.parse(value, strict=False)
def validate_closed_at(self):
if 'closedAt' not in self.data or not self.data['closedAt']:
return False
parsed = pendulum.parse(self.data['closedAt']).in_timezone('Australia/Canberra').start_of('day')
if parsed < pendulum.now('Australia/Canberra').add(days=2).start_of('day'):
return False
return True
def from_json(user_json):
user = user_json["users"]
supplier_code = None
supplier_name = None
application_id = None
notification_count = None
terms_accepted_at = pendulum.parse(user['termsAcceptedAt']).in_tz('UTC')
try:
supplier = user["supplier"]
if supplier:
supplier_code = supplier.get('supplierCode')
supplier_name = supplier.get('name')
notification_count = supplier.get('notificationCount')
except KeyError:
pass
application = user.get('application')
if application:
application_id = application.get('id')
return User(
user_id=user["id"],
email_address=user['emailAddress'],
supplier_code=supplier_code,
supplier_name=supplier_name,
locked=user.get('locked', False),
active=user.get('active', True),
name=user['name'],
role=user['role'],
terms_accepted_at=terms_accepted_at,
application_id=application_id,
notification_count=notification_count
)
def fetch_records(self, url: furl) -> list:
token, used_tokens = None, set()
while True:
records, token = self.fetch_page(url, token=token)
if token in used_tokens:
raise ValueError('Found duplicate resumption token "{}" from {!r}'.format(token, self))
used_tokens.add(token)
for record in records:
datestamp = record.xpath('ns0:header/ns0:datestamp', namespaces=self.namespaces)[0].text
if datestamp:
datestamp = pendulum.parse(datestamp)
else:
datestamp = None
yield (
record.xpath('ns0:header/ns0:identifier', namespaces=self.namespaces)[0].text,
etree.tostring(record, encoding=str),
datestamp,
)
if not token or not records:
break
def fetch_records(self, url, end_day):
page = 1
last_seen_day = None
while True:
page += 1
url.args['page'] = page
resp = self.requests.get(url.url)
if last_seen_day and resp.status_code == 422:
# We've asked for too much. Time to readjust date range
url.args['modified_since'] = last_seen_day.isoformat()
page = 0
continue
for item in resp.json():
resp = self.requests.get(item['url'])
detail = resp.json()
last_seen_day = pendulum.parse(detail['modified_date']).date()
if last_seen_day > end_day:
return
yield item['url'], detail
if len(resp.json()) < self.page_size:
return # We've hit the end of our results
def pendulum_now_mock():
with patch.object(pendulum, "now", return_value=pendulum.parse(END_DATE)):
yield
def datetime_value_parser(value):
"""
Parse datetime strings into datetime objects
"""
return pendulum.parse(value)
def _increase_date_by_month(current_date: str) -> str:
return pendulum.parse(current_date).add(months=1).to_date_string()
def parse(key,
purpose: str,
token: bytes,
encoder=JsonEncoder,
validate: bool = True,
rules=None,
required_claims=None):
"""
Parse a paseto token.
Takes a key, a purpose (which must be either 'local' or 'public'), and
a `token`, which must be a bytes object.
By default, it validates known registered claims (currently just 'exp').
To disable validation, set "validate" to False. Cryptographic validity
cannot be turned off (decryption and authentication are still performed).
You can also turn on/off validation of specific rules by passing a list to
"rules". If you pass an empty list to "rules", you must also specify
"validate=False" or it will raise an exception.
You may pass an alternative encoder if you don't want to use JSON. It
should have loads/dumps methods available, and output a bytes object (not
a str).
:param key: decryption/validation key. Must match the purpose type
:param purpose: one of 'local', 'public'
:param token: bytes object with the raw paseto token
:param encoder: optional encoder to use instead of JSON
:param validate: bool indicating if claims should be validated with rules
:param rules: list of rule names to apply to override the default rules
:param required_claims: list of claim names that must be present (like exp)
:return:
"""
if purpose not in {'local', 'public'}:
raise InvalidPurposeException('invalid purpose')
if not key:
raise ValueError('key is required')
if purpose == 'local':
result = PasetoV2.decrypt(token, key)
else:
result = PasetoV2.verify(token, key)
decoded_message = encoder.loads(result['message'])
decoded_footer = encoder.loads(
result['footer']) if result['footer'] else None
if required_claims:
missing_claims = set(required_claims).difference(
set(decoded_message.keys()))
if missing_claims:
raise PasetoValidationError(
f'required claims missing {missing_claims}')
rules = DEFAULT_RULES if not rules else set(rules)
if validate and not rules:
raise ValueError('must set validate=False to use no rules')
rule_set = {'exp'}
unknown_rules = rules.difference(rule_set)
if unknown_rules:
raise ValueError(f'unknown rules: {unknown_rules}')
if validate:
# validate all the claims
if 'exp' in rules and 'exp' in decoded_message:
# validate expiration
exp = decoded_message['exp']
when = pendulum.parse(exp)
if pendulum.now() > when:
raise PasetoTokenExpired('token expired')
return {'message': decoded_message, 'footer': decoded_footer}
def test_get_report_record_timestamp_hourly():
test_report = TestReport()
test_report.report_aggregation = "Hourly"
assert pendulum.parse("2020-01-01T15:00:00").timestamp(
) == test_report.get_report_record_timestamp("2020-01-01|15")
def from_dict(cls, data: dict) -> "FlowRunLog":
return cls(
cast(pendulum.DateTime, pendulum.parse(data["timestamp"])),
logging.getLevelName(data["level"]), # actually gets level int from name
data["message"],
)
def get_plan_data(self, publish=False):
'''
取得信号楼命令数据
:return:
'''
self.ensure_one()
location = self.plan_out_rail.location
if self.plan_details:
self.plan_details = [(5, 0, 0)]
self._search_route()
if self.state == 'unpublish' and publish:
self.state = 'preparing'
if self.plan_details:
# 道岔和区段
instructs = []
for detail in self.plan_details:
instructs.append(detail.build_instruct())
# 由于存在数据库里为utc时间，所以需要加8小时给前端
date_obj = pendulum.parse(str(self.date))
tmp = utility.time_int_to_time(self.plan_out_time)
if tmp['next_day']:
date_obj.add(days=1)
date_str = date_obj.format('YYYY-MM-DD')
time_str = pendulum.parse(tmp['time']).format('HH:mm:ss')
start_time = pendulum.parse(date_str + " " + time_str)
# 还需要减去排计划的提前量
start_time = start_time.subtract(
minutes=location.plan_rail_pre_min)
real_out_time = None
if self.real_out_time:
tmp = utility.time_int_to_time(self.real_out_time)
date_obj = pendulum.parse(str(self.date))
if tmp['next_day']:
date_obj.add(days=1)
real_out_time = pendulum.parse(
date_str + " " + time_str).format('YYYY-MM-DD HH:mm:ss')
data = {
"id": self.id,
"type":
"train_out", # 计划类型 train_dispatch 调车 train_back 收车 train_out 发车
# 计划执行时间
"start_time": start_time.format('YYYY-MM-DD HH:mm:ss'),
# 完成时间
"end_time": real_out_time,
"trainId": self.train_id.train_no, # 车号
"trainNo": self.real_train_no
if self.real_train_no else self.plan_train_no, # 车次号
"job": '发车', # 作业内容
"operation": "add", # delete update add // 计划状态
"state": self.get_state_name(self.state), # 待执行,信号未开放，执行中，已完成
"start_rail": self.plan_out_rail.no, # 源股道
"start_rail_alias": self.plan_out_rail.alias,
"end_rail": self.plan_out_end_rail.no, # 目标股道
"end_rail_alias": self.plan_out_end_rail.alias,
"instructs": instructs,
"location": self.get_location_spell()
}
return data
else:
return None
def parse_time_of_day(x):
return pendulum.parse(x).time()
def cast_date(value, cur):
if value is None:
return None
return pendulum.parse(value).date()
def test_round_datetime_to_period(dt_str, period, expected_str):
dt = pendulum.parse(dt_str)
expected_dt = pendulum.parse(expected_str)
assert round_datetime_to_period(dt, period) == expected_dt
def _transform_dt(self, time_str):
return pendulum.parse(time_str).in_timezone("UTC")
@pytest.mark.parametrize(
"granularity,intervals_len",
[
(ReportGranularity.LIFETIME, 1),
(ReportGranularity.DAY, 3),
(ReportGranularity.HOUR, 61),
],
)
def test_get_time_interval(pendulum_now_mock, granularity, intervals_len):
intervals = BasicReports._get_time_interval(start_date="2020-01-01", ending_date="2020-03-01", granularity=granularity)
assert len(list(intervals)) == intervals_len
@patch.object(pendulum, "now", return_value=pendulum.parse("2018-12-25"))
def test_get_time_interval_past(pendulum_now_mock_past):
intervals = BasicReports._get_time_interval(start_date="2020-01-01", ending_date="2020-01-01", granularity=ReportGranularity.DAY)
assert len(list(intervals)) == 1
def test_stream_slices_advertisers():
slices = Advertisers(**CONFIG).stream_slices()
assert list(slices) == [None]
@pytest.mark.parametrize(
"config_name, slices_expected",
[
(CONFIG, ADV_IDS),
(CONFIG_SANDBOX, [{"advertiser_id": 2000}]),
def check_for_compatible_agents(labels: Iterable[str], since_minutes: int = 1) -> str:
"""
Checks for agents compatible with a set of labels returning a user-friendly message
indicating the status, roughly one of the following cases:
- There is a healthy agent with matching labels
- There are N healthy agents with matching labels
- There is an unhealthy agent with matching labels but no healthy agents matching
- There are N unhealthy agents with matching labels but no healthy agents matching
- There are no healthy agents at all and no unhealthy agents with matching labels
- There are healthy agents but no healthy or unhealthy agent has matching labels
Args:
- labels: A set of labels; typically associated with a flow run
- since_minutes: The amount of time in minutes to allow an agent to be idle and
considered active/healthy still
Returns:
A message string
"""
client = prefect.Client()
labels = set(labels)
labels_blurb = f"labels {labels!r}" if labels else "empty labels"
result = client.graphql(
{"query": {"agent": {"last_queried", "labels", "name", "id"}}}
)
agents = result.get("data", {}).get("agent")
if agents is None:
raise ValueError(f"Received bad result while querying for agents: {result}")
# Parse last query times
for agent in agents:
agent.last_queried = cast(
Optional[pendulum.DateTime],
pendulum.parse(agent.last_queried) if agent.last_queried else None,
)
# Drop agents that have not queried
agents = [agent for agent in agents if agent.last_queried is not None]
# Drop agents that have not sent a recent hearbeat
since = pendulum.now().subtract(minutes=since_minutes)
healthy_agents = [agent for agent in agents if agent.last_queried >= since]
# Search for the flow run labels in running agents
matching_healthy = []
matching_unhealthy = []
for agent in agents:
empty_labels_match = not agent.labels and not labels
if empty_labels_match or (labels and labels.issubset(agent.labels)):
if agent in healthy_agents:
matching_healthy.append(agent)
else:
matching_unhealthy.append(agent)
if len(matching_healthy) == 1:
agent = matching_healthy[0]
# Display the single matching agent
name_blurb = f" ({agent.name})" if agent.name else ""
return (
f"Agent {agent.id}{name_blurb} has matching labels and last queried "
f"{agent.last_queried.diff_for_humans()}. It should deploy your flow run."
)
if len(matching_healthy) > 1:
# Display that there are multiple matching agents
return (
f"Found {len(matching_healthy)} healthy agents with matching labels. One "
"of them should pick up your flow."
)
# We now know we have no matching healthy agents...
if not healthy_agents and not matching_unhealthy:
# Display that there are no matching agents all-time
return (
"There are no healthy agents in your tenant and it does not look like an "
"agent with the required labels has been run recently. Start an agent with "
f"{labels_blurb} to run your flow."
)
if len(matching_unhealthy) == 1:
agent = matching_unhealthy[0]
# Display that there is a single matching unhealthy agent
name_blurb = f" ({agent.name})" if agent.name else ""
return (
f"Agent {agent.id}{name_blurb} has matching labels and last queried "
f"{agent.last_queried.diff_for_humans()}. Since it hasn't queried recently, it looks "
f"unhealthy. Restart it or start a new agent with {labels_blurb} to deploy "
f"your flow run."
)
if len(matching_unhealthy) > 1:
# Display that there are multiple matching unhealthy agents
return (
f"Found {len(matching_unhealthy)} agents with matching labels but they "
"have not queried recently and look unhealthy. Restart one of them or "
f"start a new agent with {labels_blurb} deploy your flow run."
)
# No matching healthy or unhealthy agents
return (
f"You have {len(healthy_agents)} healthy agents in your tenant but do not have "
f"an agent with {labels_blurb}. Start an agent with matching labels to deploy "
"your flow run."
)
def test_get_report_record_timestamp_daily():
test_report = TestReport()
test_report.report_aggregation = "Daily"
assert pendulum.parse("2020-01-01").timestamp(
) == test_report.get_report_record_timestamp("2020-01-01")
def validate_date_of_birth(date_of_birth):
try:
pendulum.parse(date_of_birth)
except pendulum.parsing.exceptions.ParserError as err:
raise ClientException('dateOfBirth contains timezone information') from err
return True
def test_get_report_record_timestamp_without_aggregation():
test_report = TestReport()
test_report.report_aggregation = None
assert pendulum.parse("2020-07-20").timestamp(
) == test_report.get_report_record_timestamp("7/20/2020")
def generate_day_options(start, stop):
days = period(parse(start), parse(stop)).range('days', 1)
return list([(d.strftime('%Y-%m-%d'), d.strftime('%A %-d %B')) for d in days])
def test_get_updated_state_new_state():
test_report = TestReport()
stream_state = {123: {"Time": pendulum.parse("2020-01-01").timestamp()}}
latest_record = {"AccountId": 123, "Time": "2020-01-02"}
new_state = test_report.get_updated_state(stream_state, latest_record)
assert new_state[123]["Time"] == pendulum.parse("2020-01-02").timestamp()
def sync_endpoint(initial_url, state):
'''Syncs the url and paginates through until there are no more "next"
urls. Yields schema, record, and state messages. Modifies state by
setting the NEXT field every time we get a next url from Shippo. This
allows us to resume paginating if we're terminated.
'''
stream = parse_stream_from_url(initial_url)
yield singer.SchemaMessage(stream=stream,
schema=load_schema(stream),
key_properties=["object_id"])
# The Shippo API does not return data from long ago, so we only try to
# replicate the last 60 days
# Some streams allow us to page by date, so we can request historical data for them
sliding_window_key = SLIDING_WINDOW_STREAMS.get(stream)
if sliding_window_key:
bounded_start = get_start(state)
sliding_query_start = bounded_start
sliding_query_end = bounded_start.add(days=SLIDING_WINDOW_DAYS)
url = initial_url.format(
sliding_window_key,
sliding_query_start.strftime("%Y-%m-%dT%I:%M:%SZ"),
sliding_query_end.strftime("%Y-%m-%dT%I:%M:%SZ"))
else:
bounded_start = max(get_start(state), pendulum.now().subtract(days=60))
url = initial_url
LOGGER.info("Replicating all %s from %s", stream, bounded_start)
rows_read = 0
rows_written = 0
with metrics.record_counter(parse_stream_from_url(url)) as counter:
endpoint_start = pendulum.now()
while url:
state[NEXT] = url
yield singer.StateMessage(value=state)
data = request(url)
for row in data['results']:
counter.increment()
rows_read += 1
updated = pendulum.parse(row[OBJECT_UPDATED])
if updated >= bounded_start:
row = fix_extra_map(row)
yield singer.RecordMessage(stream=stream, record=row)
rows_written += 1
if data.get(NEXT):
url = data.get(NEXT)
elif sliding_window_key and sliding_query_end < endpoint_start:
sliding_query_start = sliding_query_end
sliding_query_end = sliding_query_start.add(
days=SLIDING_WINDOW_DAYS)
url = initial_url.format(
sliding_window_key,
sliding_query_start.strftime("%Y-%m-%dT%I:%M:%SZ"),
sliding_query_end.strftime("%Y-%m-%dT%I:%M:%SZ"))
else:
url = None
if rows_read:
LOGGER.info("Done syncing %s. Read %d records, wrote %d (%.2f%%)",
stream, rows_read, rows_written,
100.0 * rows_written / float(rows_read))
def state(self, value):
self._state = pendulum.parse(value[self.state_pk])
self._start_time = self._state.isoformat()
def get_start(state):
if LAST_START_DATE in state:
return pendulum.parse(state[LAST_START_DATE]).subtract(days=2)
return pendulum.parse(CONFIG[START_DATE])
def __init__(self, config, state):
self.config = self.get_default_config()
self.config.update(config)
self.config['start_date'] = pendulum.parse(self.config['start_date'])
self.state = state
def get(self, key, **kwargs):
'''
This is used to get data from another variable, or another time step, possibly aggregated
'''
try:
hashkey = kwargs.get('hashkey')
parentkey = kwargs.get('parentkey')
date = kwargs.get('date')
date_as_string = date.to_datetime_string()
depth = kwargs.get('depth')
offset = kwargs.get('offset')
timestep = kwargs.get('timestep')
flatten = kwargs.get('flatten', True)
start = kwargs.get('start')
end = kwargs.get('end')
agg = kwargs.get('agg', 'mean')
default = kwargs.get('default')
parts = key.split('/')
if len(parts) == 3:
ref_key, ref_id, attr_id = parts
network_id = None
else:
network_id, ref_key, ref_id, attr_id = parts
network_id = int(ref_id)
# network_id is not used yet - this is for future connections with other networks
ref_id = int(ref_id)
attr_id = int(attr_id)
result = None
value = None
rs_value = self.rs_values.get((ref_key, ref_id, attr_id))
# store results from get function
if key not in self.store:
self.store[key] = EMPTY_VALUES[rs_value['type']]
# calculate offset
offset_date_as_string = None
if offset:
offset_timestep = self.dates.index(date) + offset + 1
else:
offset_timestep = timestep
if offset_timestep < 1 or offset_timestep > len(self.dates):
pass
elif not start or end: # TODO: change this when start/end are added to key
stored_result = None
if rs_value['type'] == 'timeseries':
offset_date = self.dates[offset_timestep - 1]
offset_date_as_string = offset_date.to_datetime_string()
stored_result = self.store[key].get(offset_date_as_string)
elif rs_value['type'] in ['scalar', 'array', 'descriptor']:
stored_result = self.store[key]
if stored_result is not None:
return stored_result
default_flavor = None
if rs_value.type == 'timeseries':
default_flavor = 'native'
elif rs_value.type == 'array':
default_flavor = 'native'
flavor = kwargs.get('flavor', default_flavor)
tattr = self.conn.tattrs[(ref_key, ref_id, attr_id)]
has_blocks = tattr['properties'].get('has_blocks')
# need to evaluate the data anew only as needed
# tracking parent key prevents stack overflow
if key != parentkey:
if rs_value is not None and rs_value['value'] is not None and (
not result or start or end):
eval_data = self.eval_data(
value=rs_value,
do_eval=False,
flavor=flavor,
flatten=flatten,
depth=depth,
parentkey=key,
has_blocks=has_blocks,
tsidx=timestep -
1, # convert from user timestep to python timestep
data_type=tattr.
data_type, # NOTE: the type attribute data type overrides the actual value type
date_format='%Y-%m-%d %H:%M:%S')
self.store[key] = eval_data
value = eval_data
else:
value = self.store[key]
result = value
if self.data_type == 'timeseries':
if rs_value.type == 'timeseries':
if start or end:
start = start or date
end = end or date
if type(start) == str:
start = pendulum.parse(start)
if type(end) == str:
end = pendulum.parse(end)
if key != parentkey:
# start_as_string = start.to_datetime_string()
# end_as_string = end.to_datetime_string()
start_as_string = start.to_iso8601_string(
).replace('Z', '')
end_as_string = end.to_iso8601_string().replace(
'Z', '')
# Note annoying mismatch between Pandas and Pendulum iso8601 implementations
if default_flavor == 'pandas':
result = value.loc[
start_as_string:end_as_string].agg(agg)[0]
elif default_flavor == 'native':
if flatten:
values = value
else:
values = list(value.values())[0]
vals = [
values[k] for k in values.keys()
if start_as_string <= k <= end_as_string
]
if agg == 'mean':
result = numpy.mean(vals)
elif agg == 'sum':
result = numpy.sum(vals)
else:
result = None
elif offset_date_as_string:
if key == parentkey:
# is the result already available from a parent get result? or...
result = self.store.get(
key, {}).get(offset_date_as_string)
if result is None:
# ...from the top-level function?
result = self.hashstore[hashkey][
offset_date_as_string]
else:
if flavor == 'pandas':
if has_blocks:
result = value.loc[offset_date_as_string]
else:
result = value.loc[offset_date_as_string][
0]
else:
if has_blocks:
# temp = value.get(0) or value.get('0') or {}
# result = temp.get(offset_date_as_string)
result = {
c: value[c][offset_date_as_string]
for c in value.keys()
}
else:
result = value.get(offset_date_as_string)
elif rs_value.type == 'array':
result = self.store.get(key)
if result is None:
if flavor == 'pandas':
result = pandas.DataFrame(value)
else:
result = value
elif rs_value.type in ['scalar', 'descriptor']:
result = value
if rs_value.type in ['timeseries', 'periodic timeseries']:
# TODO: account for the fact that key doesn't include start/end
# start/end should be added to key...
if has_blocks:
if 0 not in self.store[key]:
self.store[key] = {0: {}}
self.store[key][0][offset_date_as_string] = result
else:
self.store[key][offset_date_as_string] = result
else:
self.store[key] = result
return result
except:
res_info = key
raise Exception("Error getting data for key {}".format(res_info))
def _apply_conversion_window(self, current_date: str) -> str:
return pendulum.parse(current_date).subtract(
days=self.CONVERSION_WINDOW_DAYS).to_date_string()
def date_decoder(s: str):
if s is None:
return None
return pendulum.parse(s)
def pandas_datetime_to_pendulum_datetime(pd_datetime, tz):
return pendulum.parse(pd_datetime.strftime('%Y-%m-%dT%H:%M:%S%z'), tz=tz)
def _deploy_flow_run(
self,
flow_run: "GraphQLResult",
) -> None:
"""
Deploy a flow run and update Cloud with the resulting deployment info.
If any errors occur when submitting the flow run, capture the error and log to
Cloud.
Args:
- flow_run (GraphQLResult): The specific flow run to deploy
"""
# Deploy flow run and mark failed if any deployment error
try:
# Wait for the flow run's start time. The agent pre-fetches runs that may
# not need to start until up to 10 seconds later so we need to wait to
# prevent the flow from starting early
start_time = pendulum.parse(flow_run.scheduled_start_time)
delay_seconds = max(0,
(start_time - pendulum.now()).total_seconds())
if delay_seconds:
self.logger.debug(
f"Waiting {delay_seconds}s to deploy flow run {flow_run.id} on "
"time...")
time.sleep(delay_seconds)
self.logger.info(
f"Deploying flow run {flow_run.id} to execution environment..."
)
self._mark_flow_as_submitted(flow_run)
# Call the main deployment hook
deployment_info = self.deploy_flow(flow_run)
self.logger.info(f"Completed deployment of flow run {flow_run.id}")
self._safe_write_run_log(
flow_run,
message="Submitted for execution: {}".format(deployment_info),
level="INFO",
)
except Exception as exc:
# On exception, we'll mark this flow as failed
# if first failure was a state update error, we don't want to try another
# state update
if "State update failed" in str(exc):
self.logger.debug("Updating Flow Run state failed: {}".format(
str(exc)))
return
# This is to match existing past behavior, I cannot imagine we would reach
# this point with a flow run that has no id
if not getattr(flow_run, "id"):
self.logger.error("Flow run is missing an id.", exc_info=True)
return
self.logger.error(
f"Encountered exception while deploying flow run {flow_run.id}",
exc_info=True,
)
self._safe_write_run_log(
flow_run,
message=str(exc),
level="ERROR",
)
self._mark_flow_as_failed(flow_run=flow_run, message=str(exc))
self.logger.error(f"Deployment of {flow_run.id} aborted!")
def make_dates(settings, date_format=True, data_type='timeseries'):
# TODO: Make this more advanced
timestep = settings.get('time_step') or settings.get('timestep')
start = settings.get('start_time') or settings.get('start')
end = settings.get('end_time') or settings.get('end')
dates = []
if start and end and timestep:
start_date = pendulum.parse(start)
end_date = pendulum.parse(end)
timestep = timestep.lower()
if data_type == 'periodic timeseries':
start_date = pendulum.datetime(9998, 1, 1)
end_date = pendulum.datetime(9998, 12, 31, 23, 59)
period = pendulum.period(start_date, end_date)
periodic_timesteps = []
if timestep == 'day':
dates = list(period.range("days"))
pt = 0
for i, date in enumerate(dates):
if (date.month, date.day) == (start_date.month,
start_date.day):
pt = 1
else:
pt += 1
periodic_timesteps.append(pt)
elif timestep == 'week':
year = start_date.year
dates = []
for i in range(52 * (end_date.year - start_date.year)):
if i == 0:
date = start_date
else:
date = dates[-1].add(days=7)
if isleap(date.year) and date.month == 3 and date.day == 4:
date = date.add(days=1)
if date.month == 12 and date.day == 31:
date = date.add(days=1)
dates.append(date)
periodic_timesteps.append(i % 52 + 1)
elif timestep == 'month':
dates = list(period.range("months"))
periodic_timesteps = [i % 12 + 1 for i, date in enumerate(dates)]
elif timestep == 'thricemonthly':
dates = []
for dt in period.range('months'):
d1 = pendulum.datetime(dt.year, dt.month, 10)
d2 = pendulum.datetime(dt.year, dt.month, 20)
d3 = dt.last_of('month')
dates.extend([d1, d2, d3])
periodic_timesteps = [i % 36 + 1 for i, date in enumerate(dates)]
dates_as_string = [date.to_datetime_string() for date in dates]
return dates_as_string, dates, periodic_timesteps
else:
return None, None, None
def cast_timestamp(value, cur):
if value is None:
return None
return pendulum.parse(value).naive()
def cast_timestamptz(value, cur):
if value is None:
return None
return pendulum.parse(value).in_timezone("UTC")
