def acquire_lock(self, key, lease_time=3600, **kwargs) -> Lock:
lockfile = os.path.join(self.basepath, f'rssalertbot-{key}.lock')
# create our release callback
def release():
self.release_lock(key)
try:
self.locks[key] = zc.lockfile.LockFile(lockfile, content_template='{pid};{hostname}')
log.debug(f"Acquired lock '{key}' on {lockfile}")
return Lock(release, expires = pendulum.now().add(3600))
except zc.lockfile.LockError:
# we've already got a lockfile here, so we should check the age of it
# - if it's older than the lease time, we can re-acquire, otherwise
# nope!
stats = os.stat(lockfile)
expires = pendulum.from_timestamp(stats.st_mtime)
age = pendulum.now() - expires
if age > lease_time:
log.debug(f"Acquired lock '{key}' on {lockfile}")
return Lock(release, expires)
# no you can't have this lock
log.debug(f"Lock '{key}' denied")
raise LockNotAcquired()
def do_maybe_record_timer(*event):
if not dataview.timer_id:
return
item_id = dataview.timer_id
job_id = dataview.timer_job
hsh = DBITEM.get(doc_id=item_id)
item_info = f"{hsh['itemtype']} {hsh['summary']}"
now = pendulum.now()
if dataview.timer_status == 1: #running
time = dataview.timer_time + (now - dataview.timer_start)
else:
time = dataview.timer_time
completed = pendulum.now()
completed_str = format_datetime(completed)
time_str = format_duration(time)
def coroutine():
dialog = ConfirmDialog("record time", f"item: {item_info}\nelapsed time: {time_str}\n\nrecord time and close timer?")
record_close = yield From(show_dialog_as_float(dialog))
if record_close:
item.record_timer(item_id, job_id, completed, time)
set_text(dataview.show_active_view())
dataview.timer_clear()
if item_id in dataview.itemcache:
del dataview.itemcache[item_id]
loop = get_event_loop()
loop.call_later(0, data_changed, loop)
ensure_future(coroutine())
def do_maybe_cancel_timer(*event):
if not dataview.timer_id:
return
item_id = dataview.timer_id
job_id = dataview.timer_job
hsh = DBITEM.get(doc_id=item_id)
item_info = f"{hsh['itemtype']} {hsh['summary']}"
stopped_timer = False
now = pendulum.now()
if dataview.timer_status == 1: #running
time = dataview.timer_time + (now - dataview.timer_start)
else:
time = dataview.timer_time
completed = pendulum.now()
completed_str = format_datetime(completed)
time_str = format_duration(time)
def coroutine():
dialog = ConfirmDialog("cancel timer", f"item: {item_info}\nelapsed time: {time_str}\n\nclose timer without recording?")
record_cancel = yield From(show_dialog_as_float(dialog))
if record_cancel:
dataview.timer_clear()
set_text(dataview.show_active_view())
get_app().invalidate()
ensure_future(coroutine())
def acquire_lock(self, key, owner_name: str='unknown', lease_time: int=3600) -> Lock:
try:
old = DynamoLock.get(key)
# If the lock is not yet expired, and you aren't the owner, you can't have it
if (pendulum.now('UTC') < old.expires) and old.owner_name != owner_name:
log.debug(f"Lock {key} denied")
raise LockAccessDenied()
# delete the old lock
old.delete()
except DoesNotExist:
pass
# create the new lock
rec = DynamoLock(
key = key,
expires = pendulum.now('UTC').add(seconds = lease_time),
owner_name = owner_name,
)
rec.save()
log.debug(f"Lock {rec.key} acquired, expires {rec.expires}")
def release():
self.release_lock(key, owner_name)
# return lock
return Lock(release, expires=rec.expires)
def test_diff_for_humans_absolute_years():
with pendulum.test(pendulum.datetime(2012, 1, 1, 1, 2, 3)):
assert "1 year" == pendulum.now().diff_for_humans(
pendulum.now().subtract(years=1), True
)
assert "1 year" == pendulum.now().diff_for_humans(
pendulum.now().add(years=1), True
)
def test_diff_for_humans_absolute_months():
with pendulum.test(pendulum.datetime(2012, 1, 1, 1, 2, 3)):
assert "2 months" == pendulum.now().diff_for_humans(
pendulum.now().subtract(months=2), True
)
assert "2 months" == pendulum.now().diff_for_humans(
pendulum.now().add(months=2), True
)
def test_diff_for_humans_absolute_hours():
with pendulum.test(pendulum.datetime(2012, 1, 1, 1, 2, 3)):
assert "3 hours" == pendulum.now().diff_for_humans(
pendulum.now().subtract(hours=3), True
)
assert "3 hours" == pendulum.now().diff_for_humans(
pendulum.now().add(hours=3), True
)
def test_diff_for_humans_absolute_minutes():
with pendulum.test(pendulum.datetime(2012, 1, 1, 1, 2, 3)):
assert "30 minutes" == pendulum.now().diff_for_humans(
pendulum.now().subtract(minutes=30), True
)
assert "30 minutes" == pendulum.now().diff_for_humans(
pendulum.now().add(minutes=30), True
)
def test_diff_for_humans_absolute_seconds():
with pendulum.test(pendulum.datetime(2012, 1, 1, 1, 2, 3)):
assert "59 seconds" == pendulum.now().diff_for_humans(
pendulum.now().subtract(seconds=59), True
)
assert "59 seconds" == pendulum.now().diff_for_humans(
pendulum.now().add(seconds=59), True
)
def test_diff_for_humans_absolute_hours():
with pendulum.test(pendulum.today().at(12, 34, 56)):
now = pendulum.now().time()
assert now.diff_for_humans(now.subtract(hours=3), True) == "3 hours"
now = pendulum.now().time()
assert now.diff_for_humans(now.add(hours=3), True) == "3 hours"
def test_diff_for_humans_other_and_month():
with pendulum.test(pendulum.datetime(2012, 1, 1, 1, 2, 3)):
assert "4 weeks before" == pendulum.now().diff_for_humans(
pendulum.now().add(weeks=4)
)
assert "1 month before" == pendulum.now().diff_for_humans(
pendulum.now().add(months=1)
)
def test_diff_for_humans_other_and_future_month():
with pendulum.test(pendulum.datetime(2012, 1, 1, 1, 2, 3)):
assert "4 weeks after" == pendulum.now().diff_for_humans(
pendulum.now().subtract(weeks=4)
)
assert "1 month after" == pendulum.now().diff_for_humans(
pendulum.now().subtract(months=1)
)
def test_diff_for_humans_absolute_minutes():
with pendulum.test(pendulum.today().at(12, 34, 56)):
now = pendulum.now().time()
assert now.diff_for_humans(now.subtract(minutes=30), True) == "30 minutes"
now = pendulum.now().time()
assert now.diff_for_humans(now.add(minutes=30), True) == "30 minutes"
def test_diff_for_humans_absolute_seconds():
with pendulum.test(pendulum.today().at(12, 34, 56)):
now = pendulum.now().time()
assert now.diff_for_humans(now.subtract(seconds=59), True) == "59 seconds"
now = pendulum.now().time()
assert now.diff_for_humans(now.add(seconds=59), True) == "59 seconds"
def test_pointless(self):
job = factories.IngestJobFactory(raw__datestamp=pendulum.now().subtract(hours=2))
factories.IngestJobFactory(suid=job.suid, raw__datestamp=pendulum.now().subtract(hours=1))
ingest(job_id=job.id)
job.refresh_from_db()
assert job.status == job.STATUS.skipped
assert job.error_context == job.SkipReasons.pointless.value
def __call__(self, loop):
start = pendulum.now()
ps = yield from asyncio.create_subprocess_shell(
self.cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
loop=loop)
output = (yield from ps.communicate())[0]
self.elapsed += pendulum.now() - start
self.run_count += 1
return ps, output
def __repr__(self):
repr = """{simulation_name:=^30}
created: {created_date}
started: {started_date}
last: {last_date}
time: {t:g}
iteration: {iter:g}
last step: {step_time}
total time: {running_time}
Physical parameters
-------------------
{parameters}
Hook function
-------------
{hook_source}
=========== Model ===========
{model_repr}"""
repr = repr.format(simulation_name=" %s " % self.id,
parameters="\n\t".join(
[("%s:" % key).ljust(12) +
pprint.pformat(value)
for key, value
in self.parameters.items()]),
t=self.t,
iter=self.i,
model_repr=self.model,
hook_source=inspect.getsource(self._hook),
step_time=(None if not self._last_running else
pendulum.now()
.subtract(
seconds=self._last_running)
.diff()),
running_time=(pendulum.now()
.subtract(
seconds=self._total_running)
.diff()),
created_date=(self._created_timestamp
.to_cookie_string()),
started_date=(self._started_timestamp
.to_cookie_string()
if self._started_timestamp
else "None"),
last_date=(self._last_timestamp
.to_cookie_string()
if self._last_timestamp
else "None"))
return repr
def __repr__(self):
repr = """last: {last}
total: {total}"""
return repr.format(last=(pendulum.now()
.subtract(
seconds=self.last)
.diff()),
total=(pendulum.now()
.subtract(
seconds=self.total)
.diff()))
def __init__(self, model, fields, parameters, dt, t=0, tmax=None,
id=None, hook=null_hook,
scheme=schemes.RODASPR,
time_stepping=True, **kwargs):
def intersection_kwargs(kwargs, function):
"""Inspect the function signature to identify the relevant keys
in a dictionary of named parameters.
"""
func_signature = inspect.signature(function)
func_parameters = func_signature.parameters
kwargs = {key: value
for key, value
in kwargs.items() if key in func_parameters}
return kwargs
kwargs["time_stepping"] = time_stepping
self.id = str(uuid1())[:6] if not id else id
self.model = model
self.parameters = parameters
self.fields = model.fields_template(**fields)
self.t = t
self.user_dt = self.dt = dt
self.tmax = tmax
self.i = 0
self._stream = streamz.Stream()
self._pprocesses = []
self._scheme = scheme(model,
**intersection_kwargs(kwargs,
scheme.__init__))
if (time_stepping and
self._scheme not in [schemes.RODASPR,
schemes.ROS3PRL,
schemes.ROS3PRw]):
self._scheme = schemes.time_stepping(
self._scheme,
**intersection_kwargs(kwargs,
schemes.time_stepping))
self.status = 'created'
self._total_running = 0
self._last_running = 0
self._created_timestamp = pendulum.now()
self._started_timestamp = None
self._last_timestamp = None
self._actual_timestamp = pendulum.now()
self._hook = hook
self._container = None
self._iterator = self.compute()
def test_is_birthday():
with pendulum.test(pendulum.now()):
d = pendulum.now()
a_birthday = d.subtract(years=1)
assert a_birthday.is_birthday()
not_a_birthday = d.subtract(days=1)
assert not not_a_birthday.is_birthday()
also_not_a_birthday = d.add(days=2)
assert not also_not_a_birthday.is_birthday()
d1 = pendulum.datetime(1987, 4, 23)
d2 = pendulum.datetime(2014, 9, 26)
d3 = pendulum.datetime(2014, 4, 23)
assert not d2.is_birthday(d1)
assert d3.is_birthday(d1)
def sign_in(mocker, user):
secret = os.environ.get('JWT_SECRET', 'secret')
claims = dict(sub=user.id, iat=pendulum.now(), exp=pendulum.tomorrow())
mocker.patch.object(jwt, 'decode', return_value=claims)
access_token = jwt.encode(claims, secret).decode("utf-8")
return f'JWT {access_token}'
def holiday_cmd(self, mask, target, args):
"""
Show the upcoming non-weekend holidays
for all countries in config.
%%holiday
"""
country_codes = self.config.get("holiday_countries")
if not country_codes:
return
now = pendulum.now(tz=self.config.get("default_timezone"))
for country_code in country_codes.split(","):
date, name = get_next_non_weekend_holiday(now, country_code)
remaining_days = (
pendulum.date(date.year, date.month, date.day)
- now
).in_days()
plural = "" if remaining_days == 1 else "s"
yield (
f"Next holiday for {country_code}:"
f" {date.isoformat()} ({name}"
f", in {remaining_days} day{plural})"
)
async def reset(request, token):
try:
reset = db.session.query(PasswordReset).filter_by(UUID=token).first()
if not reset:
return response.json({"error": "Invalid reset token"}, 404)
if not reset.isValid:
return response.json({"error": "Reset token has already been used"}, 404)
if pendulum.now("UTC") > pendulum.instance(reset.expireTime):
return response.json({"error": "Reset token has expired."}, 400)
# Invalidate all resets for this user
# db.session.query(PasswordReset).filter_by(userId=reset.userId).update(
# {"isValid": False}
# )
db.session.commit()
user = utils.get_account_by_id(reset.userId)
userData = user.serialize()
userData["jwt"] = user.gen_token(expire_hours=1)
userData["message"] = "Valid token provided. Prompt user to change password"
return response.json(userData, 200)
except Exception as e:
return utils.exeption_handler(e, "Password reset confirmation failed", 500)
def elig(bot, trigger):
"""Cloak eligibility tool"""
need_verified_email = True
need_edit_count = 250
need_months_registered = 3
if not trigger.group(2):
return bot.say('To be eligible for a cloak you need: ' + ['', 'a verified email, '][need_verified_email] + 'to be registered for ' + str(need_months_registered) + ' months,' + ' and have ' + str(need_edit_count) + ' edits.')
query = trigger.group(3)
r = requests.get('https://en.wikipedia.org/w/api.php?action=query&meta=globaluserinfo&guiuser='******'&guiprop=editcount&format=json')
try:
required_registration_time = pendulum.now().subtract(months=need_months_registered)
actual_registration_time = pendulum.parse(str(r.json()['query']['globaluserinfo']['registration']))
except KeyError:
return bot.say('No such user (please note that usernames are case-sensitive).')
if actual_registration_time > required_registration_time:
return bot.say('User: "******" is ineligible for a cloak because they have not been registered onwiki for long enough (there may be other reasons why this user is ineligible).')
actual_edit_count = r.json()['query']['globaluserinfo']['editcount']
if actual_edit_count < need_edit_count:
return bot.say('User: "******" is ineligible for a cloak because their edit count is not high enough. They need ' + str(need_edit_count - actual_edit_count) + ' more edits to be eligible under this criterion (there may be other reasons why this user is ineligible).')
r2 = requests.get('https://en.wikipedia.org/w/api.php?action=query&list=users&ususers=' + query + '&usprop=emailable&format=json')
try:
r2.json()['query']['users'][0]['emailable']
except KeyError:
return bot.say('User: "******" is ineligible for a cloak because they do not have a verified email address (this problem may be caused be the user disabling Special:EmailUser in their English Wikipedia preferences).')
bot.say('User: "******" is eligible for a cloak. They registered onwiki about ' + actual_registration_time.diff_for_humans() + ', have ' + str(actual_edit_count) + ' edits and have a verified email address.')
def compute(self):
"""Generator which yield the actual state of the system every dt.
Yields
------
tuple : t, fields
Actual time and updated fields container.
"""
fields = self.fields
t = self.t
pars = self.parameters
self._started_timestamp = pendulum.now()
self.stream.emit(self)
try:
while True:
t, fields, pars = self._compute_one_step(t, fields, pars)
self.i += 1
self.t = t
self.fields = fields
self.parameters = pars
for pprocess in self.post_processes:
pprocess.function(self)
self.stream.emit(self)
yield self.t, self.fields
if self.tmax and (isclose(self.t, self.tmax)):
self._end_simulation()
return
except RuntimeError:
self.status = 'failed'
raise
def user(id, email_address, supplier_code, supplier_name, name, is_token_valid=True, locked=False, active=True,
role='buyer', terms_accepted_at=None):
now = pendulum.now('UTC')
hours_offset = -1 if is_token_valid else 1
password_changed_date = now + timedelta(hours=hours_offset)
if terms_accepted_at is None:
terms_accepted_at = now
user = {
"id": id,
"emailAddress": email_address,
"name": name,
"role": role,
"locked": locked,
'active': active,
'passwordChangedAt': password_changed_date.to_iso8601_string(),
'termsAcceptedAt': terms_accepted_at.to_iso8601_string()
}
if supplier_code:
supplier = {
"supplierCode": supplier_code,
"name": supplier_name,
}
user['role'] = 'supplier'
user['supplier'] = supplier
return {
"users": user
}
def diff(self, dt=None, abs=True):
"""
Returns the difference between two Time objects as an Duration.
:type dt: Time or None
:param abs: Whether to return an absolute interval or not
:type abs: bool
:rtype: Duration
"""
if dt is None:
dt = pendulum.now().time()
else:
dt = self.__class__(dt.hour, dt.minute, dt.second, dt.microsecond)
us1 = (
self.hour * SECS_PER_HOUR + self.minute * SECS_PER_MIN + self.second
) * USECS_PER_SEC
us2 = (
dt.hour * SECS_PER_HOUR + dt.minute * SECS_PER_MIN + dt.second
) * USECS_PER_SEC
klass = Duration
if abs:
klass = AbsoluteDuration
return klass(microseconds=us2 - us1)
def get_domain_metrics():
metrics = {}
query = '''
SELECT name, count(status), status::TEXT
FROM
supplier_domain INNER JOIN domain ON supplier_domain.domain_id = domain.id
WHERE status = 'assessed' OR status = 'unassessed'
GROUP BY name, status::TEXT
UNION
SELECT key, count(*),
(CASE
WHEN application.status = 'submitted' THEN 'submitted'
WHEN application.status = 'saved' THEN 'unsubmitted'
END) status
FROM
application, json_each(application.data->'services') badge
WHERE "value"::TEXT = 'true'
AND (application.status = 'saved' OR application.status = 'submitted')
AND (application.type = 'new' OR application.type = 'upgrade')
GROUP BY key, status
'''
for (domain, count, status) in db.session.execute(query).fetchall():
if domain not in metrics:
metrics[domain] = {}
metrics[domain]['domain'] = domain
metrics[domain]['timestamp'] = pendulum.now().to_iso8601_string()
metrics[domain][status] = count
metrics = list(metrics.values())
return jsonify(metrics)
def validate_closed_at(self):
if 'closedAt' not in self.data or not self.data['closedAt']:
return False
parsed = pendulum.parse(self.data['closedAt']).in_timezone('Australia/Canberra').start_of('day')
if parsed < pendulum.now('Australia/Canberra').add(days=2).start_of('day'):
return False
return True
def test_read_with_empty_metadata(self):
ts = pendulum.now()
logs, metadatas = self.es_task_handler.read(self.ti, 1, {})
self.assertEqual(1, len(logs))
self.assertEqual(len(logs), len(metadatas))
self.assertEqual(self.test_message, logs[0])
self.assertFalse(metadatas[0]['end_of_log'])
# offset should be initialized to 0 if not provided.
self.assertEqual(1, metadatas[0]['offset'])
# last_log_timestamp will be initialized using log reading time
# if not last_log_timestamp is provided.
self.assertTrue(timezone.parse(metadatas[0]['last_log_timestamp']) > ts)
# case where offset is missing but metadata not empty.
self.es.delete(index=self.index_name, doc_type=self.doc_type, id=1)
logs, metadatas = self.es_task_handler.read(self.ti, 1, {'end_of_log': False})
self.assertEqual(1, len(logs))
self.assertEqual(len(logs), len(metadatas))
self.assertEqual([''], logs)
self.assertFalse(metadatas[0]['end_of_log'])
# offset should be initialized to 0 if not provided.
self.assertEqual(0, metadatas[0]['offset'])
# last_log_timestamp will be initialized using log reading time
# if not last_log_timestamp is provided.
self.assertTrue(timezone.parse(metadatas[0]['last_log_timestamp']) > ts)
def test_sensor_tick_range(graphql_context):
external_repository = graphql_context.get_repository_location(
main_repo_location_name()).get_repository(main_repo_name())
graphql_context.instance.reconcile_scheduler_state(external_repository)
sensor_name = "always_no_config_sensor"
external_sensor = external_repository.get_external_sensor(sensor_name)
sensor_selector = infer_sensor_selector(graphql_context, sensor_name)
# test with no job state
result = execute_dagster_graphql(
graphql_context,
GET_SENSOR_TICK_RANGE_QUERY,
variables={
"sensorSelector": sensor_selector,
"dayRange": None,
"dayOffset": None
},
)
assert len(result.data["sensorOrError"]["sensorState"]["ticks"]) == 0
# turn the sensor on
graphql_context.instance.add_job_state(
JobState(external_sensor.get_external_origin(), JobType.SENSOR,
JobStatus.RUNNING))
now = pendulum.now("US/Central")
one = now.subtract(days=2).subtract(hours=1)
with pendulum.test(one):
_create_tick(graphql_context.instance)
two = now.subtract(days=1).subtract(hours=1)
with pendulum.test(two):
_create_tick(graphql_context.instance)
three = now.subtract(hours=1)
with pendulum.test(three):
_create_tick(graphql_context.instance)
result = execute_dagster_graphql(
graphql_context,
GET_SENSOR_TICK_RANGE_QUERY,
variables={
"sensorSelector": sensor_selector,
"dayRange": None,
"dayOffset": None
},
)
assert len(result.data["sensorOrError"]["sensorState"]["ticks"]) == 3
result = execute_dagster_graphql(
graphql_context,
GET_SENSOR_TICK_RANGE_QUERY,
variables={
"sensorSelector": sensor_selector,
"dayRange": 1,
"dayOffset": None
},
)
assert len(result.data["sensorOrError"]["sensorState"]["ticks"]) == 1
assert result.data["sensorOrError"]["sensorState"]["ticks"][0][
"timestamp"] == three.timestamp()
result = execute_dagster_graphql(
graphql_context,
GET_SENSOR_TICK_RANGE_QUERY,
variables={
"sensorSelector": sensor_selector,
"dayRange": 1,
"dayOffset": 1
},
)
assert len(result.data["sensorOrError"]["sensorState"]["ticks"]) == 1
assert result.data["sensorOrError"]["sensorState"]["ticks"][0][
"timestamp"] == two.timestamp()
result = execute_dagster_graphql(
graphql_context,
GET_SENSOR_TICK_RANGE_QUERY,
variables={
"sensorSelector": sensor_selector,
"dayRange": 2,
"dayOffset": None,
},
)
assert len(result.data["sensorOrError"]["sensorState"]["ticks"]) == 2
def add_log(msg):
logging = "{} - {} \n".format(pendulum.now('Asia/Shanghai').to_datetime_string(), msg)
with open("./amac_log.txt", "a") as f:
f.write(logging)
def generate_worksheet(self, workbook, stats_type):
formatters = self.setup_workbook_formatters(workbook)
if stats_type == 'aggregated':
write = self.write_total_stats
create_worksheet = self.create_totals_worksheet
elif stats_type == 'location_segmentation':
write = self.write_location_segmentation_stats
create_worksheet = self.create_location_segmentation_worksheet
elif stats_type == 'theme_segmentation':
write = self.write_theme_segmentation_stats
create_worksheet = self.create_theme_segmentation_worksheet
elif stats_type == 'impact_survey':
write = self.write_impact_survey_stats
create_worksheet = self.create_impact_survey_worksheet
for year in range(self.start_date.year, self.end_date.year + 1):
statistics_start_date = pendulum.create(year, 1, 1, 0, 0, 0)
statistics_end_date = pendulum.create(year, 12, 31, 23, 59, 59)
# Generate data by year
logger.info('tenant:{} Yearly ({}): end_date:{}'.format(
self.tenant, stats_type,
statistics_end_date.to_iso8601_string()))
# Worksheet for Totals by Year
worksheet = create_worksheet(workbook, year)
# Add label
row = 1
worksheet.write(row, 0, 'By Year', formatters['metrics_header'])
row += 1
write(worksheet=worksheet,
row=row,
statistic_type='yearly',
start_date=statistics_start_date,
end_date=statistics_end_date)
# Generate data by month
row += 1
worksheet.write(row, 0, 'By Month', formatters['metrics_header'])
row += 1
self.monthly_statistics_row_start = row
for month in range(1, 13):
statistics_end_date = pendulum.create(year, month,
1).end_of('month')
# Stop if the end date is in the next month from current date
if statistics_end_date < pendulum.now().add(months=1):
logger.info('tenant:{} Monthly ({}): end_date:{}'.format(
self.tenant, stats_type,
statistics_end_date.to_iso8601_string()))
write(worksheet=worksheet,
row=row,
statistic_type='monthly',
start_date=statistics_start_date,
end_date=statistics_end_date)
row += 1
self.monthly_statistics_row_end = row
# Generate data by week
worksheet.write(row, 0, 'By Week', formatters['metrics_header'])
time_period = pendulum.period(statistics_start_date,
pendulum.create(year, 12, 31))
row += 1
self.weekly_statistics_row_start = row
for period in time_period.range('weeks'):
# Curtail the last week of the year to end with the end of the year
# E.g. The end day of the last week of the year could lie in the next year, in this case we just
# use the last day of the year as the end day of the week
statistics_end_date = period.end_of('week') \
if period.end_of('week') < statistics_start_date.end_of('year') \
else statistics_start_date.end_of('year')
if statistics_end_date <= pendulum.now().add(weeks=1):
logger.info('tenant:{} Weekly ({}): end_date:{}'.format(
self.tenant, stats_type,
statistics_end_date.to_iso8601_string()))
write(worksheet=worksheet,
row=row,
statistic_type='weekly',
start_date=statistics_start_date,
end_date=statistics_end_date)
row += 1
self.weekly_statistics_row_end = row - 1
if stats_type == 'aggregated':
self.create_monthly_charts(workbook, self.charts)
def sync_table(connection, catalog_entry, state, columns):
# If there is an existing bookmark, use it; otherwise, use replication_key
replication_key_value = None
if not state.get("bookmarks", {}).get(catalog_entry.tap_stream_id):
singer.write_bookmark(state, catalog_entry.tap_stream_id,
catalog_entry.replication_key, None)
else:
# Start with the bookmark.
replication_key_value = singer.get_bookmark(
state, catalog_entry.tap_stream_id, catalog_entry.replication_key)
# with connection.cursor() as cursor:
cursor = connection.cursor()
# Build the sql for this stream.
select_sql = generate_select_sql(catalog_entry, columns)
# If bookmark exists, modify the query.
if replication_key_value is not None:
if (catalog_entry.schema.properties[
catalog_entry.replication_key].format == "date-time"):
replication_key_value = pendulum.parse(replication_key_value)
replication_key_max = pendulum.now("UTC").subtract(minutes=20)
select_sql += " WHERE {0} >= '{1}' AND {0} < {2} ORDER BY {3} ASC".format(
catalog_entry.replication_key,
replication_key_value,
replication_key_max,
catalog_entry.replication_key,
)
elif catalog_entry.replication_key is not None:
select_sql += " ORDER BY {} ASC".format(catalog_entry.replication_key)
# time to sync.
LOGGER.info("Running %s", select_sql)
cursor.execute(select_sql)
row = cursor.fetchone()
rows_saved = 0
with metrics.record_counter(catalog_entry.tap_stream_id) as counter:
counter.tags["table"] = catalog_entry.table
while row:
counter.increment()
rows_saved += 1
# format record
rec = process_row(row, columns)
# resolve against metadata
with Transformer() as transformer:
rec = transformer.transform(
rec,
catalog_entry.schema.to_dict(),
metadata.to_map(catalog_entry.metadata),
)
# write to singer.
singer.write_record(catalog_entry.tap_stream_id, rec)
# Perhaps the more modern way of managing state.
if catalog_entry.replication_method == "INCREMENTAL":
singer.write_bookmark(
state,
catalog_entry.tap_stream_id,
catalog_entry.replication_key,
rec[catalog_entry.replication_key],
)
if rows_saved % 100 == 0:
singer.write_state(state)
row = cursor.fetchone()
singer.write_state(state)
return counter.value
async def amain(
backend_address="ws://*****:*****@laptop",
bob_device_id="bob@laptop",
other_device_name="pc",
alice_workspace="alicews",
bob_workspace="bobws",
password="******",
administrator_token=DEFAULT_ADMINISTRATOR_TOKEN,
force=False,
):
configure_logging("WARNING")
config_dir = get_default_config_dir(os.environ)
organization_id = OrganizationID(organization_id)
backend_address = BackendAddr(backend_address)
alice_device_id = DeviceID(alice_device_id)
bob_device_id = DeviceID(bob_device_id)
alice_slugid = f"{organization_id}:{alice_device_id}"
bob_slugid = f"{organization_id}:{bob_device_id}"
# Create organization
async with backend_administrator_cmds_factory(backend_address, administrator_token) as cmds:
bootstrap_token = await cmds.organization_create(organization_id)
organization_bootstrap_addr = BackendOrganizationBootstrapAddr.build(
backend_address, organization_id, bootstrap_token
)
# Bootstrap organization and Alice user
async with backend_anonymous_cmds_factory(organization_bootstrap_addr) as cmds:
root_signing_key = SigningKey.generate()
root_verify_key = root_signing_key.verify_key
organization_addr = organization_bootstrap_addr.generate_organization_addr(root_verify_key)
alice_device = generate_new_device(alice_device_id, organization_addr)
save_device_with_password(config_dir, alice_device, password, force=force)
now = pendulum.now()
certified_user = certify_user(
None, root_signing_key, alice_device.user_id, alice_device.public_key, now
)
certified_device = certify_device(
None, root_signing_key, alice_device_id, alice_device.verify_key, now
)
await cmds.organization_bootstrap(
organization_bootstrap_addr.organization_id,
organization_bootstrap_addr.bootstrap_token,
root_verify_key,
certified_user,
certified_device,
)
# Create a workspace for Alice
config = load_config(config_dir, debug="DEBUG" in os.environ)
async with logged_core_factory(config, alice_device) as core:
await core.fs.workspace_create(f"/{alice_workspace}")
# Register a new device for Alice
token = generate_invitation_token()
other_alice_device_id = DeviceID("@".join((alice_device.user_id, other_device_name)))
other_alice_slugid = f"{organization_id}:{other_alice_device_id}"
async def invite_task():
async with backend_cmds_factory(
alice_device.organization_addr, alice_device.device_id, alice_device.signing_key
) as cmds:
await invite_and_create_device(alice_device, cmds, other_device_name, token)
async def claim_task():
async with backend_anonymous_cmds_factory(alice_device.organization_addr) as cmds:
other_alice_device = await retry(claim_device, cmds, other_alice_device_id, token)
save_device_with_password(config_dir, other_alice_device, password, force=force)
async with trio.open_nursery() as nursery:
nursery.start_soon(invite_task)
nursery.start_soon(claim_task)
# Invite Bob in
token = generate_invitation_token()
async def invite_task():
async with backend_cmds_factory(
alice_device.organization_addr, alice_device.device_id, alice_device.signing_key
) as cmds:
await invite_and_create_user(
alice_device, cmds, bob_device_id.user_id, token, is_admin=True
)
async def claim_task():
async with backend_anonymous_cmds_factory(alice_device.organization_addr) as cmds:
bob_device = await retry(claim_user, cmds, bob_device_id, token)
save_device_with_password(config_dir, bob_device, password, force=force)
async with trio.open_nursery() as nursery:
nursery.start_soon(invite_task)
nursery.start_soon(claim_task)
# Create bob workspace and share with Alice
bob_device = load_device_with_password(
config.config_dir, organization_id, bob_device_id, password
)
async with logged_core_factory(config, bob_device) as core:
await core.fs.workspace_create(f"/{bob_workspace}")
await core.fs.share(f"/{bob_workspace}", alice_device_id.user_id)
# Share Alice workspace with bob
async with logged_core_factory(config, alice_device) as core:
await core.fs.share(f"/{alice_workspace}", bob_device_id.user_id)
# Print out
click.echo(
f"""
Mount alice and bob drives using:
$ parsec core run -P {password} -D {alice_slugid}
$ parsec core run -P {password} -D {other_alice_slugid}
$ parsec core run -P {password} -D {bob_slugid}
"""
)
async def api_realm_update_roles(self, client_ctx, msg):
msg = realm_update_roles_serializer.req_load(msg)
try:
data = RealmRoleCertificateContent.verify_and_load(
msg["role_certificate"],
author_verify_key=client_ctx.verify_key,
expected_author=client_ctx.device_id,
)
except DataError as exc:
return {
"status": "invalid_certification",
"reason": f"Invalid certification data ({exc}).",
}
now = pendulum.now()
if not timestamps_in_the_ballpark(data.timestamp, now):
return {
"status": "invalid_certification",
"reason": "Invalid timestamp in certification.",
}
granted_role = RealmGrantedRole(
certificate=msg["role_certificate"],
realm_id=data.realm_id,
user_id=data.user_id,
role=data.role,
granted_by=data.author,
granted_on=data.timestamp,
)
if granted_role.granted_by.user_id == granted_role.user_id:
return {
"status": "invalid_data",
"reason": "Realm role certificate cannot be self-signed.",
}
try:
await self.update_roles(client_ctx.organization_id, granted_role,
msg["recipient_message"])
except RealmRoleAlreadyGranted:
return realm_update_roles_serializer.rep_dump(
{"status": "already_granted"})
except RealmAccessError:
return realm_update_roles_serializer.rep_dump(
{"status": "not_allowed"})
except RealmIncompatibleProfileError as exc:
return realm_update_roles_serializer.rep_dump({
"status": "incompatible_profile",
"reason": str(exc)
})
except RealmNotFoundError as exc:
return realm_update_roles_serializer.rep_dump({
"status": "not_found",
"reason": str(exc)
})
except RealmInMaintenanceError:
return realm_update_roles_serializer.rep_dump(
{"status": "in_maintenance"})
return realm_update_roles_serializer.rep_dump({"status": "ok"})
def get_last_months(months):
return pendulum.now(tz).subtract(months=months).start_of('month'), pendulum.now(tz)
async def api_realm_create(self, client_ctx, msg):
if client_ctx.profile == UserProfile.OUTSIDER:
return {
"status": "not_allowed",
"reason": "Outsider user cannot create realm"
}
msg = realm_create_serializer.req_load(msg)
try:
data = RealmRoleCertificateContent.verify_and_load(
msg["role_certificate"],
author_verify_key=client_ctx.verify_key,
expected_author=client_ctx.device_id,
)
except DataError as exc:
return {
"status": "invalid_certification",
"reason": f"Invalid certification data ({exc}).",
}
now = pendulum.now()
if not timestamps_in_the_ballpark(data.timestamp, now):
return {
"status": "invalid_certification",
"reason": "Invalid timestamp in certification.",
}
granted_role = RealmGrantedRole(
certificate=msg["role_certificate"],
realm_id=data.realm_id,
user_id=data.user_id,
role=data.role,
granted_by=data.author,
granted_on=data.timestamp,
)
if granted_role.granted_by.user_id != granted_role.user_id:
return {
"status": "invalid_data",
"reason":
"Initial realm role certificate must be self-signed.",
}
if granted_role.role != RealmRole.OWNER:
return {
"status": "invalid_data",
"reason":
"Initial realm role certificate must set OWNER role.",
}
try:
await self.create(client_ctx.organization_id, granted_role)
except RealmNotFoundError as exc:
return realm_create_serializer.rep_dump({
"status": "not_found",
"reason": str(exc)
})
except RealmAlreadyExistsError:
return realm_create_serializer.rep_dump(
{"status": "already_exists"})
return realm_create_serializer.rep_dump({"status": "ok"})
def _filter(self, data, strategies, wxcampaign: CampaignHelper):
# today = pendulum.today().to_datetime_string()
# yesterday = pendulum.yesterday().to_datetime_string()
# yesterday_count = Point.where('created_at', '<', today)\
# .where('created_at', '>', yesterday)\
# .where('campaign_id', data['campaign_id'])\
# .where('status', ADSTATUS_NORMAL)\
# .order_by('id', 'desc')\
# .first()
#
# before_yesterday_count = Point.where('created_at', '<', yesterday)\
# .where('campaign_id', data['campaign_id']) \
# .where('status', ADSTATUS_NORMAL)\
# .order_by('id', 'desc')\
# .first()
today = pendulum.today()
yesterday = pendulum.yesterday()
now = pendulum.now()
actions = []
delivery_start_time, delivery_end_time = wxcampaign.delivery_time()
cur_hour = now.hour
may_missing_key = [
'1day_action_step', '1day_action_reversion',
'1day_action_complete_order', '1day_action_complete_order_amount'
]
for key in may_missing_key:
if key not in data:
data[key] = 0
if delivery_start_time >= yesterday:
'''
凌晨1点-2点，新计划指最近一天建立的，
话费／点击次数>10 || （当天花费>190 且没有订单）， 生效时间改到6点
'''
if (int(data['click_count']) == 0 or int(data['total_cost']) / 100 / int(data['click_count']) > 10) or \
(data['total_cost'] > 19000 and data['1day_action_complete_order'] == 0):
actions.append(build_command(data, 'timeset_end', 6))
elif yesterday <= delivery_start_time < today:
'''
次新计划，满足上述条件直接暂停
'''
if (int(data['click_count']) == 0 or int(data['total_cost']) / 100 / int(data['click_count']) > 10) or \
(data['total_cost'] > 19000 and data['1day_action_complete_order'] == 0):
actions.append(build_command(data, 'suspend', None))
else:
statistic = Point.select(Point.raw('max(1day_action_complete_order) as order_num, '
'max(1day_action_complete_order_amount) as order_amount'))\
.where('created_at', '>=', today)\
.where('campaign_id', data['campaign_id'])\
.where('status', ADSTATUS_NORMAL)\
.first()
order_amount = max(statistic.order_amount,
data['1day_action_complete_order_amount'], 0)
order_num = max(statistic.order_num,
data['1day_action_complete_order'], 0)
data['sy_cost'] = int(data['sy_cost'])
ori = 0 if order_amount == 0 else data['sy_cost'] / order_amount
if wxcampaign.is_beishang():
'''
晨1点-2点，定向为仅包含北上的计划，roi低于1.5 且订单个数<2，设置7点，
roi<1.5 && 订单个数>=2 设置到10点
'''
if order_num < 2 and ori < 1.5:
actions.append(build_command(data, 'timeset_end', 7))
elif order_num > 4 and ori < 1.5:
actions.append(build_command(data, 'timeset_end', 10))
else:
'''
1-2点，定向非北上城市，（当天花费 > 200 且 历史转换为0） 暂停，
（花费> 200 历史总花费／历史总转化 >= 1.5） 设置到9点
（花费> 200 历史总花费／历史总转化 < 1.5） 设置到7点
'''
records = Point.select(Point.raw('max(sy_cost) as day_cost, '
'max(1day_action_complete_order) as order_num, '
'max(1day_action_complete_order_amount) as order_amount'))\
.where('created_at', '>=', today.to_datetime_string()) \
.where('campaign_id', data['campaign_id']) \
.group_by(Point.raw('DATE_FORMAT(update_time, "%%Y%%m%%d")')) \
.get()
total_cost = 0
total_order = 0
total_order_amount = 0
for record in records:
total_cost += int(record.day_cost)
total_order += record.order_num
total_order_amount += record.order_amount
transfer_rate = 0 if total_order == 0 else total_cost / total_order
if data['sy_cost'] > 200:
if total_order == 0:
actions.append(build_command(data, 'suspend', None))
elif transfer_rate >= 1.5:
actions.append(build_command(data, 'timeset_end', 9))
else:
actions.append(build_command(data, 'timeset_end', 7))
return actions
def current():
return (Service
.objects(date__lte=pendulum.now(TZ), master=True)
.order_by('-date')
.first()
)
def test_unexpired_cache(self):
state = Cached(cached_result_expiration=pendulum.now("utc") + timedelta(days=1))
assert duration_only(state, None, None) is True
def test_expired_cache_stateful(validator):
state = Cached(cached_result_expiration=pendulum.now("utc") - timedelta(days=1))
assert validator()(state, None, None) is False
def top(repo_ctx, sort, rows):
repos = []
columns = OrderedDict([
("html_url", "URL"),
("stargazers_count", "Stars"),
("forks_count", "Forks"),
("open_issues_count", "Open Issues"),
("updated_at", "Last update"),
("pushed_at", "Pushed At"),
], )
headers = list(columns.values())
spinner = Halo(text="Fetch information about forks", spinner="dots")
spinner.start()
if sort in {"branches", "commits", "watchers"}:
columns[sort] = sort.capitalize()
headers.append(columns[sort])
Repo = namedtuple("Repo", list(columns.keys()))
else:
Repo = namedtuple("Repo", list(columns.keys()))
for fork in repo_ctx.forks:
def_prop = [
fork.html_url,
fork.stargazers_count,
fork.forks_count,
fork.open_issues_count,
fork.updated_at,
fork.pushed_at,
]
# github api may return nonexistent profile
if sort == "branches":
try:
def_prop.append(len(list(fork.branches())))
repos.append(Repo(*def_prop))
except github3.exceptions.NotFoundError:
click.echo("\nRepository {0} not found".format(fork.html_url))
elif sort == "watchers":
try:
repo = repo_ctx.gh.repository(fork.owner.login, fork.name)
def_prop.append(repo.subscribers_count)
repos.append(Repo(*def_prop))
except github3.exceptions.NotFoundError:
click.echo("\nRepository {0} not found".format(fork.html_url))
elif sort == "commits":
try:
def_prop.append(
sum((c.contributions_count for c in fork.contributors())))
repos.append(Repo(*def_prop))
except github3.exceptions.NotFoundError:
click.echo("\nRepository {0} not found".format(fork.html_url))
else:
repos.append(Repo(*def_prop))
sorted_forks = sorted(repos, key=attrgetter(sort), reverse=True)
humanize_dates_forks = []
for fork in sorted_forks[:rows]:
days_passed_updated_at = (pendulum.now() -
pendulum.parse(fork.updated_at)).days
days_passed_pushed_at = (pendulum.now() -
pendulum.parse(fork.pushed_at)).days
human_updated_at = pendulum.now().subtract(
days=days_passed_updated_at).diff_for_humans()
human_pushed_at = pendulum.now().subtract(
days=days_passed_pushed_at).diff_for_humans()
humanize_dates_forks.append(
fork._replace(updated_at=human_updated_at,
pushed_at=human_pushed_at))
spinner.stop()
click.echo(tabulate(humanize_dates_forks, headers=headers,
tablefmt="grid"))
def initialize_run( # type: ignore
self,
state: Optional[State],
task_states: Dict[Task, State],
context: Dict[str, Any],
task_contexts: Dict[Task, Dict[str, Any]],
parameters: Dict[str, Any],
) -> FlowRunnerInitializeResult:
"""
Initializes the Task run by initializing state and context appropriately.
If the provided state is a Submitted state, the state it wraps is extracted.
Args:
- state (Optional[State]): the initial state of the run
- task_states (Dict[Task, State]): a dictionary of any initial task states
- context (Dict[str, Any], optional): prefect.Context to use for execution
to use for each Task run
- task_contexts (Dict[Task, Dict[str, Any]], optional): contexts that will be
provided to each task
- parameters(dict): the parameter values for the run
Returns:
- NamedTuple: a tuple of initialized objects:
`(state, task_states, context, task_contexts)`
"""
# overwrite context parameters one-by-one
context_params = context.setdefault("parameters", {})
for p in self.flow.parameters():
if not p.required:
context_params.setdefault(p.name, p.default)
for param, value in (parameters or {}).items():
context_params[param] = value
context.update(flow_name=self.flow.name)
context.setdefault("scheduled_start_time", pendulum.now("utc"))
# Determine the current time, allowing our formatted dates in the context
# to rely on a manually set value
now = context.get("date")
if isinstance(now, str):
# Attempt to parse into a `DateTime` object since it will often be passed
# as a serialized string from the UI we'll override the context on a
# successful parse so the type is consistent for users
try:
now = pendulum.parse(now)
context["date"] = now
except Exception:
pass
if not isinstance(now, pendulum.DateTime):
if now is not None:
self.logger.warning(
"`date` was set in the context manually but could not be parsed "
"into a pendulum `DateTime` object. Additional context variables "
"that rely on the current date i.e `today` and `tomorrow` will be "
"based on the current time instead of the `date` context variable."
)
now = pendulum.now("utc")
# add various formatted dates to context
dates = {
"date": now,
"today": now.strftime("%Y-%m-%d"),
"yesterday": now.add(days=-1).strftime("%Y-%m-%d"),
"tomorrow": now.add(days=1).strftime("%Y-%m-%d"),
"today_nodash": now.strftime("%Y%m%d"),
"yesterday_nodash": now.add(days=-1).strftime("%Y%m%d"),
"tomorrow_nodash": now.add(days=1).strftime("%Y%m%d"),
}
for key, val in dates.items():
context.setdefault(key, val)
for task in self.flow.tasks:
task_contexts.setdefault(task, {}).update(
task_name=task.name, task_slug=self.flow.slugs[task]
)
state, context = super().initialize_run(state=state, context=context)
return FlowRunnerInitializeResult(
state=state,
task_states=task_states,
context=context,
task_contexts=task_contexts,
)
def device_fitness_report(self, queue, worker_group, device):
results = self.get_worker_jobs(queue, worker_group, device)
task_successes = 0
task_failures = 0
task_runnings = 0
task_exceptions = 0
task_last_started_timestamp = None
task_ids = []
for task in results["recentTasks"]:
task_id = task["taskId"]
task_ids.append(task_id)
try:
results = ThreadPool(TASK_THREAD_COUNT).imap_unordered(
self.get_task_status, task_ids)
except Exception as e:
print(e)
for task_id, result, error in results:
if error is None:
task_state = None
# filter out jobs that are gone
if "code" in result:
if result["code"] == "ResourceNotFound":
continue
try:
task_state = result["status"]["state"]
# pprint.pprint(result)
except KeyError:
print("strange result: ")
pprint.pprint(result)
print(result["code"] == "ResourceNotFound")
continue
# record the last started time
if "runs" in result["status"]:
for run in result["status"]["runs"]:
if "started" in run:
temp_date = pendulum.parse(run["started"])
if task_last_started_timestamp is None:
task_last_started_timestamp = temp_date
# if temp_date more recent (larger), use it as the oldest date
if temp_date > task_last_started_timestamp:
task_last_started_timestamp = temp_date
# TODO: gather exception stats
if task_state == "running":
task_runnings += 1
elif task_state == "exception":
task_exceptions += 1
elif task_state == "failed":
task_failures += 1
elif task_state == "completed":
retries_left = result["status"]["retriesLeft"]
if retries_left != 5:
runs = result["status"]["runs"]
for run in runs:
if run["workerId"] == device:
run_state = run["state"]
if run_state == "exception":
task_exceptions += 1
elif run_state == "running":
task_runnings += 1
elif run_state == "completed":
task_successes += 1
elif run_state == "failed":
task_failures += 1
else:
raise Exception("Shouldn't be here!")
else:
task_successes += 1
if self.verbosity:
print("%s.%s: %s: %s" %
(queue, device, task_id, task_state))
else:
# TODO: should return exception? only getting partial truth...
pass
# print("error fetching %r: %s" % (task_id, error))
total = task_failures + task_successes
results_obj = {}
success_ratio_calculated = False
if total > 0:
success_ratio = task_successes / total
# print("sr: %s/%s=%s" % (task_successes, total, success_ratio))
results_obj["sr"] = success_ratio
success_ratio_calculated = True
else:
results_obj["sr"] = float(0)
results_obj["suc"] = task_successes
results_obj["cmp"] = total
results_obj["exc"] = task_exceptions
results_obj["rng"] = task_runnings
results_obj["ls"] = task_last_started_timestamp
# note if no jobs in queue
if queue in self.queue_counts:
if self.queue_counts[queue] == 0:
# if "notes" not in results_obj:
# results_obj["notes"] = []
results_obj.setdefault("notes", []).append("No jobs in queue.")
jobs_present = False
else:
jobs_present = True
else:
logger.warn("Strange, no queue count data for %s" % queue)
# ping alerts
if self.args.ping:
if self.args.ping_host:
cmd = [
"ssh",
self.args.ping_host,
"ping -c 1 -i 0.3 -w 1 %s.%s" %
(device, self.args.ping_domain),
]
res = subprocess.run(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
if res.returncode != 0:
if "alerts" not in results_obj:
results_obj["alerts"] = []
results_obj["alerts"].append("Not pingable!")
# TODO: write to notes that it is pingable?
else:
logger.warn("sorry, not supported yet")
# alert if success ratio is low
if success_ratio_calculated:
if success_ratio < self.alert_percent:
# if "alerts" not in results_obj:
# results_obj["alerts"] = []
results_obj.setdefault("alerts", []).append(
"Low health (less than %s)!" % self.alert_percent)
# alert if worker hasn't worked in 1 hour
dt = pendulum.now()
# TODO: take minutes as an arg
comparison_dt = dt.subtract(minutes=60)
if jobs_present and task_last_started_timestamp < comparison_dt:
results_obj.setdefault("alerts",
[]).append("No work started in last hour!")
else:
results_obj.setdefault("state", []).append("working")
# alert if lots of exceptions
if task_exceptions >= 3:
results_obj.setdefault("alerts",
[]).append("High exceptions (3+)!")
# alert if no work done
if total == 0 and task_exceptions == 0 and task_runnings == 0:
# if "alerts" not in results_obj:
# results_obj["alerts"] = []
results_obj.setdefault("alerts", []).append("No work done!")
# quarantine
if device in self.quarantine_data[queue]:
# if "alerts" not in results_obj:
# results_obj["alerts"] = []
results_obj.setdefault("alerts", []).append("Quarantined.")
return device, results_obj, None
async def test_user_create_ok(backend, apiv1_backend_sock_factory, alice,
mallory, profile):
now = pendulum.now()
user_certificate = UserCertificateContent(
author=alice.device_id,
timestamp=now,
user_id=mallory.user_id,
human_handle=None,
public_key=mallory.public_key,
profile=profile,
).dump_and_sign(alice.signing_key)
device_certificate = DeviceCertificateContent(
author=alice.device_id,
timestamp=now,
device_id=mallory.device_id,
device_label=None,
verify_key=mallory.verify_key,
).dump_and_sign(alice.signing_key)
with backend.event_bus.listen() as spy:
async with apiv1_backend_sock_factory(backend, alice) as sock:
rep = await user_create(sock,
user_certificate=user_certificate,
device_certificate=device_certificate)
assert rep == {"status": "ok"}
# No guarantees this event occurs before the command's return
await spy.wait_with_timeout(
BackendEvent.USER_CREATED,
{
"organization_id": alice.organization_id,
"user_id": mallory.user_id,
"first_device_id": mallory.device_id,
"user_certificate": user_certificate,
"first_device_certificate": device_certificate,
},
)
# Make sure mallory can connect now
async with apiv1_backend_sock_factory(backend, mallory) as sock:
rep = await user_get(sock, user_id=mallory.user_id)
assert rep["status"] == "ok"
# Check the resulting data in the backend
backend_user, backend_device = await backend.user.get_user_with_device(
mallory.organization_id, mallory.device_id)
assert backend_user == User(
user_id=mallory.user_id,
human_handle=None,
profile=profile,
user_certificate=user_certificate,
redacted_user_certificate=user_certificate,
user_certifier=alice.device_id,
created_on=now,
)
assert backend_device == Device(
device_id=mallory.device_id,
device_label=None,
device_certificate=device_certificate,
redacted_device_certificate=device_certificate,
device_certifier=alice.device_id,
created_on=now,
)
def month_range(month_offset):
month_range_list = [
pendulum.now(tz).subtract(months=month_offset).start_of('month'),
pendulum.now(tz).subtract(months=month_offset).end_of('month')
]
return month_range_list
def run(
self,
state: State = None,
upstream_states: Dict[Edge, State] = None,
context: Dict[str, Any] = None,
is_mapped_parent: bool = False,
) -> State:
"""
The main endpoint for TaskRunners. Calling this method will conditionally execute
`self.task.run` with any provided inputs, assuming the upstream dependencies are in a
state which allow this Task to run. Additionally, this method will wait and perform
Task retries which are scheduled for <= 10 minutes in the future.
Args:
- state (State, optional): initial `State` to begin task run from;
defaults to `Pending()`
- upstream_states (Dict[Edge, State]): a dictionary
representing the states of any tasks upstream of this one. The keys of the
dictionary should correspond to the edges leading to the task.
- context (dict, optional): prefect Context to use for execution
- is_mapped_parent (bool): a boolean indicating whether this task run is the run of
a parent mapped task
Returns:
- `State` object representing the final post-run state of the Task
"""
context = context or {}
with prefect.context(context):
end_state = super().run(
state=state,
upstream_states=upstream_states,
context=context,
is_mapped_parent=is_mapped_parent,
)
while (end_state.is_retrying() or end_state.is_queued()) and (
end_state.start_time <= pendulum.now("utc").add(minutes=10) # type: ignore
):
assert isinstance(end_state, (Retrying, Queued))
naptime = max(
(end_state.start_time - pendulum.now("utc")).total_seconds(), 0
)
for _ in range(int(naptime) // 30):
# send heartbeat every 30 seconds to let API know task run is still alive
self.client.update_task_run_heartbeat(
task_run_id=prefect.context.get("task_run_id")
)
naptime -= 30
time.sleep(30)
if naptime > 0:
time.sleep(naptime) # ensures we don't start too early
self.client.update_task_run_heartbeat(
task_run_id=prefect.context.get("task_run_id")
)
end_state = super().run(
state=end_state,
upstream_states=upstream_states,
context=context,
is_mapped_parent=is_mapped_parent,
)
return end_state
async def test_versions_existing_file_remove_minimal_synced(
alice_workspace, alice):
version_lister = alice_workspace.get_version_lister()
versions, version_list_is_complete = await version_lister.list(
FsPath("/files/renamed"))
assert version_list_is_complete is True
assert len(versions) == 5
# Moved /files/content to /files/renamed on day 5, moved it again later
assert versions[0][1:] == (
3,
_day(6),
_day(7),
alice.device_id,
_day(5),
False,
5,
FsPath("/files/content"),
FsPath("/files/renamed_again"),
)
# Created a new file with the same name on day 8
# This entry is deleted as we only get the one obtained by writing on it in our list
# This is the entry where we wrote on it
# Moved it again on day 9 as we renamed /files to /moved
assert versions[1][1:] == (
2,
_day(8),
_day(9),
alice.device_id,
_day(8),
False,
6,
None,
FsPath("/moved/renamed"),
)
# And moved back /moved to /files on day 11, /files/renamed is deleted on day 12
assert versions[2][1:] == (
2,
_day(11),
_day(12),
alice.device_id,
_day(8),
False,
6,
FsPath("/moved/renamed"),
None,
)
# Created a file, again, but didn't write
assert versions[3][1:] == (
1,
_day(13),
_day(14),
alice.device_id,
_day(13),
False,
0,
None,
None,
)
# Used "touch" method again, but on a created file. Wrote on it. Didn't delete since then
assert versions[4][1:3] == (2, _day(14))
assert now().add(hours=-1) < versions[4][3] < now()
assert versions[4][4:] == (alice.device_id, _day(14), False, 5, None, None)
def get_file_name(self):
now = pendulum.now()
return 'participation_metrics_{}_{}_{}_generated_{}_{}.xlsx'.format(
self.tenant, self.start_date.to_date_string(),
self.end_date.to_date_string(), now.to_date_string(),
now.int_timestamp)
async def test_versions_existing_directory(alice_workspace, alice,
skip_minimal_sync):
version_lister = alice_workspace.get_version_lister()
versions, version_list_is_complete = await version_lister.list(
FsPath("/files"), skip_minimal_sync=skip_minimal_sync)
assert version_list_is_complete is True
assert len(versions) == 8
assert versions[0][1:] == (
1,
_day(4),
_day(4),
alice.device_id,
_day(4),
True,
None,
None,
None,
)
assert versions[1][1:] == (
2,
_day(4),
_day(6),
alice.device_id,
_day(4),
True,
None,
None,
None,
)
assert versions[2][1:] == (
3,
_day(6),
_day(7),
alice.device_id,
_day(6),
True,
None,
None,
None,
)
assert versions[3][1:] == (
4,
_day(7),
_day(8),
alice.device_id,
_day(7),
True,
None,
None,
None,
)
assert versions[4][1:] == (
5,
_day(8),
_day(9),
alice.device_id,
_day(8),
True,
None,
None,
FsPath("/moved"),
)
assert versions[5][1:] == (
6,
_day(11),
_day(12),
alice.device_id,
_day(10),
True,
None,
FsPath("/moved"),
None,
)
assert versions[6][1:] == (
7,
_day(12),
_day(13),
alice.device_id,
_day(12),
True,
None,
None,
None,
)
assert versions[7][1:3] == (8, _day(13))
assert now().add(hours=-1) < versions[7][3] < now()
assert versions[7][4:] == (alice.device_id, _day(13), True, None, None,
None)
context.update(running_with_backend=True)
end_state = super().run(
state=state,
task_states=task_states,
return_tasks=return_tasks,
parameters=parameters,
task_runner_state_handlers=task_runner_state_handlers,
executor=executor,
context=context,
task_contexts=task_contexts,
)
# If start time is more than 10 minutes in the future,
# we fail the run so Lazarus can pick it up and reschedule it.
while end_state.is_queued() and (
end_state.start_time <= pendulum.now("utc").add(minutes=10) # type: ignore
):
assert isinstance(end_state, Queued)
time_remaining = max(
(end_state.start_time - pendulum.now("utc")).total_seconds(), 0
)
self.logger.debug(
(
f"Flow run is in a Queued state. Sleeping for at most {time_remaining:.2f} "
f"seconds and attempting to run again."
)
)
# Sleep until not in a queued state, then attempt to re-run
while time_remaining > 0:
delay = min(
prefect.config.cloud.check_cancellation_interval, time_remaining
def __init__(self, api):
super().__init__(api=api)
self._state = {}
self._end_date = pendulum.now()
def _set_state(self, account_id: str):
start_date = self._state[account_id] if self._state.get(
account_id) else self._api._start_date
self._state[account_id] = max(
start_date,
pendulum.now().subtract(days=self.buffer_days))
def main():
parser = argparse.ArgumentParser(description='tiktok pipeline')
parser.add_argument(
"-s",
"--start_date",
help="start date",
default=pendulum.now().subtract(days=5).to_date_string())
parser.add_argument(
"-e",
"--end_date",
help="end date",
default=pendulum.now().subtract(days=1).to_date_string())
args = parser.parse_args()
started = datetime.now()
sf_account = os.environ['SNOWFLAKE_ACCOUNT']
sf_username = os.environ['SNOWFLAKE_USERNAME']
sf_role = os.environ['SNOWFLAKE_ROLE']
sf_password = os.environ['SNOWFLAKE_PASSWORD']
sf_database = os.environ['SNOWFLAKE_DATABASE']
sf_schema = os.environ['SNOWFLAKE_SCHEMA']
sf_warehouse = os.environ['SNOWFLAKE_WAREHOUSE']
conf = {
'start_date': args.start_date,
'end_date': args.end_date,
'auth_token': os.environ['TIKTOK_API_KEY'],
'advertiser_id': os.environ['TIKTOK_ADVERTISER_ID']
}
tap = 'tap-tiktok'
target = 'atidiv-target-snowflake'
tap_config = 'tiktok_config.json'
target_config = 'snowflake_config.json'
with open(tap_config, 'w') as out:
json.dump(conf, out)
snowflake_conf = {
"snowflake_account": sf_account,
"snowflake_username": sf_username,
"snowflake_role": sf_role,
"snowflake_password": sf_password,
"snowflake_database": sf_database,
"snowflake_schema": sf_schema,
"snowflake_warehouse": sf_warehouse,
"add_metadata_columns": False
}
with open(target_config, 'w') as out:
json.dump(snowflake_conf, out)
r = os.system(
f" ~/.virtualenvs/{tap}/bin/{tap} --config {tap_config} | ~/.virtualenvs/{target}/bin/{target} -c {target_config}"
)
if r != 0:
raise Exception('error occurred!')
ended = datetime.now()
pretty(f'execution time : {ended - started}')
def now():
return hi.say() + "it is " + pendulum.now('Europe/Paris').format(
'dddd Do [of] MMMM YYYY HH:mm:ss A')
def test_hmins2epoch(hours, mins):
now = pendulum.now(tz='Europe/Berlin').replace(hour=hours, minute=mins)
assert hmins2epoch(hours, mins) == now.format('X', formatter='alternative')
def test_datetime_parameter_returns_datetimes_if_passed_string(self):
now = pendulum.now()
state = self.dt_flow.run(dt=str(now))
assert state.result[self.dt].result == now
assert state.result[self.x].result == now
class Achievement(EmbeddedDocument):
title = StringField()
description = StringField()
key = StringField()
date = DateTimeField(default=pendulum.now)
validUntil = DateTimeField(default=lambda: pendulum.now().add(days=7))
async def on_command_error(self, ctx: context.Context, error) -> None:
error = getattr(error, 'original', error)
if isinstance(error, commands.CommandNotFound):
return
elif isinstance(error, commands.CommandOnCooldown):
cooldown_types = {
commands.BucketType.default: f'for the whole bot.',
commands.BucketType.user: f'for you.',
commands.BucketType.member: f'for you.',
commands.BucketType.role: f'for your role.',
commands.BucketType.guild: f'for this server.',
commands.BucketType.channel: f'for this channel.',
commands.BucketType.category: f'for this channel category.'
}
await ctx.send(
f'The command `{ctx.command}` is on cooldown {cooldown_types.get(error.cooldown.type, "for you.")} You can retry in '
f'`{self.bot.utils.format_seconds(seconds=error.retry_after, friendly=True)}`'
)
return
elif isinstance(error, commands.MaxConcurrencyReached):
cooldowns = {
commands.BucketType.default: f'.',
commands.BucketType.user: f' per user.',
commands.BucketType.member: f' per member.',
commands.BucketType.role: f' per role.',
commands.BucketType.guild: f' per server.',
commands.BucketType.channel: f' per channel.',
commands.BucketType.category: f' per channel category.',
}
await ctx.send(
f'The command `{ctx.command}` is being ran at its maximum of {error.number} time(s){cooldowns.get(error.per, ".")} Retry a bit later.'
)
return
elif isinstance(error, commands.BotMissingPermissions):
permissions = '\n'.join(
[f'> {permission}' for permission in error.missing_perms])
message = f'I am missing the following permissions required to run the command `{ctx.command}`.\n{permissions}'
try:
await ctx.send(message)
except discord.Forbidden:
try:
await ctx.author.send(message)
except discord.Forbidden:
return
return
elif isinstance(error, commands.MissingPermissions):
permissions = '\n'.join(
[f'> {permission}' for permission in error.missing_perms])
await ctx.send(
f'You are missing the following permissions required to run the command `{ctx.command}`.\n{permissions}'
)
return
elif isinstance(error, commands.MissingRequiredArgument):
await ctx.send(
f'You missed the `{error.param.name}` argument. Use `{self.bot.config.prefix}help {ctx.command}` for more information on what arguments to use.'
)
return
elif isinstance(error, commands.BadUnionArgument):
await ctx.send(
f'I was unable to convert the `{error.param}` argument. Use `{self.bot.config.prefix}help {ctx.command}` for more information on what arguments '
f'to use.')
return
elif isinstance(error, commands.MissingRole):
await ctx.send(
f'The role `{error.missing_role}` is required to run this command.'
)
return
elif isinstance(error, commands.BotMissingRole):
await ctx.send(
f'The bot requires the role `{error.missing_role}` to run this command.'
)
return
elif isinstance(error, commands.MissingAnyRole):
await ctx.send(
f'The roles {", ".join([f"`{role}`" for role in error.missing_roles])} are required to run this command.'
)
return
elif isinstance(error, commands.BotMissingAnyRole):
await ctx.send(
f'The bot requires the roles {", ".join([f"`{role}`" for role in error.missing_roles])} to run this command.'
)
return
elif isinstance(error, commands.BadArgument):
if isinstance(error, commands.MessageNotFound):
await ctx.send(
f'A message for the argument `{error.argument}` was not found.'
)
elif isinstance(error, commands.MemberNotFound):
await ctx.send(
f'A member for the argument `{error.argument}` was not found.'
)
elif isinstance(error, commands.UserNotFound):
await ctx.send(
f'A user for the argument `{error.argument}` was not found.'
)
elif isinstance(error, commands.ChannelNotFound):
await ctx.send(
f'A channel for the argument `{error.argument}` was not found.'
)
elif isinstance(error, commands.RoleNotFound):
await ctx.send(
f'A role for the argument `{error.argument}` was not found.'
)
elif isinstance(error, commands.EmojiNotFound):
await ctx.send(
f'An emoji for the argument `{error.argument}` was not found.'
)
elif isinstance(error, commands.ChannelNotReadable):
await ctx.send(
f'I do not have permission to read the channel `{error.argument}`'
)
elif isinstance(error, commands.PartialEmojiConversionFailure):
await ctx.send(
f'The argument `{error.argument}` did not match the partial emoji format.'
)
elif isinstance(error, commands.BadInviteArgument):
await ctx.send(
f'The invite that matched that argument was not valid or is expired.'
)
elif isinstance(error, commands.BadBoolArgument):
await ctx.send(
f'The argument `{error.argument}` was not a valid True/False value.'
)
elif isinstance(error, commands.BadColourArgument):
await ctx.send(
f'The argument `{error.argument}` was not a valid colour.')
elif isinstance(error, BadLiteralArgument):
await ctx.send(
f'The argument `{error.param.name}` must be one of {", ".join([f"`{arg}`" for arg in error.valid_arguments])}.'
)
return
else:
error_messages = {
exceptions.ArgumentError:
f'{error}',
exceptions.ImageError:
f'{error}',
exceptions.VoiceError:
f'{error}',
NodeNotFound:
f'There are no lavalink nodes available right now.',
commands.TooManyArguments:
f'You used too many arguments. Use `{self.bot.config.prefix}help {ctx.command}` for more information on what argument to use.',
commands.UnexpectedQuoteError:
f'There was an unexpected quote character in the arguments you passed.',
commands.InvalidEndOfQuotedStringError:
f'There was an unexpected space after a quote character in the arguments you passed.',
commands.ExpectedClosingQuoteError:
f'There is a missing quote character in the argument you passed.',
commands.BadArgument:
f'I was unable to convert an argument that you used. Use `{self.bot.config.prefix}help {ctx.command}` for more information on what '
f'arguments to use.',
commands.CheckFailure:
f'{error}',
commands.PrivateMessageOnly:
f'The command `{ctx.command}` can only be used in private messages',
commands.NoPrivateMessage:
f'The command `{ctx.command}` can not be used in private messages.',
commands.NotOwner:
f'The command `{ctx.command}` is owner only.',
commands.NSFWChannelRequired:
f'The command `{ctx.command}` can only be run in a NSFW channel.',
commands.DisabledCommand:
f'The command `{ctx.command}` has been disabled.',
}
error_message = error_messages.get(type(error), None)
if error_message is not None:
await ctx.send(error_message)
return
await ctx.send(
f'Something went wrong while executing that command. Please use `{self.bot.config.prefix}support` for more help or information.'
)
formatter = prettify_exceptions.DefaultFormatter()
formatter.theme['_ansi_enabled'] = True
print(
f'\n{"".join(formatter.format_exception(type(error), error, error.__traceback__)).strip()}\n'
)
time = self.bot.utils.format_datetime(datetime=pendulum.now(tz='UTC'))
guild = f'`Guild:` {ctx.guild} `{ctx.guild.id}`\n' if ctx.guild else ''
info = f'Error in command `{ctx.command}`\n\n{guild}`Channel:` {ctx.channel} `{ctx.channel.id}`\n`Author:` {ctx.author} `{ctx.author.id}`\n`Time:` {time}'
embed = discord.Embed(colour=ctx.colour,
description=f'{ctx.message.content}')
embed.add_field(name='Info:', value=info)
await self.bot.errors_webhook.send(
embed=embed,
username=f'{ctx.author}',
avatar_url=str(
ctx.author.avatar_url_as(format='gif' if ctx.author.
is_avatar_animated() else 'png')))
formatter.theme['_ansi_enabled'] = False
traceback = "".join(
formatter.format_exception(type(error), error,
error.__traceback__)).strip()
if len(traceback) > 2000:
async with self.bot.session.post('https://mystb.in/documents',
data=traceback) as response:
response = await response.json()
traceback = f'https://mystb.in/{response["key"]}.python'
else:
traceback = f'```\n{traceback}\n```'
await self.bot.errors_webhook.send(
content=f'{traceback}',
username=f'{ctx.author}',
avatar_url=str(
ctx.author.avatar_url_as(format='gif' if ctx.author.
is_avatar_animated() else 'png')))
