def wait_for_export(self, stream_type, export_id):
# Poll the export status until it enters a finalized state or
# exceeds the job timeout time.
with metrics.job_timer('Export {} for {}'.format(
export_id, stream_type)):
timeout_time = pendulum.utcnow().add(seconds=self.job_timeout)
while pendulum.utcnow() < timeout_time:
status = self.poll_export(stream_type, export_id)
singer.log_info("export %s status is %s", export_id, status)
if status == "Created":
# If the status is created, the export has been made but
# not started, so enqueue the export.
self.enqueue_export(stream_type, export_id)
elif status in ["Cancelled", "Failed"]:
# Cancelled and failed exports fail the current sync.
raise ExportFailed(status)
elif status == "Completed":
return True
time.sleep(self.poll_interval)
raise ExportFailed("Export timed out after {} minutes".format(
self.job_timeout / 60))
def handle(self, *args, **options):
user = ShareUser.objects.get(username=settings.APPLICATION_USERNAME)
if options['ids']:
self.harvest_ids(user, options)
return
task_kwargs = {'force': options.get('force', False)}
if options['days_back'] is not None and (options['start']
or options['end']):
self.stdout.write(
'Please choose days-back OR a start date with end date, not both'
)
return
if options['days_back'] is not None:
task_kwargs['end'] = datetime.datetime.utcnow(
) + datetime.timedelta(days=-(options['days_back'] - 1))
task_kwargs['start'] = datetime.datetime.utcnow(
) + datetime.timedelta(days=-options['days_back'])
else:
task_kwargs['start'] = pendulum.parse(
options['start']) if options.get('start') else pendulum.utcnow(
) - datetime.timedelta(days=int(options['days_back'] or 1))
task_kwargs['end'] = pendulum.parse(
options['end']) if options.get('end') else pendulum.utcnow()
task_kwargs['end'] = task_kwargs['end'].isoformat()
task_kwargs['start'] = task_kwargs['start'].isoformat()
if options['limit'] is not None:
task_kwargs['limit'] = options['limit']
if options['set_spec']:
task_kwargs['set_spec'] = options['set_spec']
if not options['harvester'] and options['all']:
options['harvester'] = [
x.label for x in apps.get_app_configs()
if isinstance(x, ProviderAppConfig) and not x.disabled
]
for harvester in options['harvester']:
apps.get_app_config(
harvester) # Die if the AppConfig can not be loaded
task_args = (
user.id,
harvester,
)
if options['async']:
HarvesterTask().apply_async(task_args, task_kwargs)
self.stdout.write(
'Started job for harvester {}'.format(harvester))
else:
self.stdout.write('Running harvester for {}'.format(harvester))
HarvesterTask().apply(task_args, task_kwargs, throw=True)
def poll_job_until_done(job_id, client, api):
timeout_time = pendulum.utcnow().add(seconds=DEFAULT_JOB_TIMEOUT)
while pendulum.utcnow() < timeout_time:
if api.job_ready(client, job_id):
return api.get_file_ids(client, job_id)
time.sleep(DEFAULT_POLL_INTERVAL)
raise apis.ExportTimedOut(DEFAULT_JOB_TIMEOUT // 60, "minutes")
def test_swbiodiversity_harvester():
httpretty.enable()
httpretty.allow_net_connect = False
config = SourceConfig.objects.get(label=('org.swbiodiversity'))
url = config.harvester_kwargs['list_url']
harvester = config.get_harvester()
httpretty.register_uri(httpretty.GET,
url,
body=main_page,
content_type='text/html',
match_querystring=True)
collection = furl(url)
collection.args['collid'] = 223
httpretty.register_uri(httpretty.GET,
url + ';collid=(\d+)',
body=collection_page,
content_type='text/html',
match_querystring=True)
start = pendulum.utcnow() - timedelta(days=3)
end = pendulum.utcnow()
results = harvester.fetch_date_range(start, end)
for result in results:
assert result.identifier == collection.url
assert "".join(result.datum.split()) == "".join('''
<div id="innertext">
<h1>SEINet - Arizona Chapter Collections </h1>
<div>
Select a collection to see full details.
</div>
<table style="margin:10px;">
<tr>
<td>
<h3>
<a href="collprofiles.php?collid=223">
A. Michael Powell Herbarium
</a>
</h3>
<div style="margin:10px;">
<div>Sample description</div>
<div style="margin-top:5px;">
<b>Contact:</b>
Test Author ([email protected])
</div>
</div>
</td>
</tr>
</table>
</div>
'''
"".split())
httpretty.disable()
async def on_ready():
# Change the status upon connection to the default status
await bot.change_presence(game=discord.Game(name=utils.default_status, type=0))
if not hasattr(bot, 'uptime'):
bot.uptime = pendulum.utcnow()
await utils.db_check()
def should_publish_steem(self, node, price):
# check whether we need to publish again:
# - if published more than 12 hours ago, publish again
# - if published price different by more than 3%, publish again
if 'last_price' not in node.opts: # make sure we have already published once
log.debug(
'Steem should publish for the first time since launch of bts_tools'
)
return True
last_published_interval = pendulum.interval(hours=12)
variance_trigger = 0.03
if pendulum.utcnow(
) - node.opts['last_published'] > last_published_interval:
log.debug(
'Steem should publish as it has not been published for {}'.
format(last_published_interval))
return True
if abs(price - node.opts['last_price']
) / node.opts['last_price'] >= variance_trigger:
log.debug(
'Steem should publish as price has moved more than {}%'.format(
100 * variance_trigger))
return True
log.debug('No need for Steem to publish')
return False
def __init__(self, *, cfg, visible_feeds=None):
self.cfg = copy.deepcopy(cfg)
from .feeds import DEFAULT_VISIBLE_FEEDS
self.visible_feeds = list(visible_feeds or DEFAULT_VISIBLE_FEEDS)
# FIXME: deprecate self.feed_period
try:
self.feed_period = int(cfg['publish_strategy']['time_interval'] / cfg['check_time_interval'])
except KeyError:
self.feed_period = None
self.check_time_interval = pendulum.interval(seconds=cfg.get('check_time_interval', 600))
try:
self.publish_time_interval = pendulum.interval(seconds=cfg['publish_strategy']['time_interval'])
except KeyError:
self.publish_time_interval = None
self.feed_slot = cfg.get('publish_strategy', {}).get('time_slot', None)
if self.feed_slot is not None:
self.feed_slot = int(self.feed_slot)
self.nfeed_checked = 0
self.last_published = pendulum.utcnow().subtract(days=1)
log.debug('successfully initialized {}'.format(self))
def get(self):
data_list = []
cond = and_(self.TI.dag_id == self.DM.dag_id,
self.TI.state == "running", self.DM.owners == self.owners)
tasks = self.session.query(self.TI.dag_id, self.TI.task_id,
self.TI.execution_date, self.TI.start_date,
self.TI.duration).filter(cond).all()
for task in tasks:
last_runs = self.session.query(self.TI.duration).filter(
self.TI.task_id == task.task_id, self.TI.state == "success",
self.TI.duration > 5).order_by(
self.TI.execution_date.desc()).limit(10).all()
average = statistics.mean(duration[0] for duration in last_runs)
runtime = (pendulum.utcnow() - task.start_date).seconds
execution_date = self.datetime_iso(task.execution_date)
if runtime > (average + 1800):
data_list.append(
OrderedDict([('dag_id', task.dag_id),
('task_id', task.task_id),
('execution_date', execution_date),
('runtime', self.seconds_to(runtime)),
('average', self.seconds_to(average))]))
else:
continue
return jsonify(data_list)
def __init__(self, *, cfg, visible_feeds=None):
self.cfg = copy.deepcopy(cfg)
from .feeds import DEFAULT_VISIBLE_FEEDS
self.visible_feeds = list(visible_feeds or DEFAULT_VISIBLE_FEEDS)
# FIXME: deprecate self.feed_period
try:
self.feed_period = int(cfg['publish_strategy']['time_interval'] /
cfg['check_time_interval'])
except KeyError:
self.feed_period = None
self.check_time_interval = pendulum.interval(
seconds=cfg.get('check_time_interval', 600))
try:
self.publish_time_interval = pendulum.interval(
seconds=cfg['publish_strategy']['time_interval'])
except KeyError:
self.publish_time_interval = None
self.feed_slot = cfg.get('publish_strategy', {}).get('time_slot', None)
if self.feed_slot is not None:
self.feed_slot = int(self.feed_slot)
self.nfeed_checked = 0
self.last_published = pendulum.utcnow().subtract(days=1)
log.debug('successfully initialized {}'.format(self))
def dummy_trigger__callable(*, dag_run: DagRun, **kwargs):
"""
In test env we just want to trigger the etl_testing DAG with
no config.
"""
logger.info(dag_run)
trigger_dag("etl_testing", run_id=str(uuid1()), execution_date=utcnow())
def sync_programs(client, state, stream):
# http://developers.marketo.com/rest-api/assets/programs/#by_date_range
#
# Programs are queryable via their updatedAt time but require and
# end date as well. As there is no max time range for the query,
# query from the bookmark value until current.
#
# The Programs endpoint uses offsets with a return limit of 200
# per page. If requesting past the final program, an error message
# is returned to indicate that the endpoint has been fully synced.
replication_key = determine_replication_key(stream['tap_stream_id'])
singer.write_schema("programs",
stream["schema"],
stream["key_properties"],
bookmark_properties=[replication_key])
start_date = bookmarks.get_bookmark(state, "programs", replication_key)
end_date = pendulum.utcnow().isoformat()
params = {
"maxReturn": 200,
"offset": 0,
"earliestUpdatedAt": start_date,
"latestUpdatedAt": end_date,
}
endpoint = "rest/asset/v1/programs.json"
record_count = 0
while True:
data = client.request("GET",
endpoint,
endpoint_name="programs",
params=params)
# If the no asset message is in the warnings, we have exhausted
# the search results and can end the sync.
if "warnings" in data and NO_ASSET_MSG in data["warnings"]:
break
time_extracted = utils.now()
# Each row just needs the values formatted. If the record is
# newer than the original start date, stream the record.
for row in data["result"]:
record = format_values(stream, row)
if record[replication_key] >= start_date:
record_count += 1
singer.write_record("programs",
record,
time_extracted=time_extracted)
# Increment the offset by the return limit for the next query.
params["offset"] += params["maxReturn"]
# Now that we've finished every page we can update the bookmark to
# the end of the query.
state = bookmarks.write_bookmark(state, "programs", replication_key,
end_date)
singer.write_state(state)
return state, record_count
async def strawpolls(self, ctx, poll_id: str = None):
"""This command can be used to show a strawpoll setup on this server"""
# Strawpolls cannot be 'deleted' so to handle whether a poll is running or not on a server
# Just save the poll in the config file, which can then be removed when it should not be "running" anymore
all_polls = await config.get_content('strawpolls')
server_polls = all_polls.get(ctx.message.server.id) or {}
if not server_polls:
await self.bot.say("There are currently no strawpolls running on this server!")
return
# If no poll_id was provided, print a list of all current running poll's on this server
if not poll_id:
fmt = "\n".join(
"{}: https://strawpoll.me/{}".format(data['title'], _id) for _id, data in server_polls.items())
await self.bot.say("```\n{}```".format(fmt))
# Else if a valid poll_id was provided, print info about that poll
elif poll_id in server_polls.keys():
poll = server_polls[poll_id]
async with self.session.get("{}/{}".format(self.url, poll_id),
headers={'User-Agent': 'Bonfire/1.0.0'}) as response:
data = await response.json()
# The response for votes and options is provided as two separate lists
# We are enumarting the list of options, to print r (the option)
# And the votes to match it, based on the index of the option
# The rest is simple formatting
fmt_options = "\n\t".join(
"{}: {}".format(r, data['votes'][i]) for i, r in enumerate(data['options']))
author = discord.utils.get(ctx.message.server.members, id=poll['author'])
created_ago = (pendulum.utcnow() - pendulum.parse(poll['date'])).in_words()
link = "https://strawpoll.me/{}".format(poll_id)
fmt = "Link: {}\nTitle: {}\nAuthor: {}\nCreated: {} ago\nOptions:\n\t{}".format(link, data['title'],
author.display_name,
created_ago, fmt_options)
await self.bot.say("```\n{}```".format(fmt))
def make(session, args):
category = EventCategoryFactory.make(args['ec'])
return Event(category=category,
session=session,
name=args['en'],
data=args['ed'],
created=pendulum.utcnow())
def should_publish(self):
# TODO: update according to: https://bitsharestalk.org/index.php?topic=9348.0;all
#return False
if self.nfeed_checked == 0:
log.debug('Should publish at least once at launch of the bts_tools')
return True
if self.feed_period is not None and self.nfeed_checked % self.feed_period == 0:
log.debug('Should publish because time interval has passed: {} seconds'.format(self.publish_time_interval))
return True
now = pendulum.utcnow()
if self.publish_time_interval and now - self.last_published > self.publish_time_interval:
log.debug('Should publish because time interval has passed: {}'.format(self.publish_time_interval))
return True
if self.feed_slot:
target = now.replace(minute=self.feed_slot, second=0, microsecond=0)
targets = [target.subtract(hours=1), target, target.add(hours=1)]
diff = [now-t for t in targets]
# check if we just passed our time slot
if any(pendulum.interval() < d and abs(d) < 1.1*self.check_time_interval for d in diff):
log.debug('Should publish because time slot has arrived: time {:02d}:{:02d}'.format(now.hour, now.minute))
return True
log.debug('No need to publish feeds')
return False
def choose_branch(self, context: Dict) -> Union[str, Iterable[str]]:
# If the DAG Run is externally triggered, then return without
# skipping downstream tasks
if context['dag_run'] and context['dag_run'].external_trigger:
self.log.info(
"Externally triggered DAG_Run: allowing execution to proceed.")
return list(
context['task'].get_direct_relative_ids(upstream=False))
now = pendulum.utcnow()
left_window = context['dag'].following_schedule(
context['execution_date'])
right_window = context['dag'].following_schedule(left_window)
self.log.info(
'Checking latest only with left_window: %s right_window: %s now: %s',
left_window, right_window, now)
if not left_window < now <= right_window:
self.log.info('Not latest execution, skipping downstream.')
# we return an empty list, thus the parent BaseBranchOperator
# won't exclude any downstream tasks from skipping.
return []
else:
self.log.info('Latest, allowing execution to proceed.')
return list(
context['task'].get_direct_relative_ids(upstream=False))
def test_timestamp_with_timezone(self):
now = pendulum.utcnow()
user = OratorTestUser.create(email='*****@*****.**', created_at=now)
fresh_user = OratorTestUser.find(user.id)
self.assertEqual(user.created_at, fresh_user.created_at)
self.assertEqual(now, fresh_user.created_at)
def elasticsearch_janitor(self,
es_url=None,
es_index=None,
dry=False,
to_daemon=False):
"""
Looks for discrepancies between postgres and elastic search numbers
Re-indexes time periods that differ in count
"""
# get range of date_created in database; assumes current time is the max
logger.debug('Starting Elasticsearch JanitorTask')
min_date = AbstractCreativeWork.objects.all().aggregate(
models.Min('date_created'))['date_created__min']
if not min_date:
logger.warning('No CreativeWorks are present in Postgres. Exiting')
return
max_date = pendulum.utcnow()
min_date = pendulum.instance(min_date)
pseudo_bisection.apply((es_url, es_index, min_date, max_date), {
'dry': dry,
'to_daemon': to_daemon
},
throw=True)
def execute(self, context):
# If the DAG Run is externally triggered, then return without
# skipping downstream tasks
if context['dag_run'] and context['dag_run'].external_trigger:
self.log.info("Externally triggered DAG_Run: allowing execution to proceed.")
return
now = pendulum.utcnow()
left_window = context['dag'].following_schedule(
context['execution_date'])
right_window = context['dag'].following_schedule(left_window)
self.log.info(
'Checking latest only with left_window: %s right_window: %s now: %s',
left_window, right_window, now
)
if not left_window < now <= right_window:
self.log.info('Not latest execution, skipping downstream.')
downstream_tasks = context['task'].get_flat_relatives(upstream=False)
self.log.debug("Downstream task_ids %s", downstream_tasks)
if downstream_tasks:
self.skip(context['dag_run'],
context['ti'].execution_date,
downstream_tasks)
self.log.info('Done.')
else:
self.log.info('Latest, allowing execution to proceed.')
def all(self, cutoff=None, allow_full_harvest=True, **kwargs):
"""
Args:
cutoff (date, optional): The upper bound to schedule harvests to. Default to today.
allow_full_harvest (bool, optional): Allow a SourceConfig to generate a full harvest. Defaults to True.
The SourceConfig.full_harvest must be marked True and have earliest_date set.
**kwargs: Forwarded to .range
Returns:
A list of harvest jobs
"""
if cutoff is None:
cutoff = pendulum.utcnow().date()
# TODO take harvest/sourceconfig version into account here
if hasattr(self.source_config, 'latest'):
latest_date = self.source_config.latest
else:
latest_date = self.source_config.harvest_jobs.aggregate(
models.Max('end_date'))['end_date__max']
# If we can build full harvests and the earliest job that would be generated does NOT exist
# Go ahead and reset the latest_date to the earliest_date
if allow_full_harvest and self.source_config.earliest_date and self.source_config.full_harvest:
if not self.source_config.harvest_jobs.filter(
start_date=self.source_config.earliest_date).exists():
latest_date = self.source_config.earliest_date
# If nothing sets latest_date, default to the soonest possible harvest
if not latest_date:
latest_date = cutoff - self.source_config.harvest_interval
return self.range(latest_date, cutoff, **kwargs)
async def strawpolls(self, ctx, poll_id: str = None):
"""This command can be used to show a strawpoll setup on this server"""
# Strawpolls cannot be 'deleted' so to handle whether a poll is running or not on a server
# Just save the poll in the config file, which can then be removed when it should not be "running" anymore
all_polls = config.get_content('strawpolls') or {}
server_polls = all_polls.get(ctx.message.server.id) or {}
if not server_polls:
await self.bot.say("There are currently no strawpolls running on this server!")
return
# If no poll_id was provided, print a list of all current running poll's on this server
if not poll_id:
fmt = "\n".join(
"{}: https://strawpoll.me/{}".format(data['title'], _id) for _id, data in server_polls.items())
await self.bot.say("```\n{}```".format(fmt))
# Else if a valid poll_id was provided, print info about that poll
elif poll_id in server_polls.keys():
poll = server_polls[poll_id]
async with self.session.get("{}/{}".format(self.url, poll_id),
headers={'User-Agent': 'Bonfire/1.0.0'}) as response:
data = await response.json()
# The response for votes and options is provided as two separate lists
# We are enumarting the list of options, to print r (the option)
# And the votes to match it, based on the index of the option
# The rest is simple formatting
fmt_options = "\n\t".join(
"{}: {}".format(r, data['votes'][i]) for i, r in enumerate(data['options']))
author = discord.utils.get(ctx.message.server.members, id=poll['author'])
created_ago = (pendulum.utcnow() - pendulum.parse(poll['date'])).in_words()
link = "https://strawpoll.me/{}".format(poll_id)
fmt = "Link: {}\nTitle: {}\nAuthor: {}\nCreated: {} ago\nOptions:\n\t{}".format(link, data['title'],
author.display_name,
created_ago, fmt_options)
await self.bot.say("```\n{}```".format(fmt))
def sync_activities(client, state, stream, config):
# http://developers.marketo.com/rest-api/bulk-extract/bulk-activity-extract/
replication_key = determine_replication_key(stream['tap_stream_id'])
singer.write_schema(stream["tap_stream_id"],
stream["schema"],
stream["key_properties"],
bookmark_properties=[replication_key])
export_start = pendulum.parse(
bookmarks.get_bookmark(state, stream["tap_stream_id"],
replication_key))
job_started = pendulum.utcnow()
record_count = 0
while export_start < job_started:
export_id, export_end = get_or_create_export_for_activities(
client, state, stream, export_start, config)
state = wait_for_export(client, state, stream, export_id)
for row in stream_rows(client, "activities", export_id):
time_extracted = utils.now()
row = flatten_activity(row, stream)
record = format_values(stream, row)
singer.write_record(stream["tap_stream_id"],
record,
time_extracted=time_extracted)
record_count += 1
state = update_state_with_export_info(
state, stream, bookmark=export_start.isoformat())
export_start = export_end
return state, record_count
async def uptime(self):
"""Provides a printout of the current bot's uptime
EXAMPLE: !uptime
RESULT: A BAJILLION DAYS"""
await self.bot.say("Uptime: ```\n{}```".format(
(pendulum.utcnow() - self.bot.uptime).in_words()))
def _latest_only(**context):
now = pendulum.utcnow()
left_window = context["dag"].following_schedule(context["execution_date"])
right_window = context["dag"].following_schedule(left_window)
if not left_window < now <= right_window:
raise AirflowSkipException()
def refresh_token(self):
# http://developers.marketo.com/rest-api/authentication/#creating_an_access_token
params = {
"grant_type": "client_credentials",
"client_id": self.client_id,
"client_secret": self.client_secret,
}
singer.log_info("Refreshing token")
try:
url = self.get_url("identity/oauth/token")
resp = requests.get(url, params=params)
resp_time = pendulum.utcnow()
except requests.exceptions.ConnectionError as e:
raise ApiException(
"Connection error while refreshing token at {}.".format(
url)) from e
if resp.status_code != 200:
raise ApiException("Error refreshing token [{}]: {}".format(
resp.status_code, resp.content))
data = resp.json()
if "error" in data:
if data["error"] == "unauthorized":
msg = "Authorization failed: "
else:
msg = "Marketo API returned an error: "
msg += data.get("error_description", "No message from api")
raise ApiException(msg)
self.access_token = data["access_token"]
self.token_expires = resp_time.add(seconds=data["expires_in"] - 15)
singer.log_info("Token valid until %s", self.token_expires)
def test_swbiodiversity_harvester():
httpretty.enable()
httpretty.allow_net_connect = False
config = SourceConfig.objects.get(label=('org.swbiodiversity'))
url = config.harvester_kwargs['list_url']
harvester = config.get_harvester()
httpretty.register_uri(httpretty.GET, url,
body=main_page, content_type='text/html', match_querystring=True)
collection = furl(url)
collection.args['collid'] = 223
httpretty.register_uri(httpretty.GET, url + ';collid=(\d+)',
body=collection_page, content_type='text/html', match_querystring=True)
start = pendulum.utcnow() - timedelta(days=3)
end = pendulum.utcnow()
results = harvester.fetch_date_range(start, end)
for result in results:
assert result.identifier == collection.url
assert "".join(result.datum.split()) == "".join('''
<div id="innertext">
<h1>SEINet - Arizona Chapter Collections </h1>
<div>
Select a collection to see full details.
</div>
<table style="margin:10px;">
<tr>
<td>
<h3>
<a href="collprofiles.php?collid=223">
A. Michael Powell Herbarium
</a>
</h3>
<div style="margin:10px;">
<div>Sample description</div>
<div style="margin-top:5px;">
<b>Contact:</b>
Test Author ([email protected])
</div>
</div>
</td>
</tr>
</table>
</div>
'''"".split())
httpretty.disable()
async def on_ready():
# Change the status upon connection to the default status
await bot.change_presence(
game=discord.Game(name=utils.default_status, type=0))
if not hasattr(bot, 'uptime'):
bot.uptime = pendulum.utcnow()
await utils.db_check()
def test_AEA_harvester():
httpretty.enable()
httpretty.allow_net_connect = False
config = SourceConfig.objects.get(label='org.socialscienceregistry')
url = config.base_url + '/trials/search.csv'
harvester = config.get_harvester()
httpretty.register_uri(httpretty.GET,
url,
body=csv_repsonse,
content_type='text/html')
start = pendulum.utcnow() - timedelta(days=3)
end = pendulum.utcnow()
result = harvester._do_fetch(start, end)
for data in result:
assert data[0] == 'AEARCTR-0000005'
assert len(data[1]['record']) == 41
def backoff_time(self, response: requests.Response) -> Optional[float]:
# The rate limit resets on the timestamp indicated
# https://developer.okta.com/docs/reference/rate-limits
if response.status_code == requests.codes.TOO_MANY_REQUESTS:
next_reset_epoch = int(response.headers["x-rate-limit-reset"])
next_reset = pendulum.from_timestamp(next_reset_epoch)
next_reset_duration = pendulum.utcnow().diff(next_reset)
return next_reset_duration.seconds
def entered(text):
if LINES.full():
_ = LINES.get()
tt = text[:144]
now = pendulum.utcnow()
LINES.put((now, tt))
update_chat()
talk.do_text('')
def render_servers(servers, output, title='TeamspeakStats',
template='index.jinja2', datetime_fmt='%x %X %Z',
onlinetime_threshold=-1, lastseen_relative=True):
'''
Render `servers`
:param servers: list of servers to render
:param output: path to output-file
:param template_name: path to template-file
:param title: title of the resulting html-document
:param template_path: path to template-file
:param datetime_fmt: custom datetime-format
:param onlinetime_threshold: threshold for clients onlinetime
:param lastseen_relative: render last seen timestamp relative
:type servers: [tsstats.log.Server]
:type output: str
:type template_name: str
:type title: str
:type template_path: str
:type datetime_fmt: str
:type onlinetime_threshold: int
:type lastseen_relative: bool
'''
# preparse servers
prepared_servers = [
Server(sid, prepare_clients(clients, onlinetime_threshold))
for sid, clients in servers
]
# render
template_loader = ChoiceLoader([
PackageLoader(__package__, 'templates'),
FileSystemLoader(join(dirname(__file__), 'templates'))
])
template_env = Environment(loader=template_loader)
def frmttime(timestamp):
if not timestamp:
return ''
formatted = timestamp.strftime(datetime_fmt)
logger.debug('Formatting timestamp %s -> %s', timestamp, formatted)
return formatted
def lastseen(timestamp):
if lastseen_relative:
return timestamp.diff_for_humans()
else:
return frmttime(timestamp)
template_env.filters['frmttime'] = frmttime
template_env.filters['lastseen'] = lastseen
template = template_env.get_template(template)
logger.debug('Rendering template %s', template)
template.stream(title=title, servers=prepared_servers,
debug=logger.level <= logging.DEBUG,
creation_time=pendulum.utcnow())\
.dump(output, encoding='utf-8')
logger.debug('Wrote rendered template to %s', output)
async def check_raffles(self):
# This is used to periodically check the current raffles, and see if they have ended yet
# If the raffle has ended, we'll pick a winner from the entrants
raffles = await utils.get_content('raffles')
if raffles is None:
return
for raffle in raffles:
server = self.bot.get_server(raffle['server_id'])
# Check to see if this cog can find the server in question
if server is None:
continue
now = pendulum.utcnow()
expires = pendulum.parse(raffle['expires'])
# Now lets compare and see if this raffle has ended, if not just continue
if expires > now:
continue
title = raffle['title']
entrants = raffle['entrants']
raffle_id = raffle['id']
# Make sure there are actually entrants
if len(entrants) == 0:
fmt = 'Sorry, but there were no entrants for the raffle `{}`!'.format(title)
else:
winner = None
count = 0
while winner is None:
winner = server.get_member(random.SystemRandom().choice(entrants))
# Lets make sure we don't get caught in an infinite loop
# Realistically having more than 25 random entrants found that aren't in the server anymore
# Isn't something that should be an issue
count += 1
if count >= 25:
break
if winner is None:
fmt = 'I couldn\'t find an entrant that is still in this server, for the raffle `{}`!'.format(title)
else:
fmt = 'The raffle `{}` has just ended! The winner is {}!'.format(title, winner.display_name)
# No matter which one of these matches were met, the raffle has ended and we want to remove it
# We don't have to wait for it however, so create a task for it
self.bot.loop.create_task(utils.remove_content('raffles', raffle_id ))
server_settings = await utils.get_content('server_settings', str(server.id))
channel_id = server_settings.get('notification_channel', server.id)
channel = self.bot.get_channel(channel_id)
try:
await channel.send(fmt)
except discord.Forbidden:
pass
async def info(self):
"""This command can be used to print out some of my information"""
# fmt is a dictionary so we can set the key to it's output, then print both
# The only real use of doing it this way is easier editing if the info in this command is changed
fmt = {}
bot_data = await config.get_content('bot_data')
total_data = {}
for shard, values in bot_data.items():
for key, value in values.items():
if key in total_data:
total_data[key] += value
else:
total_data[key] = value
# We can pretty safely assume that the author is going to be in at least one channel with the bot
# So find the author based on that list
fmt['Official Bot Server'] = config.dev_server
fmt['Uptime'] = (pendulum.utcnow() - self.bot.uptime).in_words()
fmt['Total Servers'] = total_data.get('server_count')
fmt['Total Members'] = total_data.get('member_count')
fmt['Description'] = self.bot.description
servers_playing_music = len([
server_id for server_id, state in self.bot.get_cog(
'Music').voice_states.items() if state.is_playing()
])
hm_games = len([
server_id
for server_id, game in self.bot.get_cog('Hangman').games.items()
])
ttt_games = len([
server_id for server_id, game in self.bot.get_cog(
'TicTacToe').boards.items()
])
count_battles = 0
for battles in self.bot.get_cog('Interaction').battles.values():
count_battles += len(battles)
information = "\n".join("{}: {}".format(key, result)
for key, result in fmt.items())
information += "\n"
if servers_playing_music:
information += "Playing songs in {} different servers\n".format(
servers_playing_music)
if hm_games:
information += "{} different hangman games running\n".format(
hm_games)
if ttt_games:
information += "{} different TicTacToe games running\n".format(
ttt_games)
if count_battles:
information += "{} different battles going on\n".format(
count_battles)
await self.bot.say("```\n{}```".format(information))
async def create_strawpoll(self, ctx, title, *, options):
"""This command is used to setup a new strawpoll
The format needs to be: poll create "title here" all options here
Options need to be separated by using either one ` around each option
Or use a code block (3 ` around the options), each option on it's own line"""
# The following should use regex to search for the options inside of the two types of code blocks with `
# We're using this instead of other things, to allow most used puncation inside the options
match_single = getter.findall(options)
match_multi = multi.findall(options)
# Since match_single is already going to be a list, we just set
# The options to match_single and remove any blank entries
if match_single:
options = match_single
options = [option for option in options if option]
# Otherwise, options need to be set based on the list, split by lines.
# Then remove blank entries like the last one
elif match_multi:
options = match_multi[0].splitlines()
options = [option for option in options if option]
# If neither is found, then error out and let them know to use the help command, since this one is a bit finicky
else:
await self.bot.say(
"Please provide options for a new strawpoll! Use {}help {} if you do not know the format"
.format(ctx.prefix, ctx.command.qualified_name))
return
# Make the post request to strawpoll, creating the poll, and returning the ID
# The ID is all we really need from the returned data, as the rest we already sent/are not going to use ever
payload = {'title': title, 'options': options}
try:
async with self.session.post(self.url,
data=json.dumps(payload),
headers=self.headers) as response:
data = await response.json()
except json.JSONDecodeError:
await self.bot.say(
"Sorry, I couldn't connect to strawpoll at the moment. Please try again later"
)
return
# Save this strawpoll in the list of running strawpolls for a server
poll_id = str(data['id'])
r_filter = {'server_id': ctx.message.server.id}
sub_entry = {
'poll_id': poll_id,
'author': ctx.message.author.id,
'date': str(pendulum.utcnow()),
'title': title
}
entry = {'server_id': ctx.message.server.id, 'polls': [sub_entry]}
update = {'polls': r.row['polls'].append(sub_entry)}
if not await config.update_content('strawpolls', update, r_filter):
await config.add_content('strawpolls', entry, {'poll_id': poll_id})
await self.bot.say(
"Link for your new strawpoll: https://strawpoll.me/{}".format(
poll_id))
def parse_and_upload_vendor(vendor_id: str):
start = get_last_import_date(vendor_id)
end = pendulum.utcnow()
days = list_days_between(start, end)
for date in days:
objs = list_objects(vendor_id, date)
for obj in objs:
parse_and_upload(obj)
set_last_import_date(vendor_id, date)
def headers(self):
# http://developers.marketo.com/rest-api/authentication/#using_an_access_token
if not self.token_expires or self.token_expires <= pendulum.utcnow():
self.refresh_token()
return {
"Authorization": "Bearer {}".format(self.access_token),
"User-Agent": self.user_agent,
}
async def motd_push(self, *, message):
"""Used to push a new message to the message of the day"""
date = pendulum.utcnow().to_date_string()
key = date
entry = {'motd': message, 'date': date}
# Try to add this, if there's an entry for that date, lets update it to make sure only one motd is sent a day
# I should be managing this myself, more than one should not be sent in a day
if await utils.add_content('motd', entry):
await utils.update_content('motd', entry, key)
await self.bot.say("New motd update for {}!".format(date))
async def time(self, ctx):
"""
Get current eve time - !help time for more options
"""
if ctx.invoked_subcommand is None:
async with ctx.typing():
await asyncio.sleep(.5)
return await ctx.send(embed=embeds.Embed(
title='Current Eve Time:',
description=pendulum.utcnow().to_datetime_string()))
def main(config: CONFIGTYPE) -> COMMANDLISTTYPE:
"""Command builder.
:param config: config snippet for this plugin
:returns: Commands to create the backup
"""
# create time variables
now = pendulum.utcnow()
today_day_of_month = now.format('%d')
yesterday_day_of_month = now.yesterday().format('%d')
today_day_of_week = now.format('%w')
yesterday_day_of_week = now.yesterday().format('%w')
# adding basic rsync stuff
command = ['rsync', '-av', '--delete']
# generate paths
source = normalize_path(config['source']) + '/'
destination = normalize_path(config['destination'])
# add ssh as shell if its needed
if '@' in destination:
command.append('-e ssh')
# check for exclude and include and add it to the rsync command list
if 'exclude' in config.keys():
for item in config['exclude']:
command.append('--exclude={}'.format(item))
if 'include' in config.keys():
for item in config['include']:
command.append('--include={}'.format(item))
# handling rsync mode
if config['mode'] == 'month':
# if using days you have to specify link-dest
command.append('--link-dest=../{}'.format(yesterday_day_of_month))
destination = os.path.join(destination, today_day_of_month)
elif config['mode'] == 'week':
# if using days you have to specify link-dest
command.append('--link-dest=../{}'.format(yesterday_day_of_week))
destination = os.path.join(destination, today_day_of_week)
elif config['mode'] == 'once':
# once mode doesnt need another commands
pass
# append source and destination
command.append(source)
command.append(destination)
return [command]
async def create(message: discord.Message, tag: tag_arg, *time, timezone: tz_arg="UTC"):
""" Create a countdown with the specified tag, using the same format as `{pre}when`. """
assert tag not in time_cfg.data["countdown"], "Countdown with tag `{}` already exists.".format(tag)
timezone_name = timezone
dt, timezone = await init_dt(message, " ".join(time), timezone)
assert (dt - pendulum.utcnow()).seconds > 0, "A countdown has to be set in the future."
time_cfg.data["countdown"][tag] = dict(time=dt.to_datetime_string(), tz=timezone, tz_name=timezone_name,
author=message.author.id, channel=message.channel.id)
time_cfg.save()
await client.say(message, "Added countdown with tag `{}`.".format(tag))
async def create_strawpoll(self, ctx, title, *, options):
"""This command is used to setup a new strawpoll
The format needs to be: poll create "title here" all options here
Options need to be separated by using either one ` around each option
Or use a code block (3 ` around the options), each option on it's own line"""
# The following should use regex to search for the options inside of the two types of code blocks with `
# We're using this instead of other things, to allow most used puncation inside the options
match_single = getter.findall(options)
match_multi = multi.findall(options)
# Since match_single is already going to be a list, we just set
# The options to match_single and remove any blank entries
if match_single:
options = match_single
options = [option for option in options if option]
# Otherwise, options need to be set based on the list, split by lines.
# Then remove blank entries like the last one
elif match_multi:
options = match_multi[0].splitlines()
options = [option for option in options if option]
# If neither is found, then error out and let them know to use the help command, since this one is a bit finicky
else:
await self.bot.say(
"Please provide options for a new strawpoll! Use {}help {} if you do not know the format".format(
ctx.prefix, ctx.command.qualified_name))
return
# Make the post request to strawpoll, creating the poll, and returning the ID
# The ID is all we really need from the returned data, as the rest we already sent/are not going to use ever
payload = {'title': title,
'options': options}
try:
async with self.session.post(self.url, data=json.dumps(payload), headers=self.headers) as response:
data = await response.json()
except json.JSONDecodeError:
await self.bot.say("Sorry, I couldn't connect to strawpoll at the moment. Please try again later")
return
# Save this strawpoll in the list of running strawpolls for a server
poll_id = str(data['id'])
r_filter = {'server_id': ctx.message.server.id}
sub_entry = {'poll_id': poll_id,
'author': ctx.message.author.id,
'date': str(pendulum.utcnow()),
'title': title}
entry = {'server_id': ctx.message.server.id,
'polls': [sub_entry]}
update = {'polls': r.row['polls'].append(sub_entry)}
if not await config.update_content('strawpolls', update, r_filter):
await config.add_content('strawpolls', entry, {'poll_id': poll_id})
await self.bot.say("Link for your new strawpoll: https://strawpoll.me/{}".format(poll_id))
async def on_ready():
# Change the status upon connection to the default status
await bot.change_status(discord.Game(name=config.defaultStatus, type=0))
channel_id = config.get_content('restart_server')
# Just in case the bot was restarted while someone was battling, clear it so they do not get stuck
config.save_content('battling', {})
# Check if the bot was restarted, if so send a message to the channel the bot was restarted from
if channel_id != 0:
destination = discord.utils.find(lambda m: m.id == channel_id, bot.get_all_channels())
await bot.send_message(destination, getPhrase("RESTART_COMPLETE"))
config.save_content('restart_server', 0)
if not hasattr(bot, 'uptime'):
bot.uptime = pendulum.utcnow()
def __init__(self, price, asset, base, volume=None, last_updated=None, provider=None):
self.price = price
self.asset = asset
self.base = base
self.volume = volume # volume of the market from which this price is coming, if any
self.last_updated = last_updated or pendulum.utcnow()
self.provider = provider
if self.provider is None:
# try to get the module from the call stack
frm = inspect.stack()[1]
mod = inspect.getmodule(frm[0])
name = getattr(mod, 'NAME', None)
self.provider = name
def format_when(dt: pendulum.Pendulum, timezone: str="UTC"):
""" Format when something will happen"""
now = pendulum.utcnow()
diff = dt - now
major_diff = dt.diff_for_humans(absolute=True)
detailed_diff = diff.in_words().replace("-", "")
return "`{time} {tz}` {pronoun} **{major}{diff}{pronoun2}**.".format(
time=dt.format(dt_format),
tz=timezone,
pronoun="is in" if dt > now else "was",
major="~" + major_diff + "** / **" if major_diff not in detailed_diff else "",
diff=detailed_diff,
pronoun2=" ago" if dt < now else ""
)
async def info(self):
"""This command can be used to print out some of my information"""
# fmt is a dictionary so we can set the key to it's output, then print both
# The only real use of doing it this way is easier editing if the info
# in this command is changed
bot_data = await utils.get_content('bot_data')
total_data = {'member_count': 0,
'server_count': 0}
for entry in bot_data:
total_data['member_count'] += entry['member_count']
total_data['server_count'] += entry['server_count']
# Create the original embed object
opts = {'title': 'Dev Server',
'description': 'Join the server above for any questions/suggestions about me.',
'url': utils.dev_server}
embed = discord.Embed(**opts)
# Add the normal values
embed.add_field(name='Total Servers', value=total_data['server_count'])
embed.add_field(name='Total Members', value=total_data['member_count'])
# Count the variable values; hangman, tictactoe, etc.
hm_games = len(self.bot.get_cog('Hangman').games)
ttt_games = len(self.bot.get_cog('TicTacToe').boards)
bj_games = len(self.bot.get_cog('Blackjack').games)
count_battles = 0
for battles in self.bot.get_cog('Interaction').battles.values():
count_battles += len(battles)
if hm_games:
embed.add_field(name='Total Hangman games running', value=hm_games)
if ttt_games:
embed.add_field(name='Total TicTacToe games running', value=ttt_games)
if count_battles:
embed.add_field(name='Total battles games running', value=count_battles)
if bj_games:
embed.add_field(name='Total blackjack games running', value=bj_games)
embed.add_field(name='Uptime', value=(pendulum.utcnow() - self.bot.uptime).in_words())
embed.set_footer(text=self.bot.description)
await self.bot.say(embed=embed)
def get(cur, base):
log.debug('checking feeds for %s/%s at %s' % (cur, base, NAME))
prices = []
for dataset in DATASETS[(cur, base)]:
url = 'https://www.quandl.com/api/v3/datasets/{dataset}.json?start_date={date}'.format(
dataset=dataset,
date=(pendulum.utcnow() - pendulum.interval(days=3)).strftime('%Y-%m-%d')
)
data = requests.get(url=url, timeout=TIMEOUT).json()
if 'dataset' not in data:
raise RuntimeError('Quandl: no dataset found for url: %s' % url)
d = data['dataset']
if len(d['data']):
prices.append(d['data'][0][1])
return FeedPrice(sum(prices) / len(prices), cur, base)
async def strawpolls(self, ctx, poll_id: str = None):
"""This command can be used to show a strawpoll setup on this server
EXAMPLE: !strawpolls
RESULT: A list of all polls setup on this server"""
# Strawpolls cannot be 'deleted' so to handle whether a poll is running or not on a server
# Just save the poll, which can then be removed when it should not be "running" anymore
polls = await config.get_content('strawpolls', ctx.message.server.id)
# Check if there are any polls setup on this server
try:
polls = polls[0]['polls']
except TypeError:
await self.bot.say("There are currently no strawpolls running on this server!")
return
# Print all polls on this server if poll_id was not provided
if poll_id is None:
fmt = "\n".join(
"{}: https://strawpoll.me/{}".format(data['title'], data['poll_id']) for data in polls)
await self.bot.say("```\n{}```".format(fmt))
else:
# Since strawpoll should never allow us to have more than one poll with the same ID
# It's safe to assume there's only one result
try:
poll = [p for p in polls if p['poll_id'] == poll_id][0]
except IndexError:
await self.bot.say("That poll does not exist on this server!")
return
async with self.session.get("{}/{}".format(self.url, poll_id),
headers={'User-Agent': 'Bonfire/1.0.0'}) as response:
data = await response.json()
# The response for votes and options is provided as two separate lists
# We are enumarting the list of options, to print r (the option)
# And the votes to match it, based on the index of the option
# The rest is simple formatting
fmt_options = "\n\t".join(
"{}: {}".format(result, data['votes'][i]) for i, result in enumerate(data['options']))
author = discord.utils.get(ctx.message.server.members, id=poll['author'])
created_ago = (pendulum.utcnow() - pendulum.parse(poll['date'])).in_words()
link = "https://strawpoll.me/{}".format(poll_id)
fmt = "Link: {}\nTitle: {}\nAuthor: {}\nCreated: {} ago\nOptions:\n\t{}".format(link, data['title'],
author.display_name,
created_ago, fmt_options)
await self.bot.say("```\n{}```".format(fmt))
def elasticsearch_janitor(self, es_url=None, es_index=None, dry=False, to_daemon=True):
"""
Looks for discrepancies between postgres and elastic search numbers
Re-indexes time periods that differ in count
"""
# get range of date_created in database; assumes current time is the max
logger.debug('Starting Elasticsearch JanitorTask')
min_date = AbstractCreativeWork.objects.all().aggregate(models.Min('date_created'))['date_created__min']
if not min_date:
logger.warning('No CreativeWorks are present in Postgres. Exiting')
return
max_date = pendulum.utcnow()
min_date = pendulum.instance(min_date)
pseudo_bisection.apply((es_url, es_index, str(min_date), str(max_date)), {'dry': dry, 'to_daemon': to_daemon}, throw=True)
def should_publish_steem(self, node, price):
# check whether we need to publish again:
# - if published more than 12 hours ago, publish again
# - if published price different by more than 3%, publish again
if 'last_price' not in node.opts: # make sure we have already published once
log.debug('Steem should publish for the first time since launch of bts_tools')
return True
last_published_interval = pendulum.interval(hours=12)
variance_trigger = 0.03
if pendulum.utcnow() - node.opts['last_published'] > last_published_interval: # FIXME: replace node.opts['last_published'] with self.last_published[node]
log.debug('Steem should publish as it has not been published for {}'.format(last_published_interval))
return True
if abs(price - node.opts['last_price']) / node.opts['last_price'] >= variance_trigger:
log.debug('Steem should publish as price has moved more than {}%'.format(100*variance_trigger))
return True
log.debug('No need for Steem to publish')
return False
async def motd(self, *, date=None):
"""This command can be used to print the current MOTD (Message of the day)
This will most likely not be updated every day, however messages will still be pushed to this every now and then
EXAMPLE: !motd
RESULT: 'This is an example message of the day!'"""
if date is None:
motd = await utils.get_content('motd')
try:
# Lets set this to the first one in the list first
latest_motd = motd[0]
for entry in motd:
d = pendulum.parse(entry['date'])
# Check if the date for this entry is newer than our currently saved latest entry
if d > pendulum.parse(latest_motd['date']):
latest_motd = entry
date = latest_motd['date']
motd = latest_motd['motd']
# This will be hit if we do not have any entries for motd
except TypeError:
await self.bot.say("No message of the day!")
else:
fmt = "Last updated: {}\n\n{}".format(date, motd)
await self.bot.say(fmt)
else:
try:
motd = await utils.get_content('motd', str(pendulum.parse(date).date()))
date = motd['date']
motd = motd['motd']
fmt = "Message of the day for {}:\n\n{}".format(date, motd)
await self.bot.say(fmt)
# This one will be hit if we return None for that day
except TypeError:
await self.bot.say("No message of the day for {}!".format(date))
# This will be hit if pendulum fails to parse the date passed
except ValueError:
now = pendulum.utcnow().to_date_string()
await self.bot.say("Invalid date format! Try like {}".format(now))
async def info(self):
"""This command can be used to print out some of my information"""
# fmt is a dictionary so we can set the key to it's output, then print both
# The only real use of doing it this way is easier editing if the info in this command is changed
fmt = {}
all_members = list(self.bot.get_all_members())
# We can pretty safely assume that the author is going to be in at least one channel with the bot
# So find the author based on that list
authors = []
for author_id in config.owner_ids:
authors.append(discord.utils.get(all_members, id=author_id).name)
fmt['Official Bot Server'] = config.dev_server
fmt['Author'] = ", ".join(authors)
fmt['Uptime'] = (pendulum.utcnow() - self.bot.uptime).in_words()
fmt['Total Servers'] = len(self.bot.servers)
fmt['Total Members'] = len(all_members)
fmt['Description'] = self.bot.description
information = "\n".join("{}: {}".format(key, result) for key, result in fmt.items())
await self.bot.say("```\n{}```".format(information))
def __init__(self, key):
self.key = key
self.values = {}
self.refreshed = pendulum.utcnow()
loop.create_task(self.update())
# pylint: disable=redefined-builtin
import os
import shutil
from unittest.mock import patch
import pendulum
import pytest
import yaml
import helper
from dothebackup import runner
# GLOBAL VARIABLES
# ----------------
now = pendulum.utcnow()
today_day_of_month = now.format('%d')
yesterday_day_of_month = now.yesterday().format('%d')
today_day_of_week = now.format('%w')
yesterday_day_of_week = now.yesterday().format('%w')
once_input = {
'type': 'rsync',
'mode': 'once',
'source': '/foo/bar',
'destination': '/bumm/zack'
}
once_expected = [
def check_feeds(nodes):
# 1- get all feeds from all feed providers (FP) at once. Only use FP designated as active
# 2- compute price from the FeedSet using adequate strategy. FP can (should) be weighted
# (eg: BitcoinAverage: 4, BitFinex: 1, etc.) if no volume is present
# 3- order of computation should be clearly defined and documented, or configurable in the yaml file (preferable)
#
global feeds, feed_control
try:
feeds, publish_list = get_feed_prices(nodes[0], core.config['monitoring']['feeds'][nodes[0].type()]) # use first node if we need to query the blockchain, eg: for bit20 asset composition
feed_control.nfeed_checked += 1
status = feed_control.publish_status({(k, get_base_for(k)): v for k, v in feeds.items()})
log.debug('Got feeds on {}: {}'.format(nodes[0].type(), status))
for node in nodes:
if node.role != 'feed_publisher':
continue
# if an exception occurs during publishing feeds for a witness (eg: inactive witness),
# then we should still go on for the other nodes (and not let exceptions propagate)
try:
if node.type() == 'bts':
if feed_control.should_publish():
base_error_msg = 'Cannot publish feeds for {} witness {}: '.format(node.type(), node.name)
if check_node_is_ready(node, base_error_msg) is False:
continue
base_msg = '{} witness {} feeds: '.format(node.type(), node.name)
# publish median value of the price, not latest one
median_feeds = {(c, get_base_for(c)): statistics.median(price_history[c]) for c in feeds}
publish_feeds = {(asset, base): median_feeds[(asset, base)] for asset, base in publish_list}
log.info(base_msg + 'publishing feeds: {}'.format(feed_control.format_feeds(publish_feeds)))
publish_bts_feed(node, cfg['bts'], publish_feeds, base_msg)
feed_control.last_published = pendulum.utcnow() # FIXME: last_published is only for 'bts' now...
elif node.type() == 'steem':
price = statistics.median(price_history['STEEM'])
# publish median value of the price, not latest one
if feed_control.should_publish_steem(node, price):
base_error_msg = 'Cannot publish feeds for steem witness {}: '.format(node.name)
if check_node_is_ready(node, base_error_msg) is False:
continue
publish_steem_feed(node, cfg['steem'], price)
node.opts['last_price'] = price
node.opts['last_published'] = pendulum.utcnow()
# elif node.type() == 'muse':
# ...
else:
log.error('Unknown blockchain type for feeds publishing: {}'.format(node.type()))
except Exception as e:
log.exception(e)
except core.NoFeedData as e:
log.warning(e)
except Exception as e:
log.exception(e)
threading.Timer(cfg['check_time_interval'], check_feeds, args=(nodes,)).start()
async def uptime(self):
"""Provides a printout of the current bot's uptime"""
await self.bot.say("%s ```\n{}```".format((pendulum.utcnow() - self.bot.uptime).in_words()) %getPhrase("CORE:UPTIME"))
async def uptime(self):
"""Provides a printout of the current bot's uptime
EXAMPLE: !uptime
RESULT: A BAJILLION DAYS"""
await self.bot.say("Uptime: ```\n{}```".format((pendulum.utcnow() - self.bot.uptime).in_words()))
def handle(self, *args, **options):
user = ShareUser.objects.get(username=settings.APPLICATION_USERNAME)
if options['ids']:
self.harvest_ids(user, options)
return
task_kwargs = {'force': options.get('force', False)}
if options['days_back'] is not None and (options['start'] or options['end']):
self.stdout.write('Please choose days-back OR a start date with end date, not both')
return
if options['days_back'] is not None:
task_kwargs['end'] = datetime.datetime.utcnow() + datetime.timedelta(days=-(options['days_back'] - 1))
task_kwargs['start'] = datetime.datetime.utcnow() + datetime.timedelta(days=-options['days_back'])
else:
task_kwargs['start'] = pendulum.parse(options['start']) if options.get('start') else pendulum.utcnow() - datetime.timedelta(days=int(options['days_back'] or 1))
task_kwargs['end'] = pendulum.parse(options['end']) if options.get('end') else pendulum.utcnow()
task_kwargs['end'] = task_kwargs['end'].isoformat()
task_kwargs['start'] = task_kwargs['start'].isoformat()
if options['limit'] is not None:
task_kwargs['limit'] = options['limit']
if options['set_spec']:
task_kwargs['set_spec'] = options['set_spec']
if not options['harvester'] and options['all']:
options['harvester'] = [x.label for x in apps.get_app_configs() if isinstance(x, ProviderAppConfig) and not x.disabled]
for harvester in options['harvester']:
apps.get_app_config(harvester) # Die if the AppConfig can not be loaded
task_args = (user.id, harvester,)
if options['async']:
HarvesterTask().apply_async(task_args, task_kwargs)
self.stdout.write('Started job for harvester {}'.format(harvester))
else:
self.stdout.write('Running harvester for {}'.format(harvester))
HarvesterTask().apply(task_args, task_kwargs, throw=True)
async def raffle_create(self, ctx):
"""This is used in order to create a new server raffle
EXAMPLE: !raffle create
RESULT: A follow-along for setting up a new raffle"""
author = ctx.message.author
server = ctx.message.server
channel = ctx.message.channel
now = pendulum.utcnow()
await self.bot.say(
"Ready to start a new raffle! Please respond with the title you would like to use for this raffle!")
msg = await self.bot.wait_for_message(author=author, channel=channel, timeout=120)
if msg is None:
await self.bot.say("You took too long! >:c")
return
title = msg.content
fmt = "Alright, your new raffle will be titled:\n\n{}\n\nHow long would you like this raffle to run for? " \
"The format should be [number] [length] for example, `2 days` or `1 hour` or `30 minutes` etc. " \
"The minimum for this is 10 minutes, and the maximum is 3 months"
await self.bot.say(fmt.format(title))
# Our check to ensure that a proper length of time was passed
check = lambda m: re.search("\d+ (minutes?|hours?|days?|weeks?|months?)", m.content.lower()) is not None
msg = await self.bot.wait_for_message(author=author, channel=channel, timeout=120, check=check)
if msg is None:
await self.bot.say("You took too long! >:c")
return
# Lets get the length provided, based on the number and type passed
num, term = re.search("\d+ (minutes?|hours?|days?|weeks?|months?)", msg.content.lower()).group(0).split(' ')
# This should be safe to convert, we already made sure with our check earlier this would match
num = int(num)
# Now lets ensure this meets our min/max
if "minute" in term and (num < 10 or num > 129600):
await self.bot.say(
"Length provided out of range! The minimum for this is 10 minutes, and the maximum is 3 months")
return
elif "hour" in term and num > 2160:
await self.bot.say(
"Length provided out of range! The minimum for this is 10 minutes, and the maximum is 3 months")
return
elif "day" in term and num > 90:
await self.bot.say(
"Length provided out of range! The minimum for this is 10 minutes, and the maximum is 3 months")
return
elif "week" in term and num > 12:
await self.bot.say(
"Length provided out of range! The minimum for this is 10 minutes, and the maximum is 3 months")
return
elif "month" in term and num > 3:
await self.bot.say(
"Length provided out of range! The minimum for this is 10 minutes, and the maximum is 3 months")
return
# Pendulum only accepts the plural version of terms, lets make sure this is added
term = term if term.endswith('s') else '{}s'.format(term)
# If we're in the range, lets just pack this in a dictionary we can pass to set the time we want, then set that
payload = {term: num}
expires = now.add(**payload)
# Now we're ready to add this as a new raffle
entry = {'title': title,
'expires': expires.to_datetime_string(),
'entrants': [],
'author': author.id,
'server_id': server.id}
# We don't want to pass a filter to this, because we can have multiple raffles per server
await utils.add_content('raffles', entry)
await self.bot.say("I have just saved your new raffle!")
def handle(self, *args, **options):
user = ShareUser.objects.get(username=settings.APPLICATION_USERNAME)
if options['ids']:
self.harvest_ids(user, options)
return
task_kwargs = {'force': options.get('force', False), 'ignore_disabled': options.get('ignore_disabled', False)}
if options['days_back'] is not None and (options['start'] or options['end']):
self.stdout.write('Please choose days-back OR a start date with end date, not both')
return
if options['days_back'] is not None:
task_kwargs['end'] = datetime.datetime.utcnow() + datetime.timedelta(days=-(options['days_back'] - 1))
task_kwargs['start'] = datetime.datetime.utcnow() + datetime.timedelta(days=-options['days_back'])
else:
task_kwargs['start'] = pendulum.parse(options['start']) if options.get('start') else pendulum.utcnow() - datetime.timedelta(days=int(options['days_back'] or 1))
task_kwargs['end'] = pendulum.parse(options['end']) if options.get('end') else pendulum.utcnow()
task_kwargs['end'] = task_kwargs['end'].isoformat()
task_kwargs['start'] = task_kwargs['start'].isoformat()
if options['limit'] is not None:
task_kwargs['limit'] = options['limit']
if options['set_spec']:
task_kwargs['set_spec'] = options['set_spec']
if not options['source-config'] and options['all']:
options['source-config'] = SourceConfig.objects.filter(disabled=False).values_list('label', flat=True)
for label in options['source-config']:
assert SourceConfig.objects.filter(label=label).exists()
task_args = (user.id, label,)
if options['async']:
HarvesterTask().apply_async(task_args, task_kwargs)
self.stdout.write('Started harvest job for source config {}'.format(label))
else:
self.stdout.write('Running harvester for {}'.format(label))
HarvesterTask().apply(task_args, task_kwargs, throw=True)
async def update(self):
self.values = await get_content(self.key)
self.refreshed = pendulum.utcnow()