def __init__(self, player_service, game_stats_service):
self._dirty_games = set()
self._dirty_queues = set()
self.player_service = player_service
self.game_stats_service = game_stats_service
self.game_id_counter = 0
# Populated below in really_update_static_ish_data.
self.featured_mods = dict()
# A set of mod ids that are allowed in ranked games (everyone loves caching)
self.ranked_mods = set()
# The ladder map pool. Each entry is an (id, name, filename) tuple.
self.ladder_maps = set()
# Temporary proxy for the ladder service
self.ladder_service = None
# The set of active games
self.games = dict()
# Cached versions for files by game_mode ( featured mod name )
# For use by the patcher
self.game_mode_versions = dict()
# Synchronously initialise the game-id counter and static-ish-data.
loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.async(self.initialise_game_counter()))
loop.run_until_complete(loop.create_task(self.update_data()))
self._update_cron = aiocron.crontab('*/10 * * * *', func=self.update_data)
async def start(self):
await super().start()
self.me = await self.get_me()
logger.info(
"AmimeWatch running with Pyrogram v%s (Layer %s) started on @%s. Hi.",
__version__,
layer,
self.me.username,
)
loop = asyncio.get_event_loop()
langs.load()
modules.load(self)
self.video_queue = video_queue.VideoQueue(self, loop)
self.scrapper = scrapper.Scrapper()
self.day_releases = None
aiocron.crontab("0 * * * *",
func=backup.save,
args=(self, ),
start=True)
aiocron.crontab("*/10 * * * *",
func=day_releases.reload,
args=(self, ),
start=True)
pool = concurrent.futures.ThreadPoolExecutor(max_workers=self.workers -
4)
future = loop.run_in_executor(
pool, asyncio.ensure_future(day_releases.load(self)))
await asyncio.gather(future, return_exceptions=True)
def __init__(self, client):
_Cog.__init__(self, client)
tz = timezone('US/Eastern')
self.redis = redis_access(url=config.redis_url, params=config.redis_params)
self.engine = engine(url=config.postgres_url, params=config.postgres_params)
self.df = sql_to_df('last_message', self.engine, 'user_id')
self.deverify_days = sql_to_df('deverify_days', self.engine, 'index')
aiocron.crontab('0 3 * * *', func=self.cronjobs, tz=tz)
def __init__(self, client):
self.bot = client
self._announcements = [
Announcement(1, 45, get_media_path("attention-attention.mp3")),
Announcement(2, 0, get_media_path("attention-attention-2.mp3")),
]
for anc in self._announcements:
aiocron.crontab(f"{anc.minute} {anc.hour} * * *", func=self.attention, args=[anc.audio_file], start=True)
def init_requirements(self, loop):
super().init_requirements(loop)
periodic_db_saver = crontab(settings.PERIODIC_DB_SAVE_CRONTAB,
self._periodic_dbs_save,
start=False)
periodic_db_saver.start()
self.app["jsonrpc_dispatcher"] = get_jsonrpc_dispatcher()
def poll(self):
if not self.poll_schedule:
raise ManagerError("poll_schedule is not defined")
src_build_config = mongo.get_src_build_config()
@asyncio.coroutine
def check_pending_to_build():
confs = [
src['_id']
for src in src_build_config.find({'pending_to_build': True})
if type(src['_id']) == str
]
logging.info("Found %d configuration(s) to build (%s)" %
(len(confs), repr(confs)))
for conf_name in confs:
logging.info("Launch build for '%s'" % conf_name)
try:
self.merge(conf_name)
except Exception as e:
import traceback
logging.error(
"Build for configuration '%s' failed: %s\n%s" %
(conf_name, e, traceback.format_exc()))
raise
cron = aiocron.crontab(self.poll_schedule,
func=partial(check_pending_to_build),
start=True,
loop=self.job_manager.loop)
def __init__(self, player_service, game_stats_service):
self._dirty_games = set()
self._dirty_queues = set()
self.player_service = player_service
self.game_stats_service = game_stats_service
self.game_id_counter = 0
# Populated below in really_update_static_ish_data.
self.featured_mods = dict()
# A set of mod ids that are allowed in ranked games (everyone loves caching)
self.ranked_mods = set()
# The ladder map pool. Each entry is an (id, name, filename) tuple.
self.ladder_maps = set()
# Temporary proxy for the ladder service
self.ladder_service = None
# The set of active games
self.games = dict()
# Cached versions for files by game_mode ( featured mod name )
# For use by the patcher
self.game_mode_versions = dict()
# Synchronously initialise the game-id counter and static-ish-data.
loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.async(self.initialise_game_counter()))
loop.run_until_complete(loop.create_task(self.update_data()))
self._update_cron = aiocron.crontab('0 * * * *', func=self.update_data)
def __init__(self):
self.file_path = config.GEO_IP_DATABASE_PATH
self.db = None
# crontab: min hour day month day_of_week
# Run every Wednesday because GeoLite2 is updated every Tuesday
self._update_cron = aiocron.crontab('0 0 0 * * 3', func=self.check_update_geoip_db)
asyncio.ensure_future(self.check_update_geoip_db())
async def initialize(self) -> None:
await self.initialise_game_counter()
await self.update_data()
self._update_cron = aiocron.crontab("*/10 * * * *",
func=self.update_data)
await self._message_queue_service.declare_exchange(
config.MQ_EXCHANGE_NAME)
def __init__(self, *args, bigquery, **kwargs):
self.bigquery = bigquery
super().__init__(*args, **kwargs)
self.ensurer = crontab(
"* * * * * */15",
func=self._ensure_sender,
loop=self.loop,
)
def __init__(self, *args, bigquery, **kwargs):
self.bigquery = bigquery
super().__init__(*args, **kwargs)
self.ensurer = crontab(
"* * * * * */15",
func=self._ensure_sender,
loop=self.loop,
)
async def initialize(self) -> None:
await self.check_update_geoip_db()
# crontab: min hour day month day_of_week
# Run every Wednesday because GeoLite2 is updated every first Tuesday
# of the month.
self._update_cron = aiocron.crontab("0 0 0 * * 3",
func=self.check_update_geoip_db)
self._check_file_timer = Timer(60 * 10,
self.check_geoip_db_file_updated,
start=True)
def start_ssh_server(loop,name,passwords,keys=['bin/ssh_host_key'],shell=None,
host='',port=8022):
for key in keys:
assert os.path.exists(key),"Missing key '%s' (use: 'ssh-keygen -f %s' to generate it" % (key,key)
HubSSHServer.PASSWORDS = passwords
HubSSHServer.NAME = name
HubSSHServer.SHELL = shell
cron = aiocron.crontab(HUB_REFRESH_COMMANDS,func=shell.__class__.refresh_commands,
start=True, loop=loop)
yield from asyncssh.create_server(HubSSHServer, host, port, loop=loop,
server_host_keys=keys)
def _set_origin_processing_crontab(self, airline_name: str, origin_name: str, destination: str, crontab: str):
key = "{}_{}_{}".format(airline_name, origin_name, destination)
if key in self.timetable:
self.timetable[key].stop()
processor = partial(self._process_origin, airline_name, origin_name, destination)
schedule_executor = aiocron.crontab(crontab, processor, start=True)
logger.debug(
"Scheduling airline=%s, origin=%s, destination=%s at '%s'",
airline_name, origin_name, destination, crontab
)
self.timetable["key"] = schedule_executor
def test_null_callback():
loop = asyncio.new_event_loop()
t = crontab('* * * * * *', loop=loop)
assert t.handle is None # not started
future = asyncio.async(t.next(4), loop=loop)
loop.run_until_complete(future)
assert future.result() == (4,)
def test_null_callback():
loop = asyncio.new_event_loop()
t = crontab('* * * * * *', loop=loop)
assert t.handle is None # not started
future = asyncio. async (t.next(4), loop=loop)
loop.run_until_complete(future)
assert future.result() == (4, )
async def initialize(self) -> None:
if self._task is not None:
self._logger.error(
"Service already runnning or not properly shut down.")
return
await self.update_data()
self._update_cron = aiocron.crontab("*/10 * * * *",
func=self.update_data)
self._accept_input = True
self._logger.debug("RatingService starting...")
self._task = asyncio.create_task(self._handle_rating_queue())
def schedule_script(self, name, script_content, crontab_definition=None):
if name not in self.scheduled_scripts:
script_coro = partial(self.run_script, name, script_content)
self.scheduled_scripts[name] = {
"task":
self.loop.create_task(script_coro()),
"aiocron_instance":
aiocron.crontab(crontab_definition)
if crontab_definition else None,
}
async def on_ready(self):
logger.info("Logged in as '%s'" % self.user.name)
logger.info("Client id: %s" % self.user.id)
if self.settings.rank_enabled:
logger.info('Start cronjob with settings: {}'.format(
self.settings.rank_cron_string))
cron = aiocron.crontab(self.settings.rank_cron_string,
func=self.cronjob_print_rank_in_channel,
start=False)
cron.start()
self.start_recording_activities()
def test_next():
loop = asyncio.new_event_loop()
def t():
return 1
t = crontab('* * * * * *', func=t, loop=loop)
future = asyncio. async (t.next(), loop=loop)
loop.run_until_complete(future)
assert future.result() == 1
def __init__(self, service, family=None):
super().__init__(service, family)
if not self.interval:
raise ChParameterError("interval= property missing, required for cron service '{0}'".format(self.name))
# Support specials with or without the @
real_interval = _CRON_SPECIALS.get(self.interval) or _CRON_SPECIALS.get('@'+self.interval) or self.interval
# make a status note
self.note = "{0} ({1})".format(self.interval, real_interval) if self.interval != real_interval else real_interval
self._cron = crontab(real_interval, func=self._cron_hit, start=False)
def test_next():
loop = asyncio.new_event_loop()
def t():
return 1
t = crontab('* * * * * *', func=t, loop=loop)
future = asyncio.async(t.next(), loop=loop)
loop.run_until_complete(future)
assert future.result() == 1
def __init__(self, bot):
self.bot = bot
self.bg_task = self.bot.loop.create_task(self.contest())
self._update_cron = aiocron.crontab('1 0 * * *',
func=self.contest,
start=True)
# Make update command to check values in contesthistory.json later.
self.contest_history = dataIO.load_json('contests/contesthistory.json')
self.contests = dataIO.load_json('contests/contests.json')
if not self.contest_history:
self.contest_history = {"contests": []}
def _set_origin_processing_crontab(self, airline_name: str,
origin_name: str, destination: str,
crontab: str):
key = "{}_{}_{}".format(airline_name, origin_name, destination)
if key in self.timetable:
self.timetable[key].stop()
processor = partial(self._process_origin, airline_name, origin_name,
destination)
schedule_executor = aiocron.crontab(crontab, processor, start=True)
logger.debug(
"Scheduling airline=%s, origin=%s, destination=%s at '%s'",
airline_name, origin_name, destination, crontab)
self.timetable["key"] = schedule_executor
def __init__(self):
self.players = dict()
# Static-ish data fields.
self.privileged_users = {}
self.uniqueid_exempt = {}
self.client_version_info = ('0.0.0', None)
self._dirty_players = set()
asyncio.get_event_loop().run_until_complete(
asyncio.ensure_future(self.update_data()))
self._update_cron = aiocron.crontab('*/10 * * * *',
func=self.update_data)
def __init__(self, cog, crontab="*/1 * * * *"):
self.name = cog.qualified_name
self.cog = cog
self.bot = self.cog.bot
self.redis = self.bot.redis
self.crontab = crontab
self.start_listeners()
self.cron = aiocron.crontab(self.crontab, func=self.watch, start=False)
@self.bot.listen()
async def on_ready():
self.cron.start()
def __init__(self, db_pool: aiomysql.Pool):
self.players = dict()
self.db_pool = db_pool
# Static-ish data fields.
self.privileged_users = {}
self.uniqueid_exempt = {}
self.client_version_info = ('0.0.0', None)
self.blacklisted_email_domains = {}
self._dirty_players = set()
self.ladder_queue = None
asyncio.get_event_loop().run_until_complete(asyncio.async(self.update_data()))
self._update_cron = aiocron.crontab('*/10 * * * *', func=self.update_data)
async def search(
q: str = Query(None, max_length=280),
page: Optional[int] = Query(
None,
ge=eval(cfg["search"]["pg_range"])["ge"],
le=eval(cfg["search"]["pg_range"])["le"],
),
):
return Search()._query(q, page)
# Re-queries and populates database at scheduled time
# Use cron expression to set refresh rate
@aiocron.crontab(cfg["CRAWLER"]["refresh_rate"])
def poll(self):
if not self.poll_schedule:
raise ManagerError("poll_schedule is not defined")
src_dump = get_src_dump()
@asyncio.coroutine
def check_pending_to_upload():
sources = [src['_id'] for src in src_dump.find({'pending_to_upload': True}) if type(src['_id']) == str]
logging.info("Found %d resources to upload (%s)" % (len(sources),repr(sources)))
for src_name in sources:
logging.info("Launch upload for '%s'" % src_name)
try:
self.upload_src(src_name)
except ResourceNotFound:
logging.error("Resource '%s' needs upload but is not registered in manager" % src_name)
cron = aiocron.crontab(self.poll_schedule,func=partial(check_pending_to_upload),
start=True, loop=self.job_manager.loop)
def main():
cron = crontab('* * * * * */2')
for i in range(3):
try:
yield from cron.next()
except Exception:
pass
else:
print('yielded (%i)' % i)
for i in range(3):
try:
res = yield from mycron2.next(i)
except Exception as e:
print(repr(e))
else:
print(res)
def __init__(self, service, family=None):
super().__init__(service, family)
if not self.interval:
raise ChParameterError(
"interval= property missing, required for cron service '{0}'".
format(self.name))
# Support specials with or without the @
real_interval = _CRON_SPECIALS.get(
self.interval) or _CRON_SPECIALS.get(
'@' + self.interval) or self.interval
# make a status note
self.note = "{0} ({1})".format(
self.interval,
real_interval) if self.interval != real_interval else real_interval
self._cron = crontab(real_interval, func=self._cron_hit, start=False)
def submit(self, pfunc, schedule=None):
"""
Helper to submit and run tasks. Tasks will run async'ly.
pfunc is a functools.partial
schedule is a string representing a cron schedule, task will then be scheduled
accordingly.
"""
logger.info("Building task: %s" % pfunc)
if schedule:
logger.info("Scheduling task %s: %s" % (pfunc, schedule))
cron = aiocron.crontab(schedule,
func=pfunc,
start=True,
loop=self.loop)
return cron
else:
ff = asyncio.ensure_future(pfunc())
return ff
async def main(assets, logger):
app = App(assets, logger)
tasks = [
asyncio.create_task(app.redis_reader()),
asyncio.create_task(app.user_input())
]
cron = aiocron.crontab('*/1 * * * * 0', func=app.cron_job,
start=True) # every 1 min
# cron = aiocron.crontab('0 15 * * 1-5 0', func=app.cron_job, start=True) # 월-금, 15:00:00
# i = 0
# while True:
# i += 1
# res = await cron.next(i)
# await asyncio.gather(*tasks)
for t in asyncio.as_completed(tasks):
res = await t
cron.stop()
break
def create_app(*, github_token, github_payload_key, repo, redis_url,
loop=None):
app = web.Application(loop=loop)
app["repo"] = repo
app["github_token"] = github_token
app["github_payload_key"] = github_payload_key
app["redis.url"] = redis_url
app.on_startup.append(_create_redis_pool)
app.on_cleanup.append(_shutdown_redis_pool)
app.router.add_post("/hooks/news", news_hook)
app.router.add_post("/hooks/rebase", needs_rebase_hook)
app["cron.rebase.check_prs"] = crontab("*/15 * * * *",
functools.partial(check_prs, app),
loop=loop)
return app
def poll(self, state, func, col):
'''
Search for source in collection 'col' with a pending flag list
containing 'state' and and call 'func' for each document found
(with doc as only param)
'''
if not self.poll_schedule:
raise ManagerError("poll_schedule is not defined")
@asyncio.coroutine
def check_pending(state):
# sources = [src for src in col.find({'pending': state}) if type(src['_id']) == str] # TODO: remove this line
sources = [
src for src in col.find({'pending': state})
if isinstance(src['_id'], str)
]
if sources:
logger.info("Found %d resources with pending flag %s (%s)",
len(sources), state,
repr([src["_id"] for src in sources]))
for src in sources:
logger.info("Run %s for pending flag %s on source '%s'", func,
state, src["_id"])
try:
# first reset flag to make sure we won't call func multiple time
col.update({"_id": src["_id"]},
{"$pull": {
"pending": state
}})
func(src)
except ResourceNotFound:
logger.error(
"Resource '%s' has a pending flag set to %s but is not registered in manager",
src["_id"], state)
cron = aiocron.crontab(self.poll_schedule,
func=partial(check_pending, state),
start=True,
loop=self.job_manager.loop)
def main():
parser = argparse.ArgumentParser()
parser.prog = "python -m aiocron"
parser.add_argument(
"-n",
type=int,
default=1,
help="loop N times. 0 for infinite loop",
)
parser.add_argument("crontab", help='quoted crontab. like "* * * * *"')
parser.add_argument("command", nargs='+', help="shell command to run")
args = parser.parse_args()
cron = args.crontab
try:
croniter(cron)
except ValueError:
parser.error('Invalid cron format')
cmd = args.command
loop = asyncio.get_event_loop()
def calback():
subprocess.call(cmd)
if args.n != 0:
cron.n -= 1
if not cron.n:
loop.stop()
cron = crontab(cron, func=calback)
cron.n = args.n
cron.start()
try:
loop.run_forever()
except:
pass
def schedule_ping_frequency(self): # pragma: no cover
"Send a ping message to slack every 20 seconds"
ping = crontab('* * * * * */20', func=self.send_ping, start=False)
ping.start()
def start(self):
super().start()
crontab(self.cron, func=self.tic, start=True)
def crontab(self, *args, **kwargs):
kwargs['loop'] = self.loop
return aiocron.crontab(*args, **kwargs)
def init_requirements(self, loop):
super().init_requirements(loop)
periodic_db_saver = crontab(settings.PERIODIC_DB_SAVE_CRONTAB, self._periodic_dbs_save, start=False)
periodic_db_saver.start()
self.app["jsonrpc_dispatcher"] = get_jsonrpc_dispatcher()
def schedule_timer_message(self):
"Print out the top five newly submitted HN stories every 5 minutes"
timer = crontab('*/5 * * * *', func=self.send_timer_message, start=False)
timer.start()
async def start(self):
await super().start()
crontab(self.cron, func=self.tic, start=True, loop=self.loop)