def main():
init_config() # 初始化配置
""" 基于quartz的定时任务调度器 """
scheduler = BlockingScheduler()
""" FIELD_NAMES = ('year', 'month', 'day', 'week', 'day_of_week', 'hour', 'minute', 'second')
没用过quartz的同学去了解下CRON表达式~文档如下:
https://apscheduler.readthedocs.io/en/stable/modules/triggers/cron.html#module-apscheduler.triggers.cron
"""
year = str(gol.get_value('year') or None)
month = str(gol.get_value('month') or None)
day = str(gol.get_value('day') or None)
week = str(gol.get_value('week') or None)
day_of_week = str(gol.get_value('day_of_week') or None)
hour = str(gol.get_value('hour') or None)
minute = str(gol.get_value('minute') or None)
second = str(gol.get_value('second') or None)
scheduler.add_job(auto_job,
'cron',
year=year,
month=month,
day=day,
week=week,
day_of_week=day_of_week,
hour=hour,
minute=minute,
second=second,
id='auto_job_id')
try:
print('程序开始!-------->Job running(Tips: Ctrl + c 终止程序)')
scheduler.start()
except (KeyboardInterrupt, SystemExit):
print('退出程序!')
scheduler.remove_job('auto_job_id')
def main():
# 基于quartz的定时任务调度器
scheduler = BlockingScheduler()
""" FIELD_NAMES = ('year', 'month', 'day', 'week', 'day_of_week', 'hour', 'minute', 'second') """
scheduler.add_job(auto_job, 'cron', second='0/2', id='my_job_id')
try:
scheduler.start()
except (KeyboardInterrupt, SystemExit):
scheduler.remove_job('my_job_id')
class Main(KytosNApp):
"""Main class of amlight/scheduler NApp.
This class is the entry point for this napp.
"""
def setup(self):
"""Replace the '__init__' method for the KytosNApp subclass.
The setup method is automatically called by the controller when your
application is loaded.
So, if you have any setup routine, insert it here.
"""
self._scheduler = BlockingScheduler()
def execute(self):
"""This method is executed right after the setup method execution.
You can also use this method in loop mode if you add to the above setup
method a line like the following example:
self.execute_as_loop(30) # 30-second interval.
"""
self._scheduler.start()
def shutdown(self):
"""This method is executed when your napp is unloaded.
If you have some cleanup procedure, insert it here.
"""
self._scheduler.shutdown()
@listen_to('amlight/scheduler.add_job')
def add_job(self, event):
"""Add a job to this scheduler."""
try:
job_id = event.content['id']
except KeyError:
log.error('Scheduled job must have an id')
return
func = event.content['func']
kwargs = event.content['kwargs']
self._scheduler.add_job(func, id=job_id, **kwargs)
@listen_to('amlight/scheduler.remove_job')
def remove_job(self, event):
"""Remove a job from this scheduler."""
try:
job_id = event.content['id']
except KeyError:
log.error('Scheduled job must have an id')
return
self._scheduler.remove_job(job_id)
class ActionManager:
def __init__(self):
self.actions = []
self.curAction = None
self.id = "ActionManager_id" + str(time.time())
self.sched = BlockingScheduler()
def destroy(self):
self.sched.remove_job(self.id)
self.sched.shutdown()
def start(self):
# self.sched.add_job(self.tick, 'interval', seconds=0.1, max_instances=10, id=self.id,
# args={self.swipe_speed})
self.sched.add_job(self.tick, 'interval', seconds=0.1, max_instances=1, id=self.id)
self.sched.start()
print("ActionManager::start.....")
def addAction(self,action):
self.actions.append(action)
def _delAction(self,action):
action.exit()
def tick(self):
if self.curAction:
if self.curAction.finished == True:
self.curAction.exit()
self.curAction = None
if self.curAction == None:
if len(self.actions) > 0:
self.curAction = self.actions.pop(0)
self.curAction.enter()
if self.curAction and self.curAction.running == True:
self.curAction.tick()
print("act mgr tick.....end")
class SchedulerTest:
def __init__(self):
self.scheduler = BlockingScheduler()
def test(self):
attend_time = {"id": 5, "times_id": "8"}
id = None
for i in range(10):
attend_time["id"] = i
attend_time["times_id"] = "{}".format(i)
id = attend_time.get("times_id")
self.scheduler.add_job(start_work_test,
'cron',
kwargs=attend_time,
second='*/3',
hour='*',
id=id)
self.scheduler.start()
def stop(self, id):
self.scheduler.remove_job(job_id=id)
except Exception, e:
print(e)
traceback.print_exc()
print "[Scheduled task DONE] Sent daily subscription"
logging.basicConfig()
scheduler = BlockingScheduler()
scheduler.add_executor('threadpool')
if 'SCHED_HOUR' in os.environ:
scheduler_hour = int(os.environ['SCHED_HOUR'])
else:
scheduler_hour = 20
if 'SCHED_MIN' in os.environ:
scheduler_min = int(os.environ['SCHED_MIN'])
else:
scheduler_min = 48
job = scheduler.add_job(send_daily_subscription, 'cron', hour=scheduler_hour, minute=scheduler_min, id='job1')
try:
print "[Scheduler started] Sends everyday at %s:%s"%(scheduler_hour, scheduler_min)
scheduler.start()
except (KeyboardInterrupt, SystemExit):
scheduler.remove_job('job1')
scheduler.shutdown()
print "Scheduler shutdown"
class ScheduleService():
@inject
def __init__(self, synchornizationService : SynchronizationService):
self.__synchronizationService = synchornizationService
self.__scheduler = BlockingScheduler()
self.id = "id"
self.hasJob = False
self.isRunning = False
def stop_job(self):
if self.hasJob:
self.__scheduler.remove_job(self.id)
self.hasJob = False
def get_schedule(self, string):
if string is None:
return None
def get_permitted_digits(index):
if index == 0:
return 4
elif index == 4:
return 1
else:
return 2
regex = lambda d: r"(\*|(\d{%d}(,|-)?))*" % d
qualifiers = string.split(' ')
if len(qualifiers) != 8:
return None
for i in range(0, len(qualifiers)):
if not re.match(regex(get_permitted_digits(i)), qualifiers[i]):
return None
return {
'year': qualifiers[0],
'month': qualifiers[1],
'week': qualifiers[2],
'day': qualifiers[3],
'day_of_week': qualifiers[4],
'hour': qualifiers[5],
'minute': qualifiers[6],
'second': qualifiers[7],
}
def start_job(self, schedule):
self.stop_job()
self.__scheduler.add_job(
self.__synchronizationService.synchronize,
'cron',
year = schedule['year'],
month=schedule['month'],
day= schedule['day'],
day_of_week = schedule['day_of_week'],
hour=schedule['hour'],
minute=schedule['minute'],
second=schedule['second'],
id = self.id)
self.hasJob = True
if not self.isRunning:
self.isRunning = True
self.__scheduler.start()
args=('一次性任务', ),
next_run_time=datetime.datetime.now() +
datetime.timedelta(seconds=12),
id='one_time_task')
scheduler.add_job(func=aps_test,
args=('循环任务', ),
trigger='interval',
seconds=3,
id='interval_task')
Starter(scheduler).start()
time.sleep(10)
# 暂停任务
scheduler.pause_job('cron_task')
# 删除任务
scheduler.remove_job('interval_task')
# 添加任务
scheduler.add_job(func=aps_test,
args=('待修改任务', ),
trigger='interval',
seconds=3,
id='else_task')
time.sleep(10)
# 唤醒任务
scheduler.resume_job('cron_task')
# 修改任务
scheduler.modify_job('else_task', args=('修改后任务', ))
time.sleep(10)
# 修改参数 对于同时修改任务和参数的需求，建议使用remove 和 add 进行
scheduler.reschedule_job('cron_task', trigger='cron', second='*/8')
time.sleep(10)
class Host:
def __init__(self,
event_queue_id,
event_queue_name,
policy_storage,
log_group=None,
metrics=None,
output_dir=None):
logging.basicConfig(level=logging.INFO, format='%(message)s')
log.info("Running Azure Cloud Custodian Self-Host")
load_resources()
self.session = local_session(Session)
# Load configuration
self.options = Host.build_options(output_dir, log_group, metrics)
self.policy_storage_uri = policy_storage
self.event_queue_name = event_queue_name
self.event_queue_id = event_queue_id
# Prepare storage bits
self.policy_blob_client = None
self.blob_cache = {}
self.queue_storage_account = self.prepare_queue_storage(
self.event_queue_id, self.event_queue_name)
self.queue_service = None
# Track required event subscription updates
self.require_event_update = False
# Policy cache and dictionary
self.policy_cache = tempfile.mkdtemp()
self.policies = {}
# Configure scheduler
self.scheduler = BlockingScheduler()
logging.getLogger('apscheduler.executors.default').setLevel(
logging.ERROR)
# Schedule recurring policy updates
self.scheduler.add_job(self.update_policies,
'interval',
seconds=policy_update_seconds,
id="update_policies",
next_run_time=datetime.now())
# Schedule recurring queue polling
self.scheduler.add_job(self.poll_queue,
'interval',
seconds=queue_poll_seconds,
id="poll_queue")
self.scheduler.start()
def update_policies(self):
"""
Enumerate all policies from storage.
Use the MD5 hashes in the enumerated policies
and a local dictionary to decide if we should
bother downloading/updating each blob.
We maintain an on-disk policy cache for future
features.
"""
if not self.policy_blob_client:
self.policy_blob_client = Storage.get_blob_client_by_uri(
self.policy_storage_uri, self.session)
(client, container, prefix) = self.policy_blob_client
try:
# All blobs with YAML extension
blobs = [
b for b in client.list_blobs(container)
if Host.has_yaml_ext(b.name)
]
except AzureHttpError as e:
# If blob methods are failing don't keep
# a cached client
self.policy_blob_client = None
raise e
# Filter to hashes we have not seen before
new_blobs = self._get_new_blobs(blobs)
# Get all YAML files on disk that are no longer in blob storage
cached_policy_files = [
f for f in os.listdir(self.policy_cache) if Host.has_yaml_ext(f)
]
removed_files = [
f for f in cached_policy_files if f not in [b.name for b in blobs]
]
if not (removed_files or new_blobs):
return
# Update a copy so we don't interfere with
# iterations on other threads
policies_copy = self.policies.copy()
for f in removed_files:
path = os.path.join(self.policy_cache, f)
self.unload_policy_file(path, policies_copy)
# Get updated YML files
for blob in new_blobs:
policy_path = os.path.join(self.policy_cache, blob.name)
if os.path.exists(policy_path):
self.unload_policy_file(policy_path, policies_copy)
client.get_blob_to_path(container, blob.name, policy_path)
self.load_policy(policy_path, policies_copy)
self.blob_cache.update(
{blob.name: blob.properties.content_settings.content_md5})
# Assign our copy back over the original
self.policies = policies_copy
if self.require_event_update:
self.update_event_subscriptions()
def _get_new_blobs(self, blobs):
new_blobs = []
for blob in blobs:
md5_hash = blob.properties.content_settings.content_md5
if not md5_hash:
blob, md5_hash = self._try_create_md5_content_hash(blob)
if blob and md5_hash and md5_hash != self.blob_cache.get(
blob.name):
new_blobs.append(blob)
return new_blobs
def _try_create_md5_content_hash(self, blob):
# Not all storage clients provide the md5 hash when uploading a file
# so, we need to make sure that hash exists.
(client, container, _) = self.policy_blob_client
log.info("Applying md5 content hash to policy {}".format(blob.name))
try:
# Get the blob contents
blob_bytes = client.get_blob_to_bytes(container, blob.name)
# Re-upload the blob. validate_content ensures that the md5 hash is created
client.create_blob_from_bytes(container,
blob.name,
blob_bytes.content,
validate_content=True)
# Re-fetch the blob with the new hash
hashed_blob = client.get_blob_properties(container, blob.name)
return hashed_blob, hashed_blob.properties.content_settings.content_md5
except AzureHttpError as e:
log.warning("Failed to apply a md5 content hash to policy {}. "
"This policy will be skipped.".format(blob.name))
log.error(e)
return None, None
def load_policy(self, path, policies):
"""
Loads a YAML file and prompts scheduling updates
:param path: Path to YAML file on disk
:param policies: Dictionary of policies to update
"""
with open(path, "r") as stream:
try:
policy_config = yaml.safe_load(stream)
new_policies = PolicyCollection.from_data(
policy_config, self.options)
if new_policies:
for p in new_policies:
log.info("Loading Policy %s from %s" % (p.name, path))
p.validate()
policies.update({p.name: {'policy': p}})
# Update periodic and set event update flag
policy_mode = p.data.get('mode', {}).get('type')
if policy_mode == CONTAINER_TIME_TRIGGER_MODE:
self.update_periodic(p)
elif policy_mode == CONTAINER_EVENT_TRIGGER_MODE:
self.require_event_update = True
else:
log.warning(
"Unsupported policy mode for Azure Container Host: {}. "
"{} will not be run. "
"Supported policy modes include \"{}\" and \"{}\"."
.format(policy_mode, p.data['name'],
CONTAINER_EVENT_TRIGGER_MODE,
CONTAINER_TIME_TRIGGER_MODE))
except Exception as exc:
log.error('Invalid policy file %s %s' % (path, exc))
def unload_policy_file(self, path, policies):
"""
Unload a policy file that has changed or been removed.
Take the copy from disk and pop all policies from dictionary
and update scheduled jobs and event registrations.
"""
with open(path, "r") as stream:
try:
policy_config = yaml.safe_load(stream)
except yaml.YAMLError as exc:
log.warning('Failure loading cached policy for cleanup %s %s' %
(path, exc))
os.unlink(path)
return
removed = [
policies.pop(p['name']) for p in policy_config.get('policies', [])
]
log.info('Removing policies %s' % removed)
# update periodic
periodic_names = \
[p['name'] for p in policy_config['policies'] if p.get('mode', {}).get('schedule')]
periodic_to_remove = \
[p for p in periodic_names if p in [j.id for j in self.scheduler.get_jobs()]]
for name in periodic_to_remove:
self.scheduler.remove_job(job_id=name)
# update event
event_names = \
[p['name'] for p in policy_config['policies'] if p.get('mode', {}).get('events')]
if event_names:
self.require_event_update = True
os.unlink(path)
return path
def update_periodic(self, policy):
"""
Update scheduled policies using cron type
periodic scheduling.
"""
trigger = CronTrigger.from_crontab(policy.data['mode']['schedule'])
trigger.jitter = jitter_seconds
self.scheduler.add_job(self.run_policy,
trigger,
id=policy.name,
name=policy.name,
args=[policy, None, None],
coalesce=True,
max_instances=1,
replace_existing=True,
misfire_grace_time=20)
def update_event_subscriptions(self):
"""
Find unique list of all subscribed events and
update a single event subscription to channel
them to an Azure Queue.
"""
log.info('Updating event grid subscriptions')
destination = \
StorageQueueEventSubscriptionDestination(resource_id=self.queue_storage_account.id,
queue_name=self.event_queue_name)
# Get total unique event list to use in event subscription
policy_items = self.policies.items()
events_lists = [
v['policy'].data.get('mode', {}).get('events')
for n, v in policy_items
]
flat_events = [e for l in events_lists if l for e in l if e]
resolved_events = AzureEvents.get_event_operations(flat_events)
unique_events = set(resolved_events)
# Build event filter strings
advance_filter = StringInAdvancedFilter(key='Data.OperationName',
values=unique_events)
event_filter = EventSubscriptionFilter(
advanced_filters=[advance_filter])
# Update event subscription
AzureEventSubscription.create(destination, self.event_queue_name,
self.session.get_subscription_id(),
self.session, event_filter)
self.require_event_update = False
def poll_queue(self):
"""
Poll the Azure queue and loop until
there are no visible messages remaining.
"""
# Exit if we don't have any policies
if not self.policies:
return
if not self.queue_service:
self.queue_service = Storage.get_queue_client_by_storage_account(
self.queue_storage_account, self.session)
while True:
try:
messages = Storage.get_queue_messages(
self.queue_service,
self.event_queue_name,
num_messages=queue_message_count,
visibility_timeout=queue_timeout_seconds)
except AzureHttpError:
self.queue_service = None
raise
if len(messages) == 0:
break
log.info('Pulled %s events to process while polling queue.' %
len(messages))
for message in messages:
if message.dequeue_count > max_dequeue_count:
Storage.delete_queue_message(self.queue_service,
self.event_queue_name,
message=message)
log.warning(
"Event deleted due to reaching maximum retry count.")
else:
# Run matching policies
self.run_policies_for_event(message)
# We delete events regardless of policy result
Storage.delete_queue_message(self.queue_service,
self.event_queue_name,
message=message)
def run_policies_for_event(self, message):
"""
Find all policies subscribed to this event type
and schedule them for immediate execution.
"""
# Load up the event
event = json.loads(base64.b64decode(message.content).decode('utf-8'))
operation_name = event['data']['operationName']
# Execute all policies matching the event type
for k, v in self.policies.items():
events = v['policy'].data.get('mode', {}).get('events')
if not events:
continue
events = AzureEvents.get_event_operations(events)
if operation_name in events:
self.scheduler.add_job(self.run_policy,
id=k + event['id'],
name=k,
args=[v['policy'], event, None],
misfire_grace_time=60 * 3)
def run_policy(self, policy, event, context):
try:
policy.push(event, context)
except Exception as e:
log.error("Exception running policy: %s error: %s", policy.name, e)
def prepare_queue_storage(self, queue_resource_id, queue_name):
"""
Create a storage client using unusual ID/group reference
as this is what we require for event subscriptions
"""
# Use a different session object if the queue is in a different subscription
queue_subscription_id = ResourceIdParser.get_subscription_id(
queue_resource_id)
if queue_subscription_id != self.session.subscription_id:
session = Session(queue_subscription_id)
else:
session = self.session
storage_client = session.client(
'azure.mgmt.storage.StorageManagementClient')
account = storage_client.storage_accounts.get_properties(
ResourceIdParser.get_resource_group(queue_resource_id),
ResourceIdParser.get_resource_name(queue_resource_id))
Storage.create_queue_from_storage_account(account, queue_name,
self.session)
return account
@staticmethod
def build_options(output_dir=None, log_group=None, metrics=None):
"""
Initialize the Azure provider to apply global config across all policy executions.
"""
if not output_dir:
output_dir = tempfile.mkdtemp()
log.warning(
'Output directory not specified. Using directory: %s' %
output_dir)
config = Config.empty(**{
'log_group': log_group,
'metrics': metrics,
'output_dir': output_dir
})
return Azure().initialize(config)
@staticmethod
def has_yaml_ext(filename):
return filename.lower().endswith(('.yml', '.yaml'))
class JobScheduler(metaclass=Singleton):
__function_dict__ = {}
def __init__(self):
self.logger = Log().logger
self.scheduler = BlockingScheduler(logger=self.logger)
self.parser = DateTimeParser()
def get_lives(self):
return {job.name: job for job in self.scheduler.get_jobs()}
def add_job(self, name, trigger, trigger_args, func_args=None, func_kwargs=None):
if name not in self.get_lives():
func = self.get_functions(name)
self.scheduler.add_job(func, trigger=trigger, args=func_args, kwargs=func_kwargs,
id=name, name=name, **trigger_args)
self.logger.info('[%s][%s]作业已添加，并已启动' % (trigger, name))
def add_date_job(self, name, time, func_args=None, func_kwargs=None):
time = self.parser.set_date(time).set_time(time).datetime
self.add_job(name, 'date', {'run_date': time}, func_args, func_kwargs)
def add_interval_job(self, name, weeks=0, days=0, hours=0, minutes=0, seconds=0,
func_args=None, func_kwargs=None):
self.add_job(name, 'interval',
{'weeks': weeks, 'days': days,
'hours': hours, 'minutes': minutes, 'seconds': seconds},
func_args, func_kwargs)
def add_cron_job(self, name, year=None, month=None, day=None, week=None,
day_of_week=None, hour=None, minute=None, second=None,
func_args=None, func_kwargs=None):
self.add_job(name, 'cron',
{'year': year, 'month': month, 'day': day, 'week': week,
'day_of_week': day_of_week,
'hour': hour, 'minute': minute, 'second': second},
func_args, func_kwargs)
def add_started_job(self, name, after_seconds=1, func_args=None, func_kwargs=None):
time = datetime.now() + timedelta(seconds=after_seconds)
self.add_date_job(name, time, func_args, func_kwargs)
def delete_job(self, name):
jobs = self.get_lives()
if name in jobs:
self.scheduler.remove_job(jobs[name].id)
self.logger.info('作业[%s]已移除' % name)
@classmethod
def register(cls, name):
def add_method(f):
cls.__function_dict__[name.strip()] = f
return f
return add_method
def get_functions(self, name):
return self.__function_dict__.get(name.strip())
def get_function_doc(self, name):
return self.get_functions(name).__doc__
def get_function_names(self):
return sorted(self.__function_dict__.keys())
data_fr = FR.find_one(sort=[('_id', 1)])
os.system('scrapy crawl FR')
data_FR = FR.find_one(sort=[('_id', 1)])
time_FR = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
print(time_FR, ':已检查FR数据')
if data_fr != data_FR:
date = time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time()))
print(date, ':FR数据有更新，已发送邮件')
collection = 'FR'
send_email(collection)
ECB = db['ECB']
data_ecb = ECB.find_one(sort=[('_id', 1)])
os.system('scrapy crawl ecb')
data_ECB = ECB.find_one(sort=[('_id', 1)])
time_ECB = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
print(time_ECB, ':已检查ECB数据')
if data_ecb != data_ECB:
date = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
print(date, ':ECB数据有更新，已发送邮件')
collection = 'ECB'
send_email(collection)
sched = BlockingScheduler()
sched.add_job(job_1, 'interval', seconds=30, id='job_1')
try:
sched.start()
except Exception as e:
print('scheduler error:', e)
sched.remove_job('job_1')
class abstract_schedule(metaclass=ABCMeta):
def __init__(self):
self.sched = BlockingScheduler()
def lstCronJob(self, job_id=None):
result = {}
if not job_id:
jobs = self.sched.get_jobs()
for j in jobs:
result[j.id] = j
else:
jobs = self.sched.get_job(job_id)
result[job_id] = jobs
return result
def delCronJob(self, job_id):
jobs = self.lstCronJob(job_id)
if not jobs:
sys.stdout.write("Job %s not found" %job_id)
else:
self.sched.remove_job(job_id)
sys.stdout.write("Job %s 删除成功!"%job_id)
return True
def addCronJob(self, job_id, func, policy, args):
cron = CronTrigger(**policy)
self.sched.add_job(func, cron, args=args, id=job_id)
def start(self):
print("123123")
self.sched.add_job(self.autoAddJob, IntervalTrigger(seconds=5), id="autoAddJob")
self.sched.start()
def autoAddJob(self):
history_jobs = self.lstCronJob()
print(history_jobs, 'history_jobs')
current_jobs = self.getBackupPolicy()
print(current_jobs, 'current_jobs')
only_current_jobs = set(current_jobs.keys()).difference(set(history_jobs.keys()))
print(only_current_jobs, 'only_current_jobs')
# 当前任务调度列表中有的 历史任务列表中没有的
only_history_jobs = set(history_jobs.keys()).difference(set(current_jobs.keys()))
print(only_history_jobs, 'only_history_jobs')
#历史任务中有的当前任务列表中没有的任务
#
for j in only_history_jobs:
if j == 'autoAddJob':
continue
self.delCronJob(job_id=j)
for j in only_current_jobs:
func = current_jobs[j].pop('func')
args = current_jobs[j].pop('args')
policy = current_jobs[j]
self.addCronJob(job_id=j, func=func, policy=policy, args=args)
@abstractmethod
def getBackupPolicy(self):
pass
# #power rankings: tuesday evening at 6:30pm local time.
# #matchups: thursday evening at 7:30pm east coast time.
# #close scores (within 15.99 points): monday evening at 6:30pm east coast time.
# #trophies: tuesday morning at 7:30am local time.
# #score update: friday, monday, and tuesday morning at 7:30am local time.
# #score update: sunday at 4pm, 8pm east coast time.
# sched.add_job(bot_main, 'cron', ['get_power_rankings'], id='power_rankings',
# day_of_week='tue', hour=18, minute=30, start_date=ff_start_date, end_date=ff_end_date,
# timezone=my_timezone, replace_existing=True)
# sched.add_job(bot_main, 'cron', ['get_matchups'], id='matchups',
# day_of_week='thu', hour=19, minute=30, start_date=ff_start_date, end_date=ff_end_date,
# timezone=game_timezone, replace_existing=True)
# sched.add_job(bot_main, 'cron', ['get_close_scores'], id='close_scores',
# day_of_week='mon', hour=18, minute=30, start_date=ff_start_date, end_date=ff_end_date,
# timezone=game_timezone, replace_existing=True)
# sched.add_job(bot_main, 'cron', ['get_final'], id='final',
# day_of_week='tue', hour=7, minute=30, start_date=ff_start_date, end_date=ff_end_date,
# timezone=my_timezone, replace_existing=True)
# sched.add_job(bot_main, 'cron', ['get_scoreboard_short'], id='scoreboard1',
# day_of_week='fri,mon', hour=7, minute=30, start_date=ff_start_date, end_date=ff_end_date,
# timezone=my_timezone, replace_existing=True)
# sched.add_job(bot_main, 'cron', ['random_phrase'], id='random_phrase',
# day_of_week='sun', hour='15,42', start_date=ff_start_date, end_date=ff_end_date,
# timezone=game_timezone, replace_existing=True)
sched.remove_job("random_phrase")
print("Ready!")
sched.start()
class SchedUtility(object, metaclass=Singleton):
def __init__(self):
try:
self.Global = Global()
self.Utility = Utility()
self.InfraUtil = InfraUtility()
self.db = DBMySql('Scheduler')
self.myModulePyFile = os.path.abspath(__file__)
self.myClass = self.__class__.__name__
#Setting the infrastructure
self.Infra = self.InfraUtil.setInfra(self.Global.SchedulerInfraKey)
if not self.Infra:
raise InfraInitializationError('Could not initialize {cls}'.format(cls=(self.myModulePyFile,self.myClass)))
# we need to get the proper logger for a given module
self.logger = self.Infra.getInfraLogger(self.Global.SchedulerInfraKey)
# loading Schduler config and starting scheduler
self.__startScheduler__()
except Exception as err:
raise err
def __startScheduler__(self):
try:
mySchedulerType = self.Global.DefaultSchedulerType
mySchedulerMode = self.Global.DefaultSchedulerMode
if mySchedulerMode == 'Run':
myArgPaused = False
else:
myArgPaused = True
#fi
mySchedulerConfig = self.Utility.getACopy(self.Infra.schedulerConfigData)
if mySchedulerType == 'Background':
self.Scheduler = BackgroundScheduler(mySchedulerConfig)
else:
self.Scheduler = BlockingScheduler(mySchedulerConfig)
#fi
if not self.Scheduler.running:
self.Scheduler.start(paused = myArgPaused)
except Exception as err:
raise err
def getAllJobDetail(self):
'''
Description: Returns all jobs as stored in scheduler
'''
myJobDetail = []
for job in self.Scheduler.get_jobs():
myJobDetail.append(self.getAJobDetail(job.id))
return myJobDetail
def getAJobDetail(self, jobIdArg):
'''
Description: Print all jobs as stored in scheduler
'''
myJobId = jobIdArg
job = self.Scheduler.get_job(myJobId)
myJobDetail = job.__getstate__()
return myJobDetail
def suspendJob(self, jobIdArg):
myJobId = jobIdArg
job = self.Scheduler.get_job(myJobId)
job.pause()
def resumeJob(self, jobIdArg):
myJobId = jobIdArg
job = self.Scheduler.get_job(myJobId)
job.resume()
def getCurrentlyExecutingJob(self):
return len(self.Scheduler.get_jobs())
def removeJob(self, jobId):
try:
self.Scheduler.remove_job(jobId)
except JobLookupError as err:
print('Invalid Job !!')
def removeAllJobs(self):
try:
self.Scheduler.remove_all_jobs()
except Exception as err:
raise err
def getAllJobsFromRep(self):
for job in self.Scheduler.get_jobs():
myJobDetail = self.Scheduler.get_job(job.id)
print(job,myJobDetail)
def getNewJob(self,prefixArg):
# random number between 10 and 99 to ensure we always get 2 digit
if isinstance(prefixArg,str) and prefixArg is not None:
return prefixArg + '_' + str(datetime.datetime.fromtimestamp(time.time()).strftime('%Y%m%d_%-H%M%S_') + str(random.randrange(10,99)))
else:
return datetime.datetime.fromtimestamp(time.time()).strftime('%Y%m%d_%-H%M%S_') + str(random.randrange(10,99))
def getJobInfoFromDb(self, jobIdArg):
try:
myResponse = self.Utility.getResponseTemplate()
myJobId = self.Utility.getACopy(jobIdArg)
self.logger.debug('arg [{arg}] received'.format(arg = myJobId))
myJobCriteria = 'JobId = %s ' %repr(myJobId)
return self.db.processDbRequest(operation = self.Global.fetch, container = 'ScheduledJobs', contents = ['*'], criteria = myJobCriteria)
except Exception as err:
myErrorMsg, myTraceback = self.Utility.getErrorTraceback()
self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myErrorMsg)
self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myTraceback)
self.Utility.buildResponse(myResponse, self.Global.UnSuccess,myErrorMsg)
return myResponse
def getNextSeqForJob(self, jobIdArg):
try:
myResponse = self.Utility.getResponseTemplate()
myJobId = self.Utility.getACopy(jobIdArg)
self.logger.debug('arg [{arg}] received'.format(arg = myJobId))
myJobCriteria = 'JobId = %s ' %repr(myJobId)
return self.db.getTotalRowCount(container = 'ScheduledJobsRunLog', criteria = myJobCriteria) + 1
except Exception as err:
myErrorMsg, myTraceback = self.Utility.getErrorTraceback()
self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myErrorMsg)
self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myTraceback)
return myErrorMsg
def getCurrentSeqForJob(self, jobIdArg):
try:
myResponse = self.Utility.getResponseTemplate()
myJobId = self.Utility.getACopy(jobIdArg)
self.logger.debug('arg [{arg}] received'.format(arg = myJobId))
myJobCriteria = 'JobId = %s ' %repr(myJobId)
return self.db.getTotalRowCount(container = 'ScheduledJobsRunLog', criteria = myJobCriteria)
except Exception as err:
myErrorMsg, myTraceback = self.Utility.getErrorTraceback()
self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myErrorMsg)
self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myTraceback)
return myErrorMsg
def getElapsedStatsForJob(self, jobIdArg):
try:
myResponse = self.Utility.getResponseTemplate()
myJobId = self.Utility.getACopy(jobIdArg)
self.logger.debug('arg [{arg}] received'.format(arg = myJobId))
myJobCriteria = 'JobId = %s ' %repr(myJobId)
return self.db.getTotalRowCount(container = 'ScheduledJobsRunLog', criteria = myJobCriteria)
except Exception as err:
myErrorMsg, myTraceback = self.Utility.getErrorTraceback()
self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myErrorMsg)
self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myTraceback)
return myErrorMsg
def processJobStartEvent(self, jobIdArg):
'''
1. Mark job started in ScheduledJobs
2. Create new entry for this job in ScheduledJobsRunLog
'''
try:
# initializing
myResponse = self.Utility.getResponseTemplate()
myJobId = self.Utility.getACopy(jobIdArg)
self.logger.debug('arg [{arg}] received'.format(arg=myJobId))
myJobDetailsFromDb = self.getJobInfoFromDb(myJobId)['Data']
if myJobDetailsFromDb:
# building data for SchedulerJobsRunLog
myJobCriteria = ' JobId = %s' %repr(myJobId)
myNextSeqForJob = self.getNextSeqForJob(myJobId)
# will mark the job started and creat the run log for this run
self.db.processDbRequest(operation='change', container='ScheduledJobs', \
dataDict={'Status': 'Executing'}, criteria = myJobCriteria, commitWork=True )
# creating run information
self.db.processDbRequest(operation='create', container='ScheduledJobsRunLog', \
dataDict={'JobId':myJobId, 'Seq' : myNextSeqForJob, 'ExecutionStarted': self.Utility.getCurrentTime()}, commitWork=True )
self.Utility.buildResponse(myResponse, self.Global.Success, self.Global.Success, {'Seq':myNextSeqForJob})
else:
self.Utility.buildResponse(myResponse, self.Global.UnSuccess, 'Cound not find job details for job {job}'.format(job = myJobId))
return myResponse
except Exception as err:
myErrorMsg, myTraceback = self.Utility.getErrorTraceback()
self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myErrorMsg)
self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myTraceback)
self.Utility.buildResponse(myResponse, self.Global.UnSuccess,myErrorMsg)
#raise err # will raise the error so this can be logged by scheduler as an error occurred in processing job
return myResponse
def processJobFinishEvent(self, jobIdArg, execDetailsArg):
'''
1. Mark job completed (update failure cnt and total count and consc fail count, lastrunstatus) in ScheduledJobs
2. Update ScheduledJobsRunlog container
'''
try:
# initializing
myResponse = self.Utility.getResponseTemplate()
myJobId = self.Utility.getACopy(jobIdArg)
myExecDetails = execDetailsArg
myJobStatus = self.Global.NextJobRun
self.logger.debug('arg [{arg}] received'.format(arg=myJobId))
myJobDetailsFromDb = self.getJobInfoFromDb(myJobId)['Data']
if myJobDetailsFromDb:
self.logger.debug('Job details found, proceeding with finish event')
myJobCriteria = 'JobId = %s' %repr(myJobId)
myCurrentSeqForJob = self.getCurrentSeqForJob(myJobId)
myJobRunCriteria = ' JobId = %s and Seq = %s ' %(repr(myJobId), myCurrentSeqForJob)
self.logger.debug('Job criteria {criteria}'.format(criteria = myJobCriteria))
self.logger.debug('Job criteria with seq {criteria}'.format(criteria = myJobRunCriteria))
myJobDetailsFromSched = self.getAJobDetail(myJobId)
# Updating execution details in ScheduledJobsRunLog
self.logger.debug('udating statistics of this run')
myDbResult = self.db.processDbRequest(operation = 'change', container = 'ScheduledJobsRunLog', \
dataDict={
'Status': myExecDetails['Status'], 'ElapsedSeconds':myExecDetails['Data']['ElapsedSecs'],
'ExecutionCompleted': self.Utility.getCurrentTime(), 'ExecutionDetail': json.dumps(myExecDetails['Data'])
}, criteria = myJobRunCriteria, commitWork=True )
self.logger.debug('ScheduledJobsRunLog: db results >> {results}'.format(results = myDbResult))
# Updating execution details in ScheduledJobs
#if myExecDetails['Status'] == self.Global.Success:
# if success, reset consecfailcnt to 0, increment totalrun by 1 and update next run
myElapsedStats = self.db.executeDynamicSql(\
operation = 'fetch', \
sql_text = 'select min(ElapsedSeconds) "Min", max(ElapsedSeconds) "Max", avg(ElapsedSeconds) "Avg" from ScheduledJobsRunLog')
self.logger.debug('Elapsed Stats: {stats}'.format(stats = myElapsedStats))
myDbResult = self.db.processDbRequest(operation='change', container='ScheduledJobs', \
dataDict={
'Status': myJobStatus, 'LastRunStatus': myExecDetails['Status'], 'TotalRun' : myJobDetailsFromDb[0]['TotalRun'] + 1,
'NextRun' : myJobDetailsFromSched['next_run_time'].strftime('%Y-%m-%d% %H:%M:%S'), 'LatConsecFailCnt' : 0,
'MinElapsedSecs' : myElapsedStats['Data'][0]['Min'], 'MaxElapsedSecs' : myElapsedStats['Data'][0]['Min'] ,
'AvgElapsedSecs' : myElapsedStats['Data'][0]['Avg']
}, criteria = myJobCriteria, commitWork=True )
self.logger.debug('ScheduledJobs: last stats update >> {result}'.format(result = myDbResult))
#self.Utility.buildResponse(myResponse, self.Global.Success,self.Global.Success)
'''
else:
# process job was unsuccessful
if myJobDetailsFromDb[0]['LatConsecFailCnt'] >= self.Global.SchedConsecFailCntThreshold:
myJobStatus = self.Global.SuspendMode
self.logger.info('suspending job {job}'.format(job=myJobId))
self.suspendJob(myJobId)
myDbResult = self.db.processDbRequest(operation='change', container='ScheduledJobs', \
dataDict={
'Status': myJobStatus, 'LastRunStatus': myExecDetails['Status'], 'TotalRun' : myJobDetailsFromDb[0]['TotalRun'] + 1,
'next_run' : myJobDetailsFromSched['next_run_time'], 'LatConsecFailCnt' : myJobDetailsFromDb[0]['LatConsecFailCnt'] + 1,
'TotalFailure' : myJobDetailsFromDb[0]['TotalFailure' + 1]
}, criteria = myJobCriteria, commitWork=True )
# will suspend the job if total failure count has been reached beyond Total consecutive failure threshold
self.Utility.buildResponse(myResponse, self.Global.UnSuccess,self.Global.UnSuccess)
raise processJobError(myExecDetails['Message'])
'''
self.Utility.buildResponse(myResponse, self.Global.Success,self.Global.Success)
return myResponse
except Exception as err:
myErrorMsg, myTraceback = self.Utility.getErrorTraceback()
self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myErrorMsg)
self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myTraceback)
self.Utility.buildResponse(myResponse, self.Global.UnSuccess, myErrorMsg)
return myResponse
class JobScheduler(object):
def __init__(self, every=30, unit='second'):
self.mongo = mongopool.get()
self.cursor = self.mongo.get_database('apscheduler').get_collection(
'jobs')
self.every = every
self.unit = unit
self.scheduler = BlockingScheduler(logger=logger)
self.scheduler.configure(jobstores=jobstores,
executors=executors,
job_defaults=job_defaults,
timezone=pytz.timezone('Asia/Saigon'))
self._set_trigger(every, unit)
def _set_trigger(self, every, unit):
now = datetime.now().astimezone(pytz.timezone('Asia/Saigon'))
if unit == 'second':
self.trigger = CronTrigger(second='*/{}'.format(every),
start_date=now)
elif unit == 'minute':
self.trigger = CronTrigger(minute='*/{}'.format(every),
start_date=now)
elif unit == 'hour':
self.trigger = CronTrigger(hour='*/{}'.format(every),
start_date=now)
elif unit == 'day':
self.trigger = CronTrigger(day='*/{}'.format(every),
start_date=now)
else:
raise Exception(message='Unknown time unit')
def add_jobstore(self, jobstore, alias):
self.scheduler.add_jobstore(jobstore, alias)
def add_executor(self, executor, alias):
self.scheduler.add_executor(executor, alias)
def add_job(self,
job_fn,
id='id1',
name='job1',
jobstore='default',
executor='default',
args=None,
kwargs=None):
now = datetime.now().astimezone(pytz.timezone('Asia/Saigon'))
history = list(self.cursor.find({'_id': id}))
if history:
#TODO: process missing jobs
self.cursor.delete_one({'_id': id})
next_run_time = self.trigger.get_next_fire_time(None, now)
if kwargs:
kwargs['run_time'] = next_run_time
else:
kwargs = {'run_time': next_run_time}
self.scheduler.add_job(job_fn,
trigger=self.trigger,
next_run_time=next_run_time,
id=id,
name=name,
jobstore=jobstore,
executor=executor,
args=args,
kwargs=kwargs)
def remove_job(self, id, jobstore='default'):
self.scheduler.remove_job(job_id=id, jobstore=jobstore)
def callback(self, callback_fn, mark=EVENT_ALL):
self.scheduler.add_listener(callback_fn)
def start(self):
mongopool.put(self.mongo)
self.scheduler.start()
def shutdown(self):
self.scheduler.shutdown()
self.scheduler.scheduled_job
class JobManage():
def __init__(self):
jobstores = {'default': MemoryJobStore()}
executors = {
'default': ThreadPoolExecutor(50)
# 'processpool': ProcessPoolExecutor(3)
}
job_defaults = {'coalesce': False, 'max_instances': 50}
self.sched = BlockingScheduler(jobstores=jobstores,
executors=executors,
job_defaults=job_defaults)
self.addError()
self.addJobExecuted()
def addJob(self, func, jobId=None, cron=None, args=[], kwargs={}):
'''
只支持cron的形式
* * * * * command
分 时 日 月 周 命令
第1列表示分钟1～59 每分钟用*或者 */1表示
第2列表示小时1～23（0表示0点）
第3列表示日期1～31
第4列表示月份1～12
第5列标识号星期0～6（0表示星期天）
第6列要运行的命令
'''
if cron is None:
raise Exception("cron cannot be Null")
(minute, hour, day, month, week) = cron.split(" ")
self.sched.add_job(func,
trigger='cron',
id=jobId,
hour=hour,
minute=minute,
day=day,
month=month,
week=week,
args=args,
kwargs=kwargs)
def removeJob(self, jobId):
self.sched.remove_job(jobId)
def start(self):
self.sched.start()
def shutdown(self):
self.sched.shutdown()
def printJobs(self):
self.sched.print_jobs()
def getJobs(self):
return self.sched.get_jobs()
def addError(self, func=None):
if func is None:
func = self.listener
self.sched.add_listener(func, EVENT_JOB_ERROR)
def addJobExecuted(self, func=None):
if func is None:
func = self.listener
self.sched.add_listener(func, EVENT_JOB_EXECUTED)
def listener(self, event):
if event.exception:
log.error("任务【%s】 任务出错 : %s" % (event.job_id, event.traceback))
else:
log.debug("任务【%s】已经跑完，结束时间 : %s " % (event.job_id, getNow()))
# jobMange = JobManage()
# print(time.time())
#
#
# sched.start()
# 获得job列表
# def my_job():
# print(time.time())
#
#
# sched = BlockingScheduler()
# job = sched.add_job(my_job, 'interval', seconds=2, id='123')
# print(sched.get_job(job_id='123'))
# print(sched.get_jobs())
# 移除作业
sched = BlockingScheduler()
def my_job():
print(time.time())
job = sched.add_job(my_job, 'interval', seconds=2, id='123')
sched.add_job(my_job, 'interval', minutes=2, id='my_job_id')
print(sched.get_jobs())
sched.remove_job('my_job_id')
print(sched.get_jobs())
job.remove()
print(sched.get_jobs())
class eagle_eye_bot(object):
def __init__(self, stock_list=[]):
self._scheduler = BlockingScheduler()
self.stock_list = stock_list
self._money_flows = {}
self._start_time = {}
for stock in stock_list:
self._money_flows[stock] = money_flow_level()
self._start_time[stock] = '00:00:00'
def start(self):
self._scheduler.add_job(self._start_monitor_job,
'cron',
day_of_week='mon-fri',
hour=9,
minute=29,
second=58) # 启动监控9:29:58,先获取集合竞价的数据
self._scheduler.add_job(self._end_monitor_job,
'cron',
day_of_week='mon-fri',
hour=11,
minute=30,
second=5) # 结束上午的监控11:30:05
self._scheduler.add_job(self._start_monitor_job,
'cron',
day_of_week='mon-fri',
hour=13,
minute=0,
second=0) # 启动下午的监控
self._scheduler.add_job(self._end_monitor_job,
'cron',
day_of_week='mon-fri',
hour=15,
minute=0,
second=5) # 结束下午的监控15:00:05
try:
self._scheduler.start()
print('start to monitor tick data...')
except (KeyboardInterrupt, SystemExit):
self._scheduler.remove_all_jobs()
def _start_monitor_job(self):
self._scheduler.add_job(self.__stock_monitor_on_second,
'interval',
seconds=3,
id='monitor_tick')
def _end_monitor_job(self):
self._scheduler.remove_job(job_id='monitor_tick')
def add_stock(self, stock_code, levels=[]):
"""
add code to be monitored
:param stock_code:
:return:
"""
levels = fit_levels(levels)
self.stock_list.append(stock_code)
self._money_flows[stock_code] = money_flow_level(
levels[0], levels[1], levels[2], levels[3])
self._start_time[stock_code] = '00:00:00'
def remove_stock(self, stock_code):
"""
:param stock_code:
:return:
"""
self.stock_list.remove(stock_code)
del self._money_flows[stock_code]
del self._start_time[stock_code]
def get_stock_money_flow(self, stock_code):
"""
:param stock_code:
:return:
"""
return self._money_flows[stock_code]
def __stock_monitor_on_second(self): # 定时获取tick数据处理
data = ts.get_realtime_quotes(self.stock_list)
for _, item in data.iterrows():
if item['time'] != self._start_time[item['code']]: # 是否是新成交的数据
self.__add_tick(item)
self._start_time[item['code']] = item['time']
def __add_tick(self, tick_data):
price = float(tick_data['price'])
if price <= float(tick_data['ask']): # 小于等于买一价，卖盘
tick_type = -1
elif price >= float(tick_data['bid']): # 大于等于卖一价，买盘
tick_type = 1
else:
tick_type = 0
self._money_flows[tick_data['code']].add_tick(
float(tick_data['amount']), float(tick_data['volume']), tick_type)
def run_all():
while True:
global MSG
global IMG
global FILE
sendtype = show_login()
result = False
if sendtype == 1:
result = login_normal()
elif sendtype == 2:
result = login_vip()
else:
break
if result == True:
print("登陆成功！")
print("请准备扫描登陆微信")
itchat.auto_login(hotReload=True)
print("登陆微信成功！您有以下这些群:")
rooms = itchat.get_chatrooms(update=True)
for i in rooms:
print(i['NickName'])
getans = show_time()
if getans == 1:
time_set = input("请输入一个间隔时间:如3即从先在开始每3小时自动发送")
get_type = show()
if get_type == 1:
MSG = input("请输入您要发送的文字:")
print("准备发送中。。。")
def msgset():
setmsg(MSG)
sched = BlockingScheduler()
int_trigger = IntervalTrigger(hour=int(time_set), id="my_job")
sched.add_job(msgset, int_trigger, id="my_job")
print('Press Ctrl+{0} to exit'.format('Break' if os.name == 'nt' else 'C'))
try:
sched.start()
except (KeyboardInterrupt, SystemExit):
sched.remove_job('my_job')
run_all()
elif get_type == 2:
IMG = input("请输入您要发送的图片路径:")
print("准备发送中。。。")
def imgset():
setimg(IMG)
sched = BlockingScheduler()
int_trigger = IntervalTrigger(hour=int(time_set), id="my_job")
sched.add_job(imgset, int_trigger, id="my_job")
print('Press Ctrl+{0} to exit'.format('Break' if os.name == 'nt' else 'C'))
try:
sched.start()
except (KeyboardInterrupt, SystemExit):
sched.remove_job('my_job')
run_all()
elif get_type == 3:
FILE = input("请输入您要发送的文件路径:")
print("准备发送中。。。")
def fileset():
setfile(FILE)
sched = BlockingScheduler()
int_trigger = IntervalTrigger(hour=int(time_set))
sched.add_job(fileset, int_trigger, id="my_job")
print('Press Ctrl+{0} to exit'.format('Break' if os.name == 'nt' else 'C'))
try:
sched.start()
except (KeyboardInterrupt, SystemExit):
sched.remove_job('my_job')
run_all()
else:
break
elif getans == 2:
time_set = input("请输入一个准确的时间:如2018.7.10.8.20即2018年7月10日8点20分:\n")
year, month, day, hour, minute = time_set.split('.')[0], time_set.split('.')[1], time_set.split('.')[2], time_set.split('.')[3], time_set.split('.')[4]
get_type = show()
if get_type == 1:
MSG = input("请输入您要发送的文字:")
print("准备发送中。。。")
def msgset():
setmsg(MSG)
sched = BlockingScheduler()
cron_trigger = CronTrigger(year=int(year), month=int(month), day=int(day), hour=int(hour), minute=int(minute), second=0)
sched.add_job(msgset, cron_trigger, id="my_job")
print('Press Ctrl+{0} to exit'.format('Break' if os.name == 'nt' else 'C'))
try:
sched.start()
except (KeyboardInterrupt, SystemExit):
sched.remove_job('my_job')
run_all()
elif get_type == 2:
IMG = input("请输入您要发送的图片路径:")
print("准备发送中。。。")
def imgset():
setimg(IMG)
sched = BlockingScheduler()
cron_trigger = CronTrigger(year=int(year), month=int(month), day=int(day), hour=int(hour),
minute=int(minute), second=0)
sched.add_job(imgset, cron_trigger, id="my_job")
print('Press Ctrl+{0} to exit'.format('Break' if os.name == 'nt' else 'C'))
try:
sched.start()
except (KeyboardInterrupt, SystemExit):
sched.remove_job('my_job')
run_all()
elif get_type == 3:
FILE = input("请输入您要发送的文件路径:")
print("准备发送中。。。")
def fileset():
setfile(FILE)
sched = BlockingScheduler()
cron_trigger = CronTrigger(year=int(year), month=int(month), day=int(day), hour=int(hour),
minute=int(minute), second=0)
sched.add_job(fileset, cron_trigger, id="my_job")
print('Press Ctrl+{0} to exit'.format('Break' if os.name == 'nt' else 'C'))
try:
sched.start()
except (KeyboardInterrupt, SystemExit):
sched.remove_job('my_job')
run_all()
else:
break
elif getans == 3:
run()
else:
break
else:
print("登陆失败！用户名不存在或密码错误！请重新登陆!")
run_all()
class Host:
def __init__(self, storage_id, queue_name, policy_uri,
log_group=None, metrics=None, output_dir=None):
logging.basicConfig(level=logging.INFO, format='%(message)s')
log.info("Running Azure Cloud Custodian Self-Host")
resources.load_available()
self.session = local_session(Session)
self.storage_session = self.session
storage_subscription_id = ResourceIdParser.get_subscription_id(storage_id)
if storage_subscription_id != self.session.subscription_id:
self.storage_session = Session(subscription_id=storage_subscription_id)
# Load configuration
self.options = Host.build_options(output_dir, log_group, metrics)
self.policy_storage_uri = policy_uri
self.event_queue_id = storage_id
self.event_queue_name = queue_name
# Default event queue name is the subscription ID
if not self.event_queue_name:
self.event_queue_name = self.session.subscription_id
# Prepare storage bits
self.policy_blob_client = None
self.blob_cache = {}
self.queue_storage_account = self.prepare_queue_storage(
self.event_queue_id,
self.event_queue_name)
self.queue_service = None
# Register event subscription
self.update_event_subscription()
# Policy cache and dictionary
self.policy_cache = tempfile.mkdtemp()
self.policies = {}
# Configure scheduler
self.scheduler = BlockingScheduler(Host.get_scheduler_config())
logging.getLogger('apscheduler.executors.default').setLevel(logging.ERROR)
logging.getLogger('apscheduler').setLevel(logging.ERROR)
# Schedule recurring policy updates
self.scheduler.add_job(self.update_policies,
'interval',
seconds=policy_update_seconds,
id="update_policies",
next_run_time=datetime.now(),
executor='threadpool')
# Schedule recurring queue polling
self.scheduler.add_job(self.poll_queue,
'interval',
seconds=queue_poll_seconds,
id="poll_queue",
executor='threadpool')
self.scheduler.start()
def update_policies(self):
"""
Enumerate all policies from storage.
Use the MD5 hashes in the enumerated policies
and a local dictionary to decide if we should
bother downloading/updating each blob.
We maintain an on-disk policy cache for future
features.
"""
if not self.policy_blob_client:
self.policy_blob_client = Storage.get_blob_client_by_uri(self.policy_storage_uri,
self.storage_session)
(client, container, prefix) = self.policy_blob_client
try:
# All blobs with YAML extension
blobs = [b for b in client.list_blobs(container) if Host.has_yaml_ext(b.name)]
except AzureHttpError as e:
# If blob methods are failing don't keep
# a cached client
self.policy_blob_client = None
raise e
# Filter to hashes we have not seen before
new_blobs = self._get_new_blobs(blobs)
# Get all YAML files on disk that are no longer in blob storage
cached_policy_files = [f for f in os.listdir(self.policy_cache)
if Host.has_yaml_ext(f)]
removed_files = [f for f in cached_policy_files if f not in [b.name for b in blobs]]
if not (removed_files or new_blobs):
return
# Update a copy so we don't interfere with
# iterations on other threads
policies_copy = self.policies.copy()
for f in removed_files:
path = os.path.join(self.policy_cache, f)
self.unload_policy_file(path, policies_copy)
# Get updated YML files
for blob in new_blobs:
policy_path = os.path.join(self.policy_cache, blob.name)
if os.path.exists(policy_path):
self.unload_policy_file(policy_path, policies_copy)
elif not os.path.isdir(os.path.dirname(policy_path)):
os.makedirs(os.path.dirname(policy_path))
client.get_blob_to_path(container, blob.name, policy_path)
self.load_policy(policy_path, policies_copy)
self.blob_cache.update({blob.name: blob.properties.content_settings.content_md5})
# Assign our copy back over the original
self.policies = policies_copy
def _get_new_blobs(self, blobs):
new_blobs = []
for blob in blobs:
md5_hash = blob.properties.content_settings.content_md5
if not md5_hash:
blob, md5_hash = self._try_create_md5_content_hash(blob)
if blob and md5_hash and md5_hash != self.blob_cache.get(blob.name):
new_blobs.append(blob)
return new_blobs
def _try_create_md5_content_hash(self, blob):
# Not all storage clients provide the md5 hash when uploading a file
# so, we need to make sure that hash exists.
(client, container, _) = self.policy_blob_client
log.info("Applying md5 content hash to policy {}".format(blob.name))
try:
# Get the blob contents
blob_bytes = client.get_blob_to_bytes(container, blob.name)
# Re-upload the blob. validate_content ensures that the md5 hash is created
client.create_blob_from_bytes(container, blob.name, blob_bytes.content,
validate_content=True)
# Re-fetch the blob with the new hash
hashed_blob = client.get_blob_properties(container, blob.name)
return hashed_blob, hashed_blob.properties.content_settings.content_md5
except AzureHttpError as e:
log.warning("Failed to apply a md5 content hash to policy {}. "
"This policy will be skipped.".format(blob.name))
log.error(e)
return None, None
def load_policy(self, path, policies):
"""
Loads a YAML file and prompts scheduling updates
:param path: Path to YAML file on disk
:param policies: Dictionary of policies to update
"""
with open(path, "r") as stream:
try:
policy_config = yaml.safe_load(stream)
new_policies = PolicyCollection.from_data(policy_config, self.options)
if new_policies:
for p in new_policies:
log.info("Loading Policy %s from %s" % (p.name, path))
p.validate()
policies.update({p.name: {'policy': p}})
# Update periodic
policy_mode = p.data.get('mode', {}).get('type')
if policy_mode == CONTAINER_TIME_TRIGGER_MODE:
self.update_periodic(p)
elif policy_mode != CONTAINER_EVENT_TRIGGER_MODE:
log.warning(
"Unsupported policy mode for Azure Container Host: {}. "
"{} will not be run. "
"Supported policy modes include \"{}\" and \"{}\"."
.format(
policy_mode,
p.data['name'],
CONTAINER_EVENT_TRIGGER_MODE,
CONTAINER_TIME_TRIGGER_MODE
)
)
except Exception as exc:
log.error('Invalid policy file %s %s' % (path, exc))
def unload_policy_file(self, path, policies):
"""
Unload a policy file that has changed or been removed.
Take the copy from disk and pop all policies from dictionary
and update scheduled jobs.
"""
with open(path, "r") as stream:
try:
policy_config = yaml.safe_load(stream)
except yaml.YAMLError as exc:
log.warning('Failure loading cached policy for cleanup %s %s' % (path, exc))
os.unlink(path)
return path
try:
# Some policies might have bad format, so they have never been loaded
removed = [policies.pop(p['name'])
for p in policy_config.get('policies', [])
if p['name'] in policies]
log.info('Removing policies %s' % removed)
# update periodic
periodic_names = \
[p['name'] for p in policy_config.get('policies', [])
if p.get('mode', {}).get('schedule')]
periodic_to_remove = \
[p for p in periodic_names if p in [j.id for j in self.scheduler.get_jobs()]]
for name in periodic_to_remove:
self.scheduler.remove_job(job_id=name)
except (AttributeError, KeyError) as exc:
log.warning('Failure loading cached policy for cleanup %s %s' % (path, exc))
os.unlink(path)
return path
def update_periodic(self, policy):
"""
Update scheduled policies using cron type
periodic scheduling.
"""
trigger = CronTrigger.from_crontab(policy.data['mode']['schedule'])
self.scheduler.add_job(Host.run_policy,
trigger,
id=policy.name,
name=policy.name,
args=[policy, None, None],
coalesce=True,
max_instances=1,
replace_existing=True,
misfire_grace_time=60)
def update_event_subscription(self):
"""
Create a single event subscription to channel
all events to an Azure Queue.
"""
log.info('Updating event grid subscriptions')
destination = StorageQueueEventSubscriptionDestination(
resource_id=self.queue_storage_account.id, queue_name=self.event_queue_name)
# Build event filter
event_filter = EventSubscriptionFilter(
included_event_types=['Microsoft.Resources.ResourceWriteSuccess'])
# Update event subscription
AzureEventSubscription.create(destination,
self.event_queue_name,
self.session.get_subscription_id(),
self.session, event_filter)
def poll_queue(self):
"""
Poll the Azure queue and loop until
there are no visible messages remaining.
"""
# Exit if we don't have any policies
if not self.policies:
return
if not self.queue_service:
self.queue_service = Storage.get_queue_client_by_storage_account(
self.queue_storage_account,
self.storage_session)
while True:
try:
messages = Storage.get_queue_messages(
self.queue_service,
self.event_queue_name,
num_messages=queue_message_count,
visibility_timeout=queue_timeout_seconds)
except AzureHttpError:
self.queue_service = None
raise
if len(messages) == 0:
break
log.info('Pulled %s events to process while polling queue.' % len(messages))
for message in messages:
if message.dequeue_count > max_dequeue_count:
Storage.delete_queue_message(self.queue_service,
self.event_queue_name,
message=message)
log.warning("Event deleted due to reaching maximum retry count.")
else:
# Run matching policies
self.run_policies_for_event(message)
# We delete events regardless of policy result
Storage.delete_queue_message(
self.queue_service,
self.event_queue_name,
message=message)
def run_policies_for_event(self, message):
"""
Find all policies subscribed to this event type
and schedule them for immediate execution.
"""
# Load up the event
event = json.loads(base64.b64decode(message.content).decode('utf-8'))
operation_name = event['data']['operationName']
# Execute all policies matching the event type
for k, v in self.policies.items():
events = v['policy'].data.get('mode', {}).get('events')
if not events:
continue
events = AzureEvents.get_event_operations(events)
if operation_name.upper() in (event.upper() for event in events):
self.scheduler.add_job(Host.run_policy,
id=k + event['id'],
name=k,
args=[v['policy'],
event,
None],
misfire_grace_time=60 * 3)
def prepare_queue_storage(self, queue_resource_id, queue_name):
"""
Create a storage client using unusual ID/group reference
as this is what we require for event subscriptions
"""
storage_client = self.storage_session \
.client('azure.mgmt.storage.StorageManagementClient')
account = storage_client.storage_accounts.get_properties(
ResourceIdParser.get_resource_group(queue_resource_id),
ResourceIdParser.get_resource_name(queue_resource_id))
Storage.create_queue_from_storage_account(account,
queue_name,
self.session)
return account
@staticmethod
def run_policy(policy, event, context):
try:
policy.push(event, context)
except Exception:
log.exception("Policy Failed: %s", policy.name)
@staticmethod
def build_options(output_dir=None, log_group=None, metrics=None):
"""
Initialize the Azure provider to apply global config across all policy executions.
"""
if not output_dir:
output_dir = tempfile.mkdtemp()
log.warning('Output directory not specified. Using directory: %s' % output_dir)
config = Config.empty(
**{
'log_group': log_group,
'metrics': metrics,
'output_dir': output_dir
}
)
return Azure().initialize(config)
@staticmethod
def get_scheduler_config():
if os.name == 'nt':
executor = "apscheduler.executors.pool:ThreadPoolExecutor"
else:
executor = "apscheduler.executors.pool:ProcessPoolExecutor"
return {
'apscheduler.jobstores.default': {
'type': 'memory'
},
'apscheduler.executors.default': {
'class': executor,
'max_workers': '4'
},
'apscheduler.executors.threadpool': {
'type': 'threadpool',
'max_workers': '20'
},
'apscheduler.job_defaults.coalesce': 'true',
'apscheduler.job_defaults.max_instances': '1',
'apscheduler.timezone': 'UTC',
}
@staticmethod
def has_yaml_ext(filename):
return filename.lower().endswith(('.yml', '.yaml'))
@staticmethod
@click.command(help="Periodically run a set of policies from an Azure storage container "
"against a single subscription. The host will update itself with new "
"policies and event subscriptions as they are added.")
@click.option("--storage-id", "-q", envvar=ENV_CONTAINER_STORAGE_RESOURCE_ID, required=True,
help="The resource id of the storage account to create the event queue in")
@click.option("--queue-name", "-n", envvar=ENV_CONTAINER_QUEUE_NAME,
help="The name of the event queue to create")
@click.option("--policy-uri", "-p", envvar=ENV_CONTAINER_POLICY_URI, required=True,
help="The URI to the Azure storage container that holds the policies")
@click.option("--log-group", "-l", envvar=ENV_CONTAINER_OPTION_LOG_GROUP,
help="Location to send policy logs")
@click.option("--metrics", "-m", envvar=ENV_CONTAINER_OPTION_METRICS,
help="The resource name or instrumentation key for uploading metrics")
@click.option("--output-dir", "-d", envvar=ENV_CONTAINER_OPTION_OUTPUT_DIR,
help="The directory for policy output")
def cli(**kwargs):
Host(**kwargs)
class JobScheduler(metaclass=Singleton):
__function_dict__ = {}
def __init__(self):
self.logger = Log().logger
self.scheduler = BlockingScheduler(logger=self.logger)
self.parser = DateTimeParser()
def get_lives(self):
return {job.name: job for job in self.scheduler.get_jobs()}
def add_job(self,
name,
trigger,
trigger_args,
func_args=None,
func_kwargs=None):
if name not in self.get_lives():
func = self.get_functions(name)
self.scheduler.add_job(func,
trigger=trigger,
args=func_args,
kwargs=func_kwargs,
id=name,
name=name,
**trigger_args)
self.logger.info('[%s][%s]作业已添加，并已启动' % (trigger, name))
def add_date_job(self, name, time, func_args=None, func_kwargs=None):
time = self.parser.set_date(time).set_time(time).datetime
self.add_job(name, 'date', {'run_date': time}, func_args, func_kwargs)
def add_interval_job(self,
name,
weeks=0,
days=0,
hours=0,
minutes=0,
seconds=0,
func_args=None,
func_kwargs=None):
self.add_job(
name, 'interval', {
'weeks': weeks,
'days': days,
'hours': hours,
'minutes': minutes,
'seconds': seconds
}, func_args, func_kwargs)
def add_cron_job(self,
name,
year=None,
month=None,
day=None,
week=None,
day_of_week=None,
hour=None,
minute=None,
second=None,
func_args=None,
func_kwargs=None):
self.add_job(
name, 'cron', {
'year': year,
'month': month,
'day': day,
'week': week,
'day_of_week': day_of_week,
'hour': hour,
'minute': minute,
'second': second
}, func_args, func_kwargs)
def add_started_job(self,
name,
after_seconds=1,
func_args=None,
func_kwargs=None):
time = datetime.now() + timedelta(seconds=after_seconds)
self.add_date_job(name, time, func_args, func_kwargs)
def delete_job(self, name):
jobs = self.get_lives()
if name in jobs:
self.scheduler.remove_job(jobs[name].id)
self.logger.info('作业[%s]已移除' % name)
@classmethod
def register(cls, name):
def add_method(f):
cls.__function_dict__[name.strip()] = f
return f
return add_method
def get_functions(self, name):
return self.__function_dict__.get(name.strip())
def get_function_doc(self, name):
return self.get_functions(name).__doc__
def get_function_names(self):
return sorted(self.__function_dict__.keys())
