def __init__(self, queue, interval):
super().__init__(queue, interval)
self.count = self._init_count()
self.locator = Locator()
self.TOKEN = self._update_token()
self.domain_id = _get_domain_id_from_token(self.TOKEN)
self.schedule_info = {}
self.idx = 0
def __init__(self,
experimental_non_blocking=True,
experimental_thread_pool=None):
super().__init__(experimental_non_blocking, experimental_thread_pool)
locator = Locator()
self.actuator = locator.get_actuator('Health')
self.actuator.add_health_update(self)
self.actuator.check()
def __init__(self, single_task: dict):
self.locator = single_task['locator']
self.name = single_task['name']
self.metadata = single_task['metadata']
self.method = single_task['method']
self.params = single_task['params']
transaction = Transaction(meta=self.metadata)
self._locator = Locator(transaction)
def __init__(
self,
interval=1,
):
self.config = self.parse_config(interval)
self.count = self._init_count()
self.locator = Locator()
self.TOKEN = self._update_token()
self.domain_id = _get_domain_id_from_token(self.TOKEN)
async def webhook(webhook_id: str, access_key: str, request: Request):
locator = Locator()
data = await request.json()
event_service = locator.get_service('EventService')
event_service.create({
'webhook_id': webhook_id,
'access_key': access_key,
'data': data or {}
})
return {}
def __init__(self, *args, **kwargs):
self.transaction = Transaction()
self.locator = Locator(self.transaction)
if hasattr(current_app.conf, "spaceone_scheduler_service"):
self.service_name = current_app.conf.get("spaceone_scheduler_service")
else:
raise SpaceOneSchedulerError("can not find CELERY.spaceone_scheduler_service config")
self.Service = self.locator.get_service(self.service_name, metadata=self.metadata)
self._schedule = {}
self._last_updated = None
Scheduler.__init__(self, *args, **kwargs)
self.max_interval = (kwargs.get('max_interval')
or self.app.conf.CELERYBEAT_MAX_LOOP_INTERVAL or 5)
def __init__(self,
metadata: dict = {},
transaction: Transaction = None,
**kwargs):
self.func_name = None
self.is_with_statement = False
if transaction:
self.transaction = transaction
else:
self.transaction = Transaction(metadata)
if config.get_global('SET_LOGGING', True):
set_logger(transaction=self.transaction)
self.locator = Locator(self.transaction)
self.handler = {
'authentication': {
'handlers': [],
'methods': []
},
'authorization': {
'handlers': [],
'methods': []
},
'mutation': {
'handlers': [],
'methods': []
},
'event': {
'handlers': [],
'methods': []
},
}
class InventoryDBUpdater(BaseWorker):
def __init__(self, queue, **kwargs):
BaseWorker.__init__(self, queue, **kwargs)
self.locator = Locator()
def run(self):
""" Infinite Loop
"""
# Create Manager
collecting_mgr = self.locator.get_manager('CollectingManager')
while True:
# Read from Queue
try:
binary_resource_info = queue.get(self.queue)
resource_info = json.loads(binary_resource_info.decode())
# Create Transaction
collecting_mgr.transaction = Transaction(resource_info['meta'])
# processing
method = resource_info['method']
if method == '_process_single_result':
collecting_mgr._process_single_result(
resource_info['res'], resource_info['param'])
elif method == '_watchdog_job_task_stat':
collecting_mgr._watchdog_job_task_stat(
resource_info['param'])
else:
_LOGGER.error(f'Unknown request: {resource_info}')
except Exception as e:
_LOGGER.error(f'[{self._name_}] failed to processing: {e}')
continue
def __init__(self, transaction: Transaction = None):
if transaction:
self.transaction = transaction
else:
self.transaction = Transaction()
self.locator = Locator(self.transaction)
async def create_event(webhook_id: str, access_key: str, request: Request):
locator = Locator()
try:
try:
data = await request.json()
except Exception as e:
_LOGGER.debug(f'JSON Parsing Error: {e}')
raise ERROR_UNKNOWN(message='JSON Parsing Error: Request body requires JSON format.')
event_service: EventService = locator.get_service('EventService')
event_service.create({
'webhook_id': webhook_id,
'access_key': access_key,
'data': data or {}
})
return {}
except ERROR_BASE as e:
raise HTTPException(status_code=500, detail=e.message)
except Exception as e:
raise HTTPException(status_code=500, detail=f'Unknown Error: {str(e)}')
def _change_secret_id_filter(condition):
value = condition.get('v') or condition.get('value')
operator = condition.get('o') or condition.get('operator')
map_query = {'filter': [{'k': 'secret_id', 'v': value, 'o': operator}]}
locator = Locator()
secret_group_map_model = locator.get_model('SecretGroupMap')
map_vos, total_count = secret_group_map_model.query(**map_query)
return {
'k':
'secret_group_id',
'v':
list(
map(lambda map_vo: map_vo.secret_group.secret_group_id,
map_vos)),
'o':
'in'
}
def __init__(self, transaction=None, **kwargs):
if transaction:
self.transaction = transaction
else:
self.transaction = Transaction()
self.locator = Locator(self.transaction)
for k, v in kwargs.items():
setattr(self, k, v)
class SingleTask:
def __init__(self, single_task: dict):
self.locator = single_task['locator']
self.name = single_task['name']
self.metadata = single_task['metadata']
self.method = single_task['method']
self.params = single_task['params']
transaction = Transaction(meta=self.metadata)
self._locator = Locator(transaction)
def execute(self):
""" Run method
"""
try:
if self.locator == 'SERVICE':
caller = self._locator.get_service(self.name, self.metadata)
elif self.locator == 'MANAGER':
caller = self._locator.get_manager(self.name)
except Exception as e:
_LOGGER.debug(f'[SingleTask] fail at locator {e}')
raise ERROR_TASK_LOCATOR(locator=self.locator, name=self.name)
try:
_LOGGER.debug(
f'[SingleTask] request: {self.name}.{self.method} {self.params}'
)
method = getattr(caller, self.method)
resp = method(**self.params)
_LOGGER.debug(f'[SingleTask] response: {resp}')
return resp
except Exception as e:
_LOGGER.error(
f'[SingleTask] fail to execute method: {self.method}, params: {self.params}, {e}'
)
raise ERROR_TASK_METHOD(name=self.name,
method=self.method,
params=self.params)
def _update_alert_state(alert_id, access_key, state):
locator = Locator()
try:
alert_service: AlertService = locator.get_service('AlertService')
alert_vo: Alert = alert_service.update_state({
'alert_id': alert_id,
'access_key': access_key,
'state': state
})
return {
'alert_number': alert_vo.alert_number,
'alert_id': alert_vo.alert_id,
'title': alert_vo.title,
'state': alert_vo.state,
'assignee': alert_vo.assignee,
'urgency': alert_vo.urgency
}
except ERROR_BASE as e:
raise HTTPException(status_code=500, detail=e.message)
except Exception as e:
raise HTTPException(status_code=500, detail=f'Unknown Error: {str(e)}')
def __init__(self, metadata={}, transaction=None):
self.func_name = None
self.is_with_statement = False
if transaction:
self.transaction = transaction
else:
self.transaction = Transaction(metadata)
set_logger(transaction=self.transaction)
self.locator = Locator(self.transaction)
self.handler = {
'authentication': {'handlers': [], 'methods': []},
'authorization': {'handlers': [], 'methods': []},
'event': {'handlers': [], 'methods': []},
}
class StatHourlyScheduler(HourlyScheduler):
def __init__(self, queue, interval, minute=':00'):
super().__init__(queue, interval, minute)
self.count = self._init_count()
self.locator = Locator()
self.TOKEN = self._update_token()
self.domain_id = _get_domain_id_from_token(self.TOKEN)
def _init_count(self):
# get current time
cur = datetime.datetime.now()
count = {
'previous': cur, # Last check_count time
'index': 0, # index
'hour': cur.hour, # previous hour
'started_at': 0, # start time of push_token
'ended_at': 0 # end time of execution in this tick
}
_LOGGER.debug(f'[_init_count] {count}')
return count
def _update_token(self):
token = config.get_global('TOKEN')
if token == "":
token = _validate_token(config.get_global('TOKEN_INFO'))
return token
def create_task(self):
# self.check_global_configuration()
domains = self.list_domains()
result = []
for domain in domains:
stp = self._create_job_request(domain)
result.append(stp)
return result
def list_domains(self):
try:
ok = self.check_count()
if ok == False:
# ERROR LOGGING
pass
# Loop all domain, then find schedule
metadata = {'token': self.TOKEN, 'domain_id': self.domain_id}
schedule_svc = self.locator.get_service('ScheduleService',
metadata)
params = {}
resp = schedule_svc.list_domains(params)
_LOGGER.debug(
f'[list_domain] num of domains: {resp["total_count"]}')
return resp['results']
except Exception as e:
_LOGGER.error(e)
return []
def check_count(self):
# check current count is correct or not
cur = datetime.datetime.now()
hour = cur.hour
# check
if (self.count['hour'] + self.config) % 24 != hour:
if self.count['hour'] == hour:
_LOGGER.error('[check_count] duplicated call in the same time')
else:
_LOGGER.error('[check_count] missing time')
# This is continuous task
count = {
'previous': cur,
'index': self.count['index'] + 1,
'hour': hour,
'started_at': cur
}
self.count.update(count)
def _update_count_ended_at(self):
cur = datetime.datetime.now()
self.count['ended_at'] = cur
def _list_schedule(self, hour, domain_id):
""" List statistics.Schedule
"""
params = {
'query': {
'filter': [{
'k': 'schedule.hours',
'v': hour,
'o': 'eq'
}],
},
'domain_id': domain_id
}
metadata = {'token': self.TOKEN, 'domain_id': self.domain_id}
schedule_svc = self.locator.get_service('ScheduleService', metadata)
schedules, total_count = schedule_svc.list(params)
_LOGGER.debug(
f'[_list_schedule] schedules: {schedules}, total_count: {total_count}'
)
return schedules
def _create_job_request(self, domain):
""" Based on domain, create Job Request
Returns:
jobs: SpaceONE Pipeline Template
"""
_LOGGER.debug(f'[_create_job_request] domain: {domain}')
metadata = {'token': self.TOKEN, 'domain_id': self.domain_id}
schedules = self._list_schedule(self.count['hour'],
domain['domain_id'])
sched_jobs = []
for schedule in schedules:
sched_job = {
'locator': 'SERVICE',
'name': 'HistoryService',
'metadata': metadata,
'method': 'create',
'params': {
'params': {
'schedule_id': schedule.schedule_id,
'domain_id': domain['domain_id']
}
}
}
sched_jobs.append(sched_job)
stp = {
'name': 'statistics_hourly_schedule',
'version': 'v1',
'executionEngine': 'BaseWorker',
'stages': sched_jobs
}
_LOGGER.debug(f'[_create_job_request] tasks: {stp}')
return stp
@staticmethod
def _create_schedule_params(schedule, domain_id):
dict_schedule = dict(schedule.to_dict())
_LOGGER.debug(f'[_create_schedule_params] schedule: {schedule}')
required_params = [
'schedule_id', 'data_source_id', 'resource_type', 'query', 'join',
'formulas', 'domain_id'
]
result = {
'schedule_id': dict_schedule['schedule_id'],
'domain_id': domain_id
}
print('#' * 30)
for param in required_params:
print(f'check : {param}')
if param in dict_schedule['options']:
result[param] = dict_schedule['options'][param]
_LOGGER.debug(f'[_create_schedule_params] params: {result}')
return result
def __init__(self, *args, **kwargs):
transaction = Transaction()
self.locator = Locator(transaction)
class CleanupScheduler(HourlyScheduler):
def __init__(self, queue, interval, minute=':00'):
super().__init__(queue, interval, minute)
self.count = self._init_count()
self.locator = Locator()
self.TOKEN = self._update_token()
self.domain_id = _get_domain_id_from_token(self.TOKEN)
def _init_count(self):
# get current time
cur = datetime.datetime.now()
count = {
'previous': cur, # Last check_count time
'index': 0, # index
'hour': cur.hour, # previous hour
'started_at': 0, # start time of push_token
'ended_at': 0 # end time of execution in this tick
}
_LOGGER.debug(f'[_init_count] {count}')
return count
def _update_token(self):
token = config.get_global('TOKEN')
if token == "":
token = _validate_token(config.get_global('TOKEN_INFO'))
return token
def create_task(self):
# self.check_global_configuration()
domains = self.list_domains()
result = []
for domain in domains:
stp = self._create_job_request(domain)
result.append(stp)
return result
def list_domains(self):
try:
ok = self.check_count()
if ok == False:
# ERROR LOGGING
pass
# Loop all domain, then find scheduled collector
metadata = {'token': self.TOKEN, 'domain_id': self.domain_id}
cleanup_svc = self.locator.get_service('CleanupService', metadata)
params = {}
resp = cleanup_svc.list_domains(params)
_LOGGER.debug(f'[list_domain] num of domains: {resp["total_count"]}')
return resp['results']
except Exception as e:
_LOGGER.error(e)
return []
def check_count(self):
# check current count is correct or not
cur = datetime.datetime.now()
hour = cur.hour
# check
if (self.count['hour'] + self.config) % 24 != hour:
if self.count['hour'] == hour:
_LOGGER.error('[check_count] duplicated call in the same time')
else:
_LOGGER.error('[check_count] missing time')
# This is continuous task
count = {
'previous': cur,
'index': self.count['index'] + 1,
'hour': hour,
'started_at': cur
}
self.count.update(count)
def _update_count_ended_at(self):
cur = datetime.datetime.now()
self.count['ended_at'] = cur
def _create_job_request(self, domain):
""" Based on domain, create Job Request
Returns:
jobs: SpaceONE Pipeline Template
"""
_LOGGER.debug(f'[_create_job_request] domain: {domain}')
metadata = {'token': self.TOKEN, 'domain_id': self.domain_id}
sched_job = {
'locator': 'SERVICE',
'name': 'CleanupService',
'metadata': metadata,
'method': 'update_collection_state',
'params': {'params': {
'options': {},
'domain_id': domain['domain_id']
}
}
}
update_job_state = {
'locator': 'SERVICE',
'name': 'CleanupService',
'metadata': metadata,
'method': 'update_job_state',
'params': {'params': {
'options': {},
'domain_id': domain['domain_id']
}
}
}
stp = {'name': 'inventory_cleanup_schedule',
'version': 'v1',
'executionEngine': 'BaseWorker',
'stages': [sched_job, update_job_state]}
_LOGGER.debug(f'[_create_job_request] tasks: {stp}')
return stp
def __init__(self, queue, interval, minute=':00'):
super().__init__(queue, interval, minute)
self.count = self._init_count()
self.locator = Locator()
self.TOKEN = self._update_token()
self.domain_id = _get_domain_id_from_token(self.TOKEN)
def __init__(self, queue, interval):
super().__init__(queue, interval)
self.locator = Locator()
self._init_config()
self._create_metadata()
class InventoryHourlyScheduler(HourlyScheduler):
def __init__(self, queue, interval, minute=':00'):
super().__init__(queue, interval, minute)
self.count = self._init_count()
self.locator = Locator()
self.TOKEN = self._update_token()
self.domain_id = _get_domain_id_from_token(self.TOKEN)
def _init_count(self):
# get current time
cur = datetime.datetime.now()
count = {
'previous': cur, # Last check_count time
'index': 0, # index
'hour': cur.hour, # previous hour
'started_at': 0, # start time of push_token
'ended_at': 0 # end time of execution in this tick
}
_LOGGER.debug(f'[_init_count] {count}')
return count
def _update_token(self):
token = config.get_global('TOKEN')
if token == "":
token = _validate_token(config.get_global('TOKEN_INFO'))
return token
def create_task(self):
# self.check_global_configuration()
schedules = self.list_schedules()
result = []
for schedule in schedules:
try:
stp = self._create_job_request(schedule)
result.append(stp)
except Exception as e:
_LOGGER.error(f'[create_task] check schedule {schedule}')
return result
def list_schedules(self):
try:
ok = self.check_count()
if ok == False:
# ERROR LOGGING
pass
# Loop all domain, then find scheduled collector
collector_svc = self.locator.get_service('CollectorService')
schedule = {'hour': self.count['hour']}
_LOGGER.debug(f'[push_token] schedule: {schedule}')
schedule_vos, total = collector_svc.scheduled_collectors(
{'schedule': schedule})
_LOGGER.debug(f'[push_token] scheduled count: {total}')
return schedule_vos
except Exception as e:
_LOGGER.error(e)
return []
def check_count(self):
# check current count is correct or not
cur = datetime.datetime.now()
hour = cur.hour
# check
if (self.count['hour'] + self.config) % 24 != hour:
if self.count['hour'] == hour:
_LOGGER.error('[check_count] duplicated call in the same time')
else:
_LOGGER.error('[check_count] missing time')
# This is continuous task
count = {
'previous': cur,
'index': self.count['index'] + 1,
'hour': hour,
'started_at': cur
}
self.count.update(count)
def _update_count_ended_at(self):
cur = datetime.datetime.now()
self.count['ended_at'] = cur
def _create_job_request(self, scheduler_vo):
""" Based on scheduler_vo, create Job Request
Args:
scheduler_vo: Scheduler VO
- scheduler_id
- name
- collector: Reference of Collector
- schedule
- filter
- collector_mode
- created_at
- last_scheduled_at
- domain_id
}
Returns:
jobs: SpaceONE Pipeline Template
Because if collector_info has credential_group_id,
we have to iterate all credentials in the credential_group
"""
_LOGGER.debug(f'[_create_job_request] scheduler_vo: {scheduler_vo}')
plugin_info = scheduler_vo.collector.plugin_info
_LOGGER.debug(f'plugin_info: {plugin_info}')
domain_id = scheduler_vo.domain_id
metadata = {
'token': self.TOKEN,
'service': 'inventory',
'resource': 'Collector',
'verb': 'collect',
'authorization': False,
'domain_id': self.domain_id
}
sched_job = {
'locator': 'SERVICE',
'name': 'CollectorService',
'metadata': metadata,
'method': 'collect',
'params': {
'params': {
'collector_id': scheduler_vo.collector.collector_id,
# if filter
# contact credential
'collect_mode': 'ALL',
'filter': {},
'domain_id': domain_id
}
}
}
stp = {
'name': 'inventory_collect_schedule',
'version': 'v1',
'executionEngine': 'BaseWorker',
'stages': [sched_job]
}
_LOGGER.debug(f'[_create_job_request] tasks: {stp}')
return stp
class SpaceOneScheduler(Scheduler):
#: how often should we sync in schedule information
#: from the backend mongo database
UPDATE_INTERVAL = datetime.timedelta(seconds=5)
Entry = SpaceOneScheduleEntry
Service = None
_metadata: dict = None
@property
def metadata(self):
if self._metadata is None:
token = config.get_global('CELERY',{}).get('auth',{}).get('token')
if token:
self._metadata = {'token': token, }
else:
self._metadata = {}
return self._metadata
def __init__(self, *args, **kwargs):
self.transaction = Transaction()
self.locator = Locator(self.transaction)
if hasattr(current_app.conf, "spaceone_scheduler_service"):
self.service_name = current_app.conf.get("spaceone_scheduler_service")
else:
raise SpaceOneSchedulerError("can not find CELERY.spaceone_scheduler_service config")
self.Service = self.locator.get_service(self.service_name, metadata=self.metadata)
self._schedule = {}
self._last_updated = None
Scheduler.__init__(self, *args, **kwargs)
self.max_interval = (kwargs.get('max_interval')
or self.app.conf.CELERYBEAT_MAX_LOOP_INTERVAL or 5)
def setup_schedule(self):
pass
def requires_update(self):
"""check whether we should pull an updated schedule
from the backend database"""
if not self._last_updated:
return True
return self._last_updated + self.UPDATE_INTERVAL < datetime.datetime.now()
def get_from_service(self):
self.sync()
d = {}
schedules = self.Service.list()
print(f"Find {len(schedules)} schedules")
for task in schedules:
d[task.schedule_id] = self.Entry(task)
return d
@property
def schedule(self):
if self.requires_update():
self._schedule = self.get_from_service()
self._last_updated = datetime.datetime.now()
return self._schedule
def sync(self):
print('Writing entries...')
values = self._schedule.values()
for entry in values:
entry.save(self.Service)
@property
def info(self):
return f' . service -> {self.service_name}'
class CleanupScheduler(HourlyScheduler):
""" Clean-up unused plugins
Some plugins are not used after upgrade
Delete unused plugins, if last get_endpoint time is larger than limit
"""
def __init__(self, queue, interval, minute=':00'):
super().__init__(queue, interval, minute)
self.count = self._init_count()
self.locator = Locator()
self.TOKEN = self._update_token()
self.domain_id = _get_domain_id_from_token(self.TOKEN)
def _init_count(self):
cur = datetime.datetime.utcnow()
count = {
'previous': cur,
'index': 0,
'hour': cur.hour,
'started_at': 0,
'end_at': 0
}
_LOGGER.debug(f'[_init_count] {count}')
return count
def _update_token(self):
token = config.get_global('TOKEN')
if token == '':
token = _validate_token(config.get_global('TOKEN_INFO'))
return token
def create_task(self):
domains = self.list_domains()
result = []
for domain in domains:
stp = self._create_job_request(domain)
result.append(stp)
return result
def list_domains(self):
try:
ok = self.check_count()
if ok == False:
# ERROR LOGGING
pass
# Loop all domain, then find schedule
metadata = {
'token': self.TOKEN,
'service': 'plugin',
'resource': 'Supervisor',
'verb': 'list_domains',
'authorization': True,
'mutation': True,
'domain_id': self.domain_id
}
svc = self.locator.get_service('SupervisorService', metadata)
params = {}
resp = svc.list_domains(params)
_LOGGER.debug(
f'[list_domain] num of domains: {resp["total_count"]}')
return resp['results']
except Exception as e:
_LOGGER.error(e)
return []
def check_count(self):
# check current count is correct or not
cur = datetime.datetime.utcnow()
hour = cur.hour
# check
if (self.count['hour'] + self.config) % 24 != hour:
if self.count['hour'] == hour:
_LOGGER.error('[check_count] duplicated call in the same time')
else:
_LOGGER.error('[check_count] missing time')
# This is continuous task
count = {
'previous': cur,
'index': self.count['index'] + 1,
'hour': hour,
'started_at': cur
}
self.count.update(count)
def _update_count_ended_at(self):
cur = datetime.datetime.utcnow()
self.count['ended_at'] = cur
def _create_job_request(self, domain):
""" Based on domain, create Job Request
Returns:
jobs: SpaceONE Pipeline Template
"""
_LOGGER.debug(f'[_create_job_request] domain: {domain}')
metadata = {
'token': self.TOKEN,
'service': 'plugin',
'resource': 'Supervisor',
'verb': 'cleanup_plugins',
'authorization': True,
'mutation': True,
'domain_id': self.domain_id
}
sched_job = {
'locator': 'SERVICE',
'name': 'SupervisorService',
'metadata': metadata,
'method': 'cleanup_plugins',
'params': {
'params': {
'domain_id': domain['domain_id']
}
}
}
stp = {
'name': 'cleanup_unused_plugins',
'version': 'v1',
'executionEngine': 'BaseWorker',
'stages': [sched_job]
}
_LOGGER.debug(f'[_create_job_request] tasks: {stp}')
return stp
class InventoryIntervalScheduler(IntervalScheduler):
def __init__(self, queue, interval):
super().__init__(queue, interval)
self.count = self._init_count()
self.locator = Locator()
self.TOKEN = self._update_token()
self.domain_id = _get_domain_id_from_token(self.TOKEN)
self.schedule_info = {}
self.idx = 0
def run(self):
# Every specific interval, check schedule
schedule.every(self.config).seconds.do(self._check_interval)
while True:
schedule.run_pending()
time.sleep(1)
def _init_count(self):
# get current time
cur = datetime.datetime.now()
count = {
'previous': cur, # Last check_count time
'index': 0, # index
'hour': cur.hour, # previous hour
'started_at': 0, # start time of push_token
'ended_at': 0 # end time of execution in this tick
}
_LOGGER.debug(f'[_init_count] {count}')
return count
def _update_token(self):
token = config.get_global('TOKEN')
if token == "":
token = _validate_token(config.get_global('TOKEN_INFO'))
return token
def push_task(self, schedule_id, interval_info):
# schedule_id: schedule_2222
# interval_info: {integval: 30, domain_id: dom-2222, collector_id: collector-3333}
task = self._create_job_request(schedule_id, interval_info)
_LOGGER.debug(f'[push_task] {task["name"]}')
try:
validate(task, schema=SPACEONE_TASK_SCHEMA)
json_task = json.dumps(task)
_LOGGER.debug(f'[push_task] Task schema: {task}')
queue.put(self.queue, json_task)
except Exception as e:
print(e)
_LOGGER.debug(f'[push_task] Task schema: {task}, {e}')
def _check_interval(self):
""" Check interval schedule
Then run schedule
"""
# patch schedule and sync
# create new interval Schedule per schedule_id
#interval_schedules = {schedule-1234: {interval: 20, domain_id: dom-1234, collector_id: collector-5678},
# schedule_2222: {integval: 30, domain_id: dom-2222, collector_id: collector-3333}
# }
interval_schedules = self._get_interval_schedules()
# Update Scheule based on DB
schedule_ids = []
for schedule_id, interval_info in interval_schedules.items():
schedule_ids.append(schedule_id)
try:
interval_value = interval_info['interval']
# Create New scheduler
if schedule_id not in self.schedule_info:
_LOGGER.debug(
f'[_check_interval] create {schedule_id}, at every {interval_value}'
)
self.schedule_info[schedule_id] = interval_info
job = schedule.every(interval_value).seconds.do(
self.push_task, schedule_id, interval_info)
# Add tag for each job
job.tag(schedule_id)
# Sync previous scheduler
else:
previous_interval_info = self.schedule_info[schedule_id]
previous_interval = previous_interval_info['interval']
if interval_value != previous_interval:
_LOGGER.debug(
f'[_check_interval] modify {schedule_id} interval {previous_interval} to {interval_value}'
)
# delete job and re-create
schedule.default_scheduler.clear(tag=schedule_id)
job = schedule.every(interval_value).seconds.do(
self.push_task, schedule_id, interval_info)
job.tag(schedule_id)
# Update self.schedule_info
self.schedule_info[schedule_id] = interval_info
else:
_LOGGER.debug(
f'[_check_interval] continue {schedule_id}, at every {previous_interval}'
)
except Exception as e:
_LOGGER.error(f'[_check_interval] contact to developer {e}')
# Delete garbage
_LOGGER.debug(
f'[_check_interval] gargabe collector: {len(schedule.default_scheduler.jobs) - 1}'
)
for job in schedule.default_scheduler.jobs:
if job.tags == set():
continue
exist = False
for schedule_id in schedule_ids:
if schedule_id in job.tags:
print(f'exist: {schedule_id}')
exist = True
break
if exist == False:
# This Job is gargage
_LOGGER.debug(f'[_check_interval] remove job: {job}')
schedule.default_scheduler.cancel_job(job)
def _get_interval_schedules(self):
""" Find all interval schedules from inventory.Collector.schedule
"""
schedule_vos = self._list_schedules()
found_schedules = {}
for schedule_vo in schedule_vos:
try:
interval_value = _get_interval_value(schedule_vo)
found_schedules.update({
schedule_vo.schedule_id: {
'interval': interval_value,
'domain_id': schedule_vo.domain_id,
'collector_id': schedule_vo.collector.collector_id
}
})
except Exception as e:
_LOGGER.error(f'[_get_interval_schedules] {e}')
_LOGGER.debug(f'[_get_interval_schedules] found: {found_schedules}')
return found_schedules
def _list_schedules(self):
try:
# Loop all domain, then find scheduled collector
collector_svc = self.locator.get_service('CollectorService')
schedule = {'interval': None}
schedule_vos, total = collector_svc.scheduled_collectors(
{'schedule': schedule})
return schedule_vos
except Exception as e:
_LOGGER.error(e)
return []
def _create_job_request(self, schedule_id, interval_info):
""" Based on scheduler_vo, create Job Request
Args:
scheduler_vo: Scheduler VO
- scheduler_id
- name
- collector: Reference of Collector
- schedule
- filter
- collector_mode
- created_at
- last_scheduled_at
- domain_id
}
Returns:
jobs: SpaceONE Pipeline Template
"""
_LOGGER.debug(f'[_create_job_request] {interval_info}')
domain_id = interval_info['domain_id']
metadata = {
'token': self.TOKEN,
'service': 'inventory',
'resource': 'Collector',
'verb': 'collect',
'authorization': True,
'domain_id': self.domain_id
}
sched_job = {
'locator': 'SERVICE',
'name': 'CollectorService',
'metadata': metadata,
'method': 'collect',
'params': {
'params': {
'collector_id': interval_info['collector_id'],
# if filter
# contact credential
'collect_mode': 'ALL',
'filter': {},
'domain_id': domain_id
}
}
}
stp = {
'name': 'inventory_collect_by_interval_schedule',
'version': 'v1',
'executionEngine': 'BaseWorker',
'stages': [sched_job]
}
#_LOGGER.debug(f'[_create_job_request] tasks: {stp}')
return stp
def __init__(self, queue, **kwargs):
BaseWorker.__init__(self, queue, **kwargs)
self.locator = Locator()
class BaseAPI(object):
locator = Locator()
def __init__(self):
self._check_variables()
self._set_grpc_method()
@property
def name(self):
return self.__class__.__name__
@property
def pb2_grpc_module(self):
return self.pb2_grpc
@property
def service_name(self):
return self.pb2.DESCRIPTOR.services_by_name[
self.__class__.__name__].full_name
def _check_variables(self):
if not hasattr(self, 'pb2'):
raise Exception(
f'gRPC Servicer has not set <pb2> variable. (servicer={self.__class__.__name__})'
)
if not hasattr(self, 'pb2_grpc'):
raise Exception(
f'gRPC Servicer has not set <pb2_grpc> variable. (servicer={self.__class__.__name__})'
)
def _get_grpc_servicer(self):
grpc_servicer = None
for base_class in self.__class__.__bases__:
if base_class.__module__ == self.pb2_grpc.__name__:
grpc_servicer = base_class
if grpc_servicer is None:
raise Exception(
f'gRPC servicer is not set. (servicer={self.__class__.__name__})'
)
return grpc_servicer
def _set_grpc_method(self):
grpc_servicer = self._get_grpc_servicer()
for f_name, f_object in inspect.getmembers(
self.__class__, predicate=inspect.isfunction):
if hasattr(grpc_servicer, f_name):
setattr(self, f_name,
self._grpc_method(f_object, config.get_service()))
@staticmethod
def get_minimal(params):
return params.get('query', {}).get('minimal', False)
@staticmethod
def _error_method(error, context):
is_logging = False
if not isinstance(error, ERROR_BASE):
error = ERROR_UNKNOWN(message=error)
is_logging = True
elif error.meta.get('type') == 'service':
is_logging = True
if is_logging:
_LOGGER.error(f'(Error) => {error.message} {error}',
extra={
'error_code': error.error_code,
'error_message': error.message,
'traceback': traceback.format_exc()
})
details = f'{error.error_code}: {error.message}'
context.abort(grpc.StatusCode[error.status_code], details)
def _generate_response(self, response_iterator, context):
try:
for response in response_iterator:
yield response
except Exception as e:
self._error_method(e, context)
def _grpc_method(self, func, service_name):
def wrapper(request_or_iterator, context):
try:
context.api_info = {
'service': service_name,
'resource': self.__class__.__name__,
'verb': func.__name__
}
response_or_iterator = func(self, request_or_iterator, context)
if isinstance(response_or_iterator, types.GeneratorType):
return self._generate_response(response_or_iterator,
context)
else:
return response_or_iterator
except Exception as e:
self._error_method(e, context)
return wrapper
@staticmethod
def _convert_message(request):
return MessageToDict(request, preserving_proto_field_name=True)
@staticmethod
def _get_metadata(context):
metadata = {}
for key, value in context.invocation_metadata():
metadata[key.strip()] = value.strip()
metadata.update(context.api_info)
# TODO: This is experimental log. Please confirm peer information is useful on k8s.
metadata.update({'peer': context.peer()})
return metadata
def _generate_message(self, request_iterator):
for request in request_iterator:
yield self._convert_message(request)
def parse_request(self, request_or_iterator, context):
if isinstance(request_or_iterator, collections.Iterable):
return self._generate_message(
request_or_iterator), self._get_metadata(context)
else:
return self._convert_message(
request_or_iterator), self._get_metadata(context)
def __init__(self):
locator = Locator()
self.actuator = locator.get_actuator('ServerInfo')
with open(config_path, 'r') as f:
_set_file_config(f)
def print_schedules(schedules: List[SpaceoneTaskData]):
print(f"id | schedule | total_run_count | last_run")
for sch in schedules:
print(
f"{sch.schedule_id} | {sch.schedule_info} | {sch.total_run_count} | {sch.last_run_at}"
)
print('\n\n')
if __name__ == '__main__':
config_server()
locator = Locator()
svc = locator.get_service('ScheduleService')
print('list all schedules')
print_schedules(svc.list())
print('add schedule')
sch_name = f"test_sche_{randint(1, 1000)}"
svc.add({
'domain_id': "sample",
'enabled': True,
'task': 'spaceone.core.celery.tasks.test_task',
'name': sch_name,
'interval': {
'period': 'seconds',
'every': randint(6, 12)
},
