def main(unused_argv):
args = FLAGS.flag_values_dict()
if FLAGS.list: args['action'] = 'list'
if FLAGS.delete: args['action'] = 'delete'
if FLAGS.create: args['action'] = 'create'
scheduler = Scheduler()
print(scheduler.process(args).replace('<br/>', '\n'))
def try_update_best_schedule(workflows, machines, best_sch, order):
new_workflows, new_machines = ExampleGen.re_create_example(
workflows, machines)
sch_name = f'ordered {best_sch.name}' if not best_sch.name.startswith(
'ordered') else best_sch.name
ordered_sch = Scheduler(sch_name,
data=[new_workflows, new_machines],
schedule_order=order,
time_types=best_sch.time_types_str,
fill_method=best_sch.fill_method_str,
priority_type=best_sch.priority_type_str)
ordered_sch.run()
if best_sch.schedule_len > ordered_sch.schedule_len:
return ordered_sch, True
return best_sch, False
def __init__(self,
run_methods,
machines,
workflows,
visuals=False,
save_fig=False,
show_fig=True,
save_sim=False,
show_machines=False):
self.run_methods: List[dict] = run_methods
self.machines: List[Machine] = machines
self.workflows: List[Workflow] = workflows
self.visuals: bool = visuals
self.save_fig: bool = save_fig
self.show_machines: bool = show_machines
self.show_fig: bool = show_fig
self.save_sim: bool = save_sim
self.schedulers: List[Scheduler] = [
Scheduler(method['name'],
data=ExampleGen.re_create_example(workflows, machines),
time_types=method.get("time_types"),
fill_method=method["fill_type"],
priority_type=method.get("priority_type"))
for method in run_methods
]
def _fetch_schedule(self, type: Type,
run_config: Dict[str, Any]) -> Dict[str, Any]:
scheduler = Scheduler()
(success, job_config) = scheduler.process({
'action':
'get',
'project':
os.environ['GCP_PROJECT'],
'email':
run_config['email'],
'html':
False,
'job_id':
type.runner(run_config['report_id'])
})
return job_config
def main(unusedargv):
scheduler = Scheduler()
with open(FLAGS.file) as reports:
runners = json.loads(''.join(reports.readlines()))
for runner in runners:
id = f"{runner['report']}_{runner['AgencyId']}_{runner['AdvertiserId']}"
Firestore().update_document(Type.SA360_RPT, f'{id}', runner)
def scheduler(self) -> Scheduler:
"""Lazy init for the scheduler
Returns None if no API Key is provided, which just means that no scheduler
functionality will be available.
Returns:
Scheduler: The scheduler
"""
return Scheduler() if 'API_KEY' in os.environ else None
def _fetch_schedule(self, type: Type,
run_config: Dict[str, Any]) -> Dict[str, Any]:
"""Fetches the job schedule.
Args:
type (Type): the typ(product) of the job.
run_config (Dict[str, Any]): the configuration.
Returns:
Dict[str, Any]: the schedule details.
"""
scheduler = Scheduler()
return scheduler.process(
**{
'action': 'get',
'project': os.environ['GCP_PROJECT'],
'email': run_config.get('email'),
'html': False,
'job_id': type.runner(run_config.get('report_id'))
})
def run(self):
# Get the order of the tasks.
# scheduled_tasks: Iterable[Task] = sorted(self.og_workflow.tasks, key=lambda t: t.start)
# Since we give just one workflow to copy there is only one in the list
is_better = False
while not is_better:
workflows, machines = ExampleGen.re_create_example(
[self.og_workflow], self.og_machines, reset_task_status=True)
workflow = workflows[0]
# Ruin part
if self.ruin_method == "random":
ruined_tasks_ids = [(t.id, t.machine_id)
for t in self.og_workflow.tasks
if random.randint(0, 10) < 5]
elif self.ruin_method == "time-based":
ruined_tasks_ids = [
(t.id, t.machine_id) for t in self.og_workflow.tasks
if self.time_space[0] < t.start < self.time_space[1]
]
else:
raise Exception("Unknown ruin method")
# Recreate part
for task in workflow.tasks:
if m_id := [
r_info[1] for r_info in ruined_tasks_ids
if r_info[0] == task.id
]:
Scheduler.schedule_task_machine(
task, [m for m in machines if m.id != m_id[0]],
TimeType.EFT)
if task.status != TaskStatus.SCHEDULED:
Scheduler.schedule_task_machine(task, machines,
TimeType.EFT)
workflow.set_scheduled(True)
if workflow.wf_len < self.og_workflow.wf_len:
is_better = True
def delete(self, firestore: Firestore, project: str, report: str,
email: str, **unused):
firestore.delete_document(Type.SA360_RPT, '_reports', report)
scheduler = Scheduler()
args = {
'action': 'list',
'email': email,
'project': project,
'html': False,
}
# Disable all runners for the now deleted report
runners = list(runner['name'].split('/')[-1]
for runner in scheduler.process(args)
if report in runner['name'])
for runner in runners:
args = {
'action': 'disable',
'email': None,
'project': project,
'job_id': runner,
}
scheduler.process(args)
def index() -> jinja2.Template:
"""The index method for the appengine.
Returns:
Template: The completed html template
"""
project = os.environ['GOOGLE_CLOUD_PROJECT']
bucket = f'{project}-report2bq-tokens'
user_email, user_id = user()
data = {}
creds = Credentials(project=project, email=user_email)
try:
template = JINJA_ENVIRONMENT.get_template('index.html')
running_jobs = Scheduler().process(**{
'action': 'list',
'project': project,
'email': user_email
})
jobs = []
for job in running_jobs:
with suppress(ValueError, KeyError, TypeError):
_attrs = job.get('pubsubTarget', {}).get('attributes', {})
_def = Type(_attrs['type'])
j = {
'id': job['name'].split('/')[-1],
'description': job['description']
if 'description' in job else '-- No description given --',
'type': _def,
'schedule': job['schedule'],
'timezone': job['timeZone'],
}
j['attributes'] = switch(_def, _attrs)
jobs.append(j)
data = {'jobs': jobs, 'user_email': user_email}
except CredentialsError as e:
template = JINJA_ENVIRONMENT.get_template('authenticate.html')
data = {
'email': user_email,
'client_id': creds.project_credentials.client_id,
}
return template.render(data)
def index():
project = os.environ['GOOGLE_CLOUD_PROJECT']
bucket = f'{project}-report2bq-tokens'
project_credentials = json.loads(OAuth.fetch_file(
bucket,
'client_secrets.json'
), encoding='utf-8')
user_email, user_id = user()
client = storage.Client(credentials=None)
has_auth = client.get_bucket(bucket).get_blob(f'{user_email}_user_token.json')
data = {}
if has_auth:
template = JINJA_ENVIRONMENT.get_template('index.html')
running_jobs = Scheduler().process(args={'action': 'list', 'project': project, 'email': user_email})
jobs = []
for job in running_jobs:
with suppress(ValueError, KeyError):
_attrs = job['pubsubTarget']['attributes']
_def = Type(_attrs['type'])
j = {
'id': job['name'].split('/')[-1],
'description': job['description'] if 'description' in job else '-- No description given --',
'type': _def,
'schedule': job['schedule'],
'timezone': job['timeZone'],
}
j['attributes'] = switch(_def, _attrs)
jobs.append(j)
data = {'jobs': jobs, 'user_email': user_email}
else:
template = JINJA_ENVIRONMENT.get_template('authenticate.html')
data = {
'email': user_email,
'client_id': project_credentials['web']['client_id'],
}
return template.render(data)