def work(self):
self.create_base()
parts = self.generate_parts(self.count())
for preds in self.solve(self.key(), parts):
score = self.score(preds)
if self.layout and self.plot_count > 0:
self.plot(preds, score)
score = self.score_final()
if isnan(score):
score = -1
else:
score = float(score)
if self.layout:
self.plot_final(score)
self.task.score = score
self.task_provider.update()
if self.model_id:
provider = ModelProvider(self.session)
model = provider.by_id(self.model_id)
model.score_local = score
provider.commit()
def dag_model_start(session: Session, data: dict):
provider = ModelProvider(session)
model = provider.by_id(data['model_id'])
dag_provider = DagProvider(session)
dag = dag_provider.by_id(data['dag'], joined_load=[Dag.project_rel])
project = dag.project_rel
src_config = Config.from_yaml(dag.config)
pipe = src_config['pipes'][data['pipe']['name']]
equations = yaml_load(model.equations)
versions = data['pipe']['versions']
if len(versions) > 0:
version = data['pipe']['version']
pipe_equations = yaml_load(version['equations'])
found_version = versions[0]
for v in versions:
if v['name'] == version['name']:
found_version = v
break
found_version['used'] = now()
if len(pipe) == 1:
pipe[list(pipe)[0]].update(pipe_equations)
else:
pipe.update(pipe_equations)
equations[data['pipe']['name']] = versions
model.equations = yaml_dump(equations)
for v in pipe.values():
v['model_id'] = model.id
v['model_name'] = model.name
config = {
'info': {
'name': data['pipe']['name'],
'project': project.name
},
'executors': pipe
}
if model.dag:
old_dag = dag_provider.by_id(model.dag)
if old_dag.name != dag.name:
model.dag = dag.id
else:
model.dag = dag.id
provider.commit()
dag_standard(session=session,
config=config,
debug=False,
upload_files=False,
copy_files_from=data['dag'])
def work(self):
self.message = f'Task id = {self.task.id}'
prob = np.load(self.prob_file)
argmax = prob.argmax(axis=1)
pd.DataFrame({
'ImageId': np.arange(1, len(argmax) + 1),
'Label': argmax
}).to_csv(self.out_file, index=False)
score = super().work()
if self.model_id:
provider = ModelProvider(self.session)
model = provider.by_id(self.model_id)
model.score_public = score
provider.commit()
def dag_model_start(session: Session, data: dict):
provider = ModelProvider(session)
model = provider.by_id(data['model_id'])
dag = DagProvider(session
).by_id(data['dag'], joined_load=[Dag.project_rel])
project = dag.project_rel
src_config = Config.from_yaml(dag.config)
pipe = src_config['pipes'][data['pipe']]
for k, v in pipe.items():
if v.get('slot') != data['slot']:
continue
params = yaml_load(data['interface_params'])
slot = {
'interface': data['interface'],
'interface_params': params,
'slot': k,
'name': model.name,
'id': data['model_id']
}
v['slot'] = slot
config = {
'info': {
'name': data['pipe'],
'project': project.name
},
'executors': pipe
}
dag_standard(
session=session,
config=config,
debug=False,
upload_files=False,
copy_files_from=data['dag']
)
model.dag = data['dag']
model.interface = data['interface']
model.interface_params = data['interface_params']
model.slot = data['slot']
provider.commit()
def work(self):
submissions = api.competition_submissions(self.competition)
submission_refs = {s.ref for s in submissions}
if self.submit_type == 'file':
self.file_submit()
else:
self.kernel_submit()
self.info('waiting for the submission on Kaggle')
step = 10
for i in range(int(self.wait_seconds // step)):
try:
submissions = api.competition_submissions(self.competition)
for s in submissions:
if s.ref not in submission_refs:
if s.status == 'complete':
if s.publicScore is None:
raise Exception('Submission is complete, '
'but publicScore is None')
score = float(s.publicScore)
if self.model_id:
provider = ModelProvider(self.session)
model = provider.by_id(self.model_id)
model.score_public = score
provider.commit()
return {'res': score}
elif s.status == 'error':
raise Exception(f'Submission error '
f'on Kaggle: {s.errorDescription}')
break
except TypeError:
pass
time.sleep(step)
raise Exception(f'Submission is not '
f'complete after {self.wait_seconds}')