DagProvider, DagStorageProvider, TaskProvider, LogProvider, StepProvider, \
FileProvider, AuxiliaryProvider
from mlcomp.db.report_info import ReportLayoutInfo
from mlcomp.server.back.supervisor import register_supervisor
from mlcomp.utils.logging import create_logger
from mlcomp.utils.io import from_module_path, zip_folder
from mlcomp.server.back.create_dags import dag_model_add, dag_model_start
from mlcomp.utils.misc import to_snake, now
from mlcomp.db.models import Model, Report, ReportLayout, Task
from mlcomp.utils.io import yaml_load, yaml_dump
from mlcomp.worker.storage import Storage
app = Flask(__name__)
CORS(app)
_read_session = Session.create_session(key='server.read')
_write_session = Session.create_session(key='server.write')
logger = create_logger(_write_session, __name__)
@app.route('/', defaults={'path': ''}, methods=['GET'])
@app.route('/<path:path>', methods=['GET'])
def send_static(path):
file = 'index.html'
if '.' in path:
file = path
module_path = from_module_path(__file__, f'../front/dist/mlcomp/')
return send_from_directory(module_path, file)
DOCKER_IMG, DOCKER_MAIN, IP, PORT, WORKER_USAGE_INTERVAL, \
SYNC_WITH_THIS_COMPUTER, CAN_PROCESS_TASKS
from mlcomp.db.core import Session
from mlcomp.db.enums import ComponentType, TaskStatus
from mlcomp.utils.logging import create_logger
from mlcomp.db.providers import DockerProvider, TaskProvider
from mlcomp.utils.schedule import start_schedule
from mlcomp.utils.misc import dict_func, now, disk, get_username, \
kill_child_processes
from mlcomp.worker.app import app
from mlcomp.db.providers import ComputerProvider
from mlcomp.db.models import ComputerUsage, Computer, Docker
from mlcomp.utils.misc import memory
from mlcomp.worker.sync import FileSync
_session = Session.create_session(key='worker')
@click.group()
def main():
pass
def error_handler(f):
name = f.__name__
wrapper_vars = {'session': Session.create_session(key=name)}
wrapper_vars['logger'] = create_logger(wrapper_vars['session'], name)
hostname = socket.gethostname()
def wrapper():
import time
import socket
from mlcomp.db.core import Session
from mlcomp.db.enums import ComponentType
from mlcomp.db.providers import ModelProvider
from mlcomp.worker.executors.base.equation import Equation
from mlcomp.worker.executors.base.executor import Executor
from mlcomp.utils.logging import create_logger
from mlcomp.utils.config import Config
try:
from kaggle import api
except OSError:
logger = create_logger(Session.create_session(), __name__)
logger.warning(
'Could not find kaggle.json. '
'Kaggle executors can not be used', ComponentType.Worker,
socket.gethostname())
class DownloadType(Enum):
Kaggle = 0
Link = 1
@Executor.register
class Download(Executor):
def __init__(
self,
from mlcomp.db.models import Computer
from mlcomp.db.providers import \
ComputerProvider, \
TaskProvider, \
StepProvider, \
ProjectProvider
from mlcomp.migration.manage import migrate
from mlcomp.utils.config import merge_dicts_smart, dict_from_list_str
from mlcomp.utils.io import yaml_load, yaml_dump
from mlcomp.utils.logging import create_logger
from mlcomp.worker.sync import sync_directed
from mlcomp.worker.tasks import execute_by_id
from mlcomp.utils.misc import memory, disk, get_username
from mlcomp.server.back.create_dags import dag_standard, dag_pipe
_session = Session.create_session(key=__name__)
def _dag(config: str,
debug: bool = False,
control_reqs=True,
params: Tuple[str] = ()):
migrate()
config_text = open(config, 'r').read()
config_parsed = yaml_load(config_text)
params = dict_from_list_str(params)
config_parsed = merge_dicts_smart(config_parsed, params)
config_text = yaml_dump(config_parsed)
type_name = config_parsed['info'].get('type', 'standard')
class FileSync:
session = Session.create_session(key='FileSync')
logger = create_logger(session, 'FileSync')
def sync_manual(self, computer: Computer, provider: ComputerProvider):
"""
button sync was clicked manually
"""
if not computer.meta:
return
meta = yaml_load(computer.meta)
if 'manual_sync' not in meta:
return
manual_sync = meta['manual_sync']
project_provider = ProjectProvider(self.session)
docker_provider = DockerProvider(self.session)
dockers = docker_provider.get_online()
project = project_provider.by_id(manual_sync['project'])
for docker in dockers:
if docker.computer == computer.name:
continue
source = provider.by_name(docker.computer)
ignore_folders = [
[join('models', project.name), []]
]
sync_directed(self.session, target=computer, source=source,
ignore_folders=ignore_folders)
del meta['manual_sync']
computer.meta = yaml_dump(meta)
provider.update()
def sync(self):
hostname = socket.gethostname()
try:
provider = ComputerProvider(self.session)
task_synced_provider = TaskSyncedProvider(self.session)
computer = provider.by_name(hostname)
sync_start = now()
if FILE_SYNC_INTERVAL == 0:
time.sleep(1)
else:
self.sync_manual(computer, provider)
computers = provider.all_with_last_activtiy()
computers = [
c for c in computers
if (now() - c.last_activity).total_seconds() < 10
]
computers_names = {c.name for c in computers}
for c, project, tasks in task_synced_provider.for_computer(
computer.name):
if c.sync_with_this_computer:
if c.name not in computers_names:
self.logger.info(f'Computer = {c.name} '
f'is offline. Can not sync',
ComponentType.WorkerSupervisor,
hostname)
continue
if c.syncing_computer:
continue
ignore_folders = [
[join('models', project.name), []]
]
computer.syncing_computer = c.name
provider.update()
sync_directed(self.session, c, computer,
ignore_folders)
for t in tasks:
task_synced_provider.add(
TaskSynced(computer=computer.name, task=t.id)
)
time.sleep(FILE_SYNC_INTERVAL)
computer.last_synced = sync_start
computer.syncing_computer = None
provider.update()
except Exception as e:
if Session.sqlalchemy_error(e):
Session.cleanup('FileSync')
self.session = Session.create_session(key='FileSync')
self.logger = create_logger(self.session, 'FileSync')
self.logger.error(
traceback.format_exc(), ComponentType.WorkerSupervisor,
hostname
)
def sync(self):
hostname = socket.gethostname()
try:
provider = ComputerProvider(self.session)
task_synced_provider = TaskSyncedProvider(self.session)
computer = provider.by_name(hostname)
sync_start = now()
if FILE_SYNC_INTERVAL == 0:
time.sleep(1)
else:
self.sync_manual(computer, provider)
computers = provider.all_with_last_activtiy()
computers = [
c for c in computers
if (now() - c.last_activity).total_seconds() < 10
]
computers_names = {c.name for c in computers}
for c, project, tasks in task_synced_provider.for_computer(
computer.name):
if c.sync_with_this_computer:
if c.name not in computers_names:
self.logger.info(f'Computer = {c.name} '
f'is offline. Can not sync',
ComponentType.WorkerSupervisor,
hostname)
continue
if c.syncing_computer:
continue
ignore_folders = [
[join('models', project.name), []]
]
computer.syncing_computer = c.name
provider.update()
sync_directed(self.session, c, computer,
ignore_folders)
for t in tasks:
task_synced_provider.add(
TaskSynced(computer=computer.name, task=t.id)
)
time.sleep(FILE_SYNC_INTERVAL)
computer.last_synced = sync_start
computer.syncing_computer = None
provider.update()
except Exception as e:
if Session.sqlalchemy_error(e):
Session.cleanup('FileSync')
self.session = Session.create_session(key='FileSync')
self.logger = create_logger(self.session, 'FileSync')
self.logger.error(
traceback.format_exc(), ComponentType.WorkerSupervisor,
hostname
)
def downgrade(migrate_engine):
session = Session.create_session(connection_string=migrate_engine.url)
provider = ReportLayoutProvider(session)
provider.session.query(ReportLayout).delete(synchronize_session=False)
provider.session.commit()