DagProvider, DagStorageProvider, TaskProvider, LogProvider, StepProvider, \ FileProvider, AuxiliaryProvider from mlcomp.db.report_info import ReportLayoutInfo from mlcomp.server.back.supervisor import register_supervisor from mlcomp.utils.logging import create_logger from mlcomp.utils.io import from_module_path, zip_folder from mlcomp.server.back.create_dags import dag_model_add, dag_model_start from mlcomp.utils.misc import to_snake, now from mlcomp.db.models import Model, Report, ReportLayout, Task from mlcomp.utils.io import yaml_load, yaml_dump from mlcomp.worker.storage import Storage app = Flask(__name__) CORS(app) _read_session = Session.create_session(key='server.read') _write_session = Session.create_session(key='server.write') logger = create_logger(_write_session, __name__) @app.route('/', defaults={'path': ''}, methods=['GET']) @app.route('/<path:path>', methods=['GET']) def send_static(path): file = 'index.html' if '.' in path: file = path module_path = from_module_path(__file__, f'../front/dist/mlcomp/') return send_from_directory(module_path, file)
DOCKER_IMG, DOCKER_MAIN, IP, PORT, WORKER_USAGE_INTERVAL, \ SYNC_WITH_THIS_COMPUTER, CAN_PROCESS_TASKS from mlcomp.db.core import Session from mlcomp.db.enums import ComponentType, TaskStatus from mlcomp.utils.logging import create_logger from mlcomp.db.providers import DockerProvider, TaskProvider from mlcomp.utils.schedule import start_schedule from mlcomp.utils.misc import dict_func, now, disk, get_username, \ kill_child_processes from mlcomp.worker.app import app from mlcomp.db.providers import ComputerProvider from mlcomp.db.models import ComputerUsage, Computer, Docker from mlcomp.utils.misc import memory from mlcomp.worker.sync import FileSync _session = Session.create_session(key='worker') @click.group() def main(): pass def error_handler(f): name = f.__name__ wrapper_vars = {'session': Session.create_session(key=name)} wrapper_vars['logger'] = create_logger(wrapper_vars['session'], name) hostname = socket.gethostname() def wrapper():
import time import socket from mlcomp.db.core import Session from mlcomp.db.enums import ComponentType from mlcomp.db.providers import ModelProvider from mlcomp.worker.executors.base.equation import Equation from mlcomp.worker.executors.base.executor import Executor from mlcomp.utils.logging import create_logger from mlcomp.utils.config import Config try: from kaggle import api except OSError: logger = create_logger(Session.create_session(), __name__) logger.warning( 'Could not find kaggle.json. ' 'Kaggle executors can not be used', ComponentType.Worker, socket.gethostname()) class DownloadType(Enum): Kaggle = 0 Link = 1 @Executor.register class Download(Executor): def __init__( self,
from mlcomp.db.models import Computer from mlcomp.db.providers import \ ComputerProvider, \ TaskProvider, \ StepProvider, \ ProjectProvider from mlcomp.migration.manage import migrate from mlcomp.utils.config import merge_dicts_smart, dict_from_list_str from mlcomp.utils.io import yaml_load, yaml_dump from mlcomp.utils.logging import create_logger from mlcomp.worker.sync import sync_directed from mlcomp.worker.tasks import execute_by_id from mlcomp.utils.misc import memory, disk, get_username from mlcomp.server.back.create_dags import dag_standard, dag_pipe _session = Session.create_session(key=__name__) def _dag(config: str, debug: bool = False, control_reqs=True, params: Tuple[str] = ()): migrate() config_text = open(config, 'r').read() config_parsed = yaml_load(config_text) params = dict_from_list_str(params) config_parsed = merge_dicts_smart(config_parsed, params) config_text = yaml_dump(config_parsed) type_name = config_parsed['info'].get('type', 'standard')
class FileSync: session = Session.create_session(key='FileSync') logger = create_logger(session, 'FileSync') def sync_manual(self, computer: Computer, provider: ComputerProvider): """ button sync was clicked manually """ if not computer.meta: return meta = yaml_load(computer.meta) if 'manual_sync' not in meta: return manual_sync = meta['manual_sync'] project_provider = ProjectProvider(self.session) docker_provider = DockerProvider(self.session) dockers = docker_provider.get_online() project = project_provider.by_id(manual_sync['project']) for docker in dockers: if docker.computer == computer.name: continue source = provider.by_name(docker.computer) ignore_folders = [ [join('models', project.name), []] ] sync_directed(self.session, target=computer, source=source, ignore_folders=ignore_folders) del meta['manual_sync'] computer.meta = yaml_dump(meta) provider.update() def sync(self): hostname = socket.gethostname() try: provider = ComputerProvider(self.session) task_synced_provider = TaskSyncedProvider(self.session) computer = provider.by_name(hostname) sync_start = now() if FILE_SYNC_INTERVAL == 0: time.sleep(1) else: self.sync_manual(computer, provider) computers = provider.all_with_last_activtiy() computers = [ c for c in computers if (now() - c.last_activity).total_seconds() < 10 ] computers_names = {c.name for c in computers} for c, project, tasks in task_synced_provider.for_computer( computer.name): if c.sync_with_this_computer: if c.name not in computers_names: self.logger.info(f'Computer = {c.name} ' f'is offline. Can not sync', ComponentType.WorkerSupervisor, hostname) continue if c.syncing_computer: continue ignore_folders = [ [join('models', project.name), []] ] computer.syncing_computer = c.name provider.update() sync_directed(self.session, c, computer, ignore_folders) for t in tasks: task_synced_provider.add( TaskSynced(computer=computer.name, task=t.id) ) time.sleep(FILE_SYNC_INTERVAL) computer.last_synced = sync_start computer.syncing_computer = None provider.update() except Exception as e: if Session.sqlalchemy_error(e): Session.cleanup('FileSync') self.session = Session.create_session(key='FileSync') self.logger = create_logger(self.session, 'FileSync') self.logger.error( traceback.format_exc(), ComponentType.WorkerSupervisor, hostname )
def sync(self): hostname = socket.gethostname() try: provider = ComputerProvider(self.session) task_synced_provider = TaskSyncedProvider(self.session) computer = provider.by_name(hostname) sync_start = now() if FILE_SYNC_INTERVAL == 0: time.sleep(1) else: self.sync_manual(computer, provider) computers = provider.all_with_last_activtiy() computers = [ c for c in computers if (now() - c.last_activity).total_seconds() < 10 ] computers_names = {c.name for c in computers} for c, project, tasks in task_synced_provider.for_computer( computer.name): if c.sync_with_this_computer: if c.name not in computers_names: self.logger.info(f'Computer = {c.name} ' f'is offline. Can not sync', ComponentType.WorkerSupervisor, hostname) continue if c.syncing_computer: continue ignore_folders = [ [join('models', project.name), []] ] computer.syncing_computer = c.name provider.update() sync_directed(self.session, c, computer, ignore_folders) for t in tasks: task_synced_provider.add( TaskSynced(computer=computer.name, task=t.id) ) time.sleep(FILE_SYNC_INTERVAL) computer.last_synced = sync_start computer.syncing_computer = None provider.update() except Exception as e: if Session.sqlalchemy_error(e): Session.cleanup('FileSync') self.session = Session.create_session(key='FileSync') self.logger = create_logger(self.session, 'FileSync') self.logger.error( traceback.format_exc(), ComponentType.WorkerSupervisor, hostname )
def downgrade(migrate_engine): session = Session.create_session(connection_string=migrate_engine.url) provider = ReportLayoutProvider(session) provider.session.query(ReportLayout).delete(synchronize_session=False) provider.session.commit()