Exemplo n.º 1
0
def json_output():
    chia_location, log_directory, config_jobs, manager_check_interval, max_concurrent, max_for_phase_1, \
        minutes_between_jobs, progress_settings, notification_settings, debug_level, view_settings, \
        instrumentation_settings = get_config_info()

    system_drives = get_system_drives()

    drives = {'temp': [], 'temp2': [], 'dest': []}
    jobs = load_jobs(config_jobs)
    for job in jobs:
        drive = job.temporary_directory.split('\\')[0]
        drives['temp'].append(drive)
        directories = {
            'dest': job.destination_directory,
            'temp2': job.temporary2_directory,
        }
        for key, directory_list in directories.items():
            if directory_list is None:
                continue
            if not isinstance(directory_list, list):
                directory_list = [directory_list]
            for directory in directory_list:
                drive = identify_drive(file_path=directory,
                                       drives=system_drives)
                if drive in drives[key]:
                    continue
                drives[key].append(drive)

    running_work = {}

    jobs = load_jobs(config_jobs)
    jobs, running_work = get_running_plots(
        jobs=jobs,
        running_work=running_work,
        instrumentation_settings=instrumentation_settings)
    check_log_progress(jobs=jobs,
                       running_work=running_work,
                       progress_settings=progress_settings,
                       notification_settings=notification_settings,
                       view_settings=view_settings,
                       instrumentation_settings=instrumentation_settings)
    print_json(jobs=jobs,
               running_work=running_work,
               view_settings=view_settings)

    has_file = False
    if len(running_work.values()) == 0:
        has_file = True
    for work in running_work.values():
        if not work.log_file:
            continue
        has_file = True
        break
    if not has_file:
        print("Restarting view due to psutil going stale...")
        system_args = [f'"{sys.executable}"'] + sys.argv
        os.execv(sys.executable, system_args)
    exit()
Exemplo n.º 2
0
def view(loop=True):
    chia_location, log_directory, config_jobs, manager_check_interval, max_concurrent, max_for_phase_1, \
        minimum_minutes_between_jobs, progress_settings, notification_settings, debug_level, view_settings, \
        instrumentation_settings, backend = get_config_info()
    view_check_interval = view_settings['check_interval']
    system_drives = get_system_drives()
    analysis = {'files': {}}
    drives = {'temp': [], 'temp2': [], 'dest': []}
    jobs = load_jobs(config_jobs)
    for job in jobs:
        directories = {
            'dest': job.destination_directory,
            'temp': job.temporary_directory,
            'temp2': job.temporary2_directory,
        }
        for key, directory_list in directories.items():
            if directory_list is None:
                continue
            if not isinstance(directory_list, list):
                directory_list = [directory_list]
            for directory in directory_list:
                drive = identify_drive(file_path=directory, drives=system_drives)
                if drive in drives[key]:
                    continue
                drives[key].append(drive)

    while True:
        running_work = {}
        try:
            analysis = analyze_log_dates(log_directory=log_directory, analysis=analysis)
            jobs = load_jobs(config_jobs)
            jobs, running_work = get_running_plots(jobs=jobs, running_work=running_work,
                                                   instrumentation_settings=instrumentation_settings, backend=backend)
            check_log_progress(jobs=jobs, running_work=running_work, progress_settings=progress_settings,
                               notification_settings=notification_settings, view_settings=view_settings,
                               instrumentation_settings=instrumentation_settings, backend=backend)
            print_view(jobs=jobs, running_work=running_work, analysis=analysis, drives=drives,
                       next_log_check=datetime.now() + timedelta(seconds=view_check_interval),
                       view_settings=view_settings, loop=loop, backend=backend)
            if not loop:
                break
            time.sleep(view_check_interval)
            has_file = False
            if len(running_work.values()) == 0:
                has_file = True
            for work in running_work.values():
                if not work.log_file:
                    continue
                has_file = True
                break
            if not has_file:
                print("Restarting view due to psutil going stale...")
                system_args = ['python'] + sys.argv
                os.execv(sys.executable, system_args)
        except KeyboardInterrupt:
            print("Stopped view.")
            exit()
Exemplo n.º 3
0
def view():
    chia_location, log_directory, config_jobs, manager_check_interval, max_concurrent, progress_settings, \
        notification_settings, debug_level, view_settings = get_config_info()
    view_check_interval = view_settings['check_interval']
    analysis = {'files': {}}
    drives = {'temp': [], 'temp2': [], 'dest': []}
    jobs = load_jobs(config_jobs)
    for job in jobs:
        drive = job.temporary_directory.split('\\')[0]
        drives['temp'].append(drive)
        directories = {
            'dest': job.destination_directory,
            'temp2': job.temporary2_directory,
        }
        for key, directory_list in directories.items():
            if directory_list is None:
                continue
            if isinstance(directory_list, list):
                for directory in directory_list:
                    drive = directory.split('\\')[0]
                    if drive in drives[key]:
                        continue
                    drives[key].append(drive)
            else:
                drive = directory_list.split('\\')[0]
                if drive in drives[key]:
                    continue
                drives[key].append(drive)

    while True:
        running_work = {}
        try:
            analysis = analyze_log_dates(log_directory=log_directory, analysis=analysis)
            jobs = load_jobs(config_jobs)
            jobs, running_work = get_running_plots(jobs=jobs, running_work=running_work)
            check_log_progress(jobs=jobs, running_work=running_work, progress_settings=progress_settings,
                               notification_settings=notification_settings, view_settings=view_settings)
            print_view(jobs=jobs, running_work=running_work, analysis=analysis, drives=drives,
                       next_log_check=datetime.now() + timedelta(seconds=60), view_settings=view_settings)
            time.sleep(view_check_interval)
            has_file = False
            if len(running_work.values()) == 0:
                has_file = True
            for work in running_work.values():
                if not work.log_file:
                    continue
                has_file = True
                break
            if not has_file:
                print("Restarting view due to psutil going stale...")
                system_args = [f'"{sys.executable}"'] + sys.argv
                os.execv(sys.executable, system_args)
        except KeyboardInterrupt:
            print("Stopped view.")
            exit()
Exemplo n.º 4
0
                f'Minimum: {minimum_stagger}, Current: {next_job_work[job_name]}'
            )
            continue
        next_job_work[job_name] = minimum_stagger
        logging.info(
            f'Setting a new stagger for {job_name}. minimum_minutes_between_jobs is larger than assigned '
            f'stagger. Minimum: {minimum_stagger}, Current: {next_job_work[job_name]}'
        )

logging.info(f'Starting loop.')
while has_active_jobs_and_work(jobs):
    # CHECK LOGS FOR DELETED WORK
    logging.info(f'Checking log progress..')
    check_log_progress(jobs=jobs,
                       running_work=running_work,
                       progress_settings=progress_settings,
                       notification_settings=notification_settings,
                       view_settings=view_settings,
                       instrumentation_settings=instrumentation_settings)
    next_log_check = datetime.now() + timedelta(seconds=manager_check_interval)

    # DETERMINE IF JOB NEEDS TO START
    logging.info(f'Monitoring jobs to start.')
    jobs, running_work, next_job_work, next_log_check = monitor_jobs_to_start(
        jobs=jobs,
        running_work=running_work,
        max_concurrent=max_concurrent,
        max_for_phase_1=max_for_phase_1,
        next_job_work=next_job_work,
        chia_location=chia_location,
        log_directory=log_directory,
        next_log_check=next_log_check,