Ejemplo n.º 1
0
Archivo: log.py Proyecto: smjurcak/csm
def download_system_logs():
    db_session = DBSession()
    logs = db_session.query(Log) \
        .order_by(Log.created_time.desc())

    contents = ''
    for log in logs:
        contents += get_datetime_string(log.created_time) + ' UTC\n'
        contents += log.level + ':' + log.msg + '\n'
        if log.trace is not None:
            contents += log.trace + '\n'
        contents += '-' * 70 + '\n'

    # Create a file which contains the size of the image file.
    temp_user_dir = create_temp_user_directory(current_user.username)
    log_file_path = os.path.normpath(os.path.join(temp_user_dir, "system_logs"))

    create_directory(log_file_path)
    make_file_writable(log_file_path)

    log_file = open(os.path.join(log_file_path, 'system_logs'), 'w')
    log_file.write(contents)
    log_file.close()

    return send_file(os.path.join(log_file_path, 'system_logs'), as_attachment=True)
Ejemplo n.º 2
0
def api_create_tar_job():
    db_session = DBSession()

    form = CreateTarForm(request.form)

    server_id = request.args.get('server')
    server_directory = request.args.get('server_directory')
    source_tars = request.args.getlist('source_tars[]')
    contents = request.args.getlist('tar_contents[]')
    additional_packages = request.args.getlist('additional_packages[]')
    new_tar_name = request.args.get('new_tar_name').strip('.tar')

    create_tar_job = CreateTarJob(
        server_id = server_id,
        server_directory = server_directory,
        source_tars = (',').join(source_tars),
        contents = (',').join(contents),
        additional_packages = (',').join(additional_packages),
        new_tar_name = new_tar_name,
        created_by = current_user.username,
        status = 'Job Submitted.')

    db_session.add(create_tar_job)
    db_session.commit()

    job_id = create_tar_job.id

    return jsonify({'status': 'OK', 'job_id': job_id})
Ejemplo n.º 3
0
def api_get_scheduled_download_jobs():
    dt_params = DataTableParams(request)
    db_session = DBSession()

    clauses = []
    if len(dt_params.search_value):
        criteria = '%' + dt_params.search_value + '%'
        clauses.append(DownloadJob.cco_filename.like(criteria))
        clauses.append(DownloadJob.scheduled_time.like(criteria))
        clauses.append(DownloadJob.created_by.like(criteria))

    query = db_session.query(DownloadJob)

    total_count = query.filter(DownloadJob.status == None).count()
    filtered_count = query.filter(and_(DownloadJob.status == None), or_(*clauses)).count()

    columns = [getattr(DownloadJob.cco_filename, dt_params.sort_order)(),
               getattr(DownloadJob.scheduled_time, dt_params.sort_order)(),
               '',
               getattr(DownloadJob.created_by, dt_params.sort_order)()]

    download_jobs = query.order_by(columns[dt_params.column_order])\
        .filter(and_(DownloadJob.status == None), or_(*clauses))\
        .slice(dt_params.start_length, dt_params.start_length + dt_params.display_length).all()

    response = dict()
    response['draw'] = dt_params.draw
    response['recordsTotal'] = total_count
    response['recordsFiltered'] = filtered_count
    response.update(get_download_job_json_dict(db_session, download_jobs))

    return jsonify(**response)
Ejemplo n.º 4
0
Archivo: log.py Proyecto: smjurcak/csm
def api_get_session_logs(table):
    id = request.args.get("record_id")

    db_session = DBSession()
    if table == 'install_job':
        install_job = db_session.query(InstallJob).filter(InstallJob.id == id).first()
    elif table == 'install_job_history':
        install_job = db_session.query(InstallJobHistory).filter(InstallJobHistory.id == id).first()
    elif table == 'inventory_job_history':
        install_job = db_session.query(InventoryJobHistory).filter(InventoryJobHistory.id == id).first()

    if install_job is None:
        abort(404)

    log_folder = install_job.session_log
    file_path = os.path.join(get_log_directory(), log_folder)

    if not os.path.isdir(file_path):
        abort(404)

    rows = []
    log_file_list = get_file_list(file_path)
    for file in log_file_list:
        row = dict()
        row['filepath'] = os.path.join(file_path, file)
        row['filename'] = file
        rows.append(row)

    return jsonify(**{'data': rows})
Ejemplo n.º 5
0
def user_create():
    if not can_create_user(current_user):
        abort(401)

    form = UserForm(request.form)
    # Need to add the Required flag back as it is globally removed during user_edit()
    add_validator(form.password, Required)

    fill_user_privileges(form.privilege.choices)

    if request.method == 'POST' and form.validate():
        db_session = DBSession()
        user = get_user(db_session, form.username.data)

        if user is not None:
            return render_template('user/edit.html', form=form, duplicate_error=True)

        user = User(
            username=form.username.data,
            password=form.password.data,
            privilege=form.privilege.data,
            fullname=form.fullname.data,
            email=form.email.data)

        user.preferences.append(Preferences())
        db_session.add(user)
        db_session.commit()

        return redirect(url_for('home'))
    else:
        # Default to Active
        form.active.data = True
        return render_template('user/edit.html', form=form)
Ejemplo n.º 6
0
def api_get_hosts_by_region(region_id, role, software):
    selected_roles = []
    selected_software = []

    if 'ALL' not in role:
        selected_roles = role.split(',')

    if 'ALL' not in software:
        selected_software = software.split(',')

    rows = []
    db_session = DBSession()    

    hosts = db_session.query(Host).filter(Host.region_id == region_id). \
        order_by(Host.hostname.asc())

    for host in hosts:
        host_roles = [] if host.roles is None else host.roles.split(',')
        if not selected_roles or any(role in host_roles for role in selected_roles):
            if host.software_platform is not None and host.software_version is not None:
                host_platform_software = host.software_platform + ' (' + host.software_version + ')'
            else:
                host_platform_software = UNKNOWN

            if not selected_software or host_platform_software in selected_software:
                row = {'hostname': host.hostname,
                       'roles': host.roles,
                       'platform_software': host_platform_software}

                rows.append(row)
    
    return jsonify(**{'data': rows})
Ejemplo n.º 7
0
Archivo: cco.py Proyecto: smjurcak/csm
def api_get_smu_details(smu_id):
    rows = []
    db_session = DBSession()

    smu_info = db_session.query(SMUInfo).filter(SMUInfo.id == smu_id).first()
    if smu_info is not None:
        row = dict()
        row['id'] = smu_info.id
        row['name'] = smu_info.name
        row['status'] = smu_info.status
        row['type'] = smu_info.type
        row['posted_date'] = smu_info.posted_date
        row['ddts'] = smu_info.ddts
        row['description'] = smu_info.description
        row['functional_areas'] = smu_info.functional_areas
        row['impact'] = smu_info.impact
        row['package_bundles'] = smu_info.package_bundles
        row['compressed_image_size'] = str(smu_info.compressed_image_size)
        row['uncompressed_image_size'] = str(smu_info.uncompressed_image_size)
        row['prerequisites'] = smu_info.prerequisites
        row['supersedes'] = smu_info.supersedes
        row['superseded_by'] = smu_info.superseded_by
        row['composite_DDTS'] = smu_info.composite_DDTS
        row['prerequisites_smu_ids'] = get_smu_ids(db_session, smu_info.prerequisites)
        row['supersedes_smu_ids'] = get_smu_ids(db_session, smu_info.supersedes)
        row['superseded_by_smu_ids'] = get_smu_ids(db_session, smu_info.superseded_by)

        rows.append(row)

    return jsonify(**{'data': rows})
Ejemplo n.º 8
0
def software_profile_edit(profile_name):
    db_session = DBSession()

    software_profile = get_software_profile(db_session, profile_name)
    if software_profile is None:
        abort(404)

    form = SoftwareProfileForm(request.form)
    server_dialog_form = ServerDialogForm(request.form)
    fill_servers(server_dialog_form.server_dialog_server.choices, get_server_list(db_session), False)

    if request.method == 'POST' and form.validate():
        if profile_name != form.profile_name.data and \
                        get_software_profile(db_session, form.profile_name.data) is not None:
            return render_template('conformance/profile_edit.html',
                                   form=form, server_dialog_form=server_dialog_form,
                                   system_option=SystemOption.get(db_session), duplicate_error=True)

        software_profile.name = form.profile_name.data
        software_profile.description = form.description.data
        software_profile.packages = ','.join([l for l in form.software_packages.data.splitlines() if l]),

        db_session.commit()

        return redirect(url_for('conformance.home'))
    else:
        form.profile_name.data = software_profile.name
        form.description.data = software_profile.description
        if software_profile.packages is not None:
            form.software_packages.data = '\n'.join(software_profile.packages.split(','))

    return render_template('conformance/profile_edit.html',
                           form=form, server_dialog_form=server_dialog_form,
                           system_option=SystemOption.get(db_session))
Ejemplo n.º 9
0
def software_profile_create():
    # if not can_create_user(current_user):
    #    abort(401)

    db_session = DBSession()

    form = SoftwareProfileForm(request.form)
    server_dialog_form = ServerDialogForm(request.form)

    fill_servers(server_dialog_form.server_dialog_server.choices, get_server_list(db_session), False)

    if request.method == 'POST' and form.validate():

        software_profile = get_software_profile(db_session, form.profile_name.data)

        if software_profile is not None:
            return render_template('conformance/profile_edit.html',
                                   form=form, system_option=SystemOption.get(db_session), duplicate_error=True)

        software_profile = SoftwareProfile(
            name=form.profile_name.data,
            description=form.description.data,
            packages=','.join([l for l in form.software_packages.data.splitlines() if l]),
            created_by=current_user.username)

        db_session.add(software_profile)
        db_session.commit()

        return redirect(url_for('conformance.home'))
    else:

        return render_template('conformance/profile_edit.html',
                               form=form, server_dialog_form=server_dialog_form,
                               system_option=SystemOption.get(db_session))
Ejemplo n.º 10
0
    def dispatch(self):
        db_session = DBSession()

        self.handle_email_jobs(db_session)
        self.handle_create_tar_jobs(db_session)

        db_session.close()
Ejemplo n.º 11
0
def command_profile_edit(profile_name):
    db_session = DBSession()

    command_profile = get_command_profile(db_session, profile_name)
    if command_profile is None:
        abort(404)

    form = CustomCommandProfileForm(request.form)

    if request.method == 'POST' and form.validate():
        if profile_name != form.profile_name.data and \
                        get_command_profile(db_session, form.profile_name.data) is not None:
            return render_template('custom_commad/command_profile_edit.html',
                                   form=form, duplicate_error=True)

        command_profile.profile_name = form.profile_name.data
        command_profile.command_list = ','.join([l for l in form.command_list.data.splitlines() if l]),

        db_session.commit()

        return redirect(url_for('custom_command.home'))
    else:
        form.profile_name.data = command_profile.profile_name
        if command_profile.command_list is not None:
            form.command_list.data = '\n'.join(command_profile.command_list.split(','))

    return render_template('custom_command/command_profile_edit.html',
                           form=form)
Ejemplo n.º 12
0
def command_profile_create():
    db_session = DBSession()

    form = CustomCommandProfileForm(request.form)

    if request.method == 'POST' and form.validate():
        command_profile = get_command_profile(db_session, form.profile_name.data)

        if command_profile is not None:
            return render_template('custom_command/command_profile_edit.html',
                                   form=form, duplicate_error=True)

        command_profile = CustomCommandProfile(
            profile_name=form.profile_name.data,
            command_list=','.join([l for l in form.command_list.data.splitlines() if l]),
            created_by=current_user.username
        )

        db_session.add(command_profile)
        db_session.commit()

        return redirect(url_for('custom_command.home'))
    else:

        return render_template('custom_command/command_profile_edit.html',
                               form=form)
Ejemplo n.º 13
0
 def start(self):
     db_session = DBSession()
     for sql in sql_statements:
         try:
             db_session.execute(sql)
         except:
             pass
Ejemplo n.º 14
0
    def get_smu_info_from_cco(self, platform, release):
        save_to_db = True
        db_session = DBSession()
        platform_release = platform + '_' + release

        try:
            self.smu_meta = SMUMeta(platform_release=platform_release)
            # Load data from the SMU XML file
            self.load()

            # This can happen if the given platform and release is not valid.
            # The load() method calls get_smu_info_from_db and failed.
            if not self.is_valid:
                logger.error('get_smu_info_from_cco() hit exception, platform_release=' + platform_release)
                return

            db_smu_meta = db_session.query(SMUMeta).filter(SMUMeta.platform_release == platform_release).first()
            if db_smu_meta:
                if db_smu_meta.created_time == self.smu_meta.created_time:
                    save_to_db = False
                else:
                    # Delete the existing smu_meta and smu_info for this platform and release
                    db_session.delete(db_smu_meta)
                    db_session.commit()

            if save_to_db:
                db_session.add(self.smu_meta)
            else:
                db_smu_meta.retrieval_time = datetime.datetime.utcnow()

            db_session.commit()

        except Exception:
            logger.exception('get_smu_info_from_cco() hit exception, platform_release=' + platform_release)
Ejemplo n.º 15
0
def api_get_distinct_host_regions(platform, software_versions):
    """
    software_versions may equal to 'ALL' or multiple software versions
    """
    clauses = []
    db_session = DBSession()

    clauses.append(Host.software_platform == platform)
    if 'ALL' not in software_versions:
        clauses.append(Host.software_version.in_(software_versions.split(',')))

    region_ids = db_session.query(Host.region_id).filter(and_(*clauses)).distinct()

    # Change a list of tuples to a list
    region_ids_list = [region_id[0] for region_id in region_ids]

    rows = []
    if not is_empty(region_ids):
        regions = db_session.query(Region).filter(Region.id.in_(region_ids_list)). \
            order_by(Region.name.asc()).all()

        for region in regions:
            rows.append({'region_id': region.id, 'region_name': region.name})

    return jsonify(**{'data': rows})
Ejemplo n.º 16
0
 def run(self):
     db_session = DBSession()   
     try:         
         system_option = SystemOption.get(db_session)            
         inventory_hour = system_option.inventory_hour
         db_session.close()
                     
         # Build a scheduler object that will look at absolute times
         scheduler = sched.scheduler(time.time, time.sleep)
         current_hour = datetime.datetime.now().hour
 
         # Put task for today at the designated hour.
         daily_time = datetime.time(inventory_hour)
         
         # If the scheduled time already passed, schedule it for tomorrow
         if current_hour > inventory_hour:
             first_time = datetime.datetime.combine(datetime.datetime.now() + datetime.timedelta(days=1), daily_time)
         else:
             first_time = datetime.datetime.combine(datetime.datetime.now(), daily_time)
         
         scheduler.enterabs(time.mktime(first_time.timetuple()), 1,
             self.scheduling, (scheduler, daily_time,))
        
         scheduler.run()
         
     except:
         logger.exception('InventoryManagerScheduler hit exception')
         db_session.close()
Ejemplo n.º 17
0
def api_get_distinct_host_roles(platform, software_versions, region_ids):
    """
    software_versions may equal to 'ALL' or multiple software versions
    region_ids may equal to 'ALL' or multiple region ids
    """
    clauses = []
    db_session = DBSession()

    clauses.append(Host.software_platform == platform)
    if 'ALL' not in software_versions:
        clauses.append(Host.software_version.in_(software_versions.split(',')))
    if 'ALL' not in region_ids:
        clauses.append(Host.region_id.in_(region_ids.split(',')))

    host_roles = db_session.query(Host.roles).filter(and_(*clauses)).distinct()

    # Change a list of tuples to a list
    # Example of roles_list  = [u'PE Router', u'PE1,R0', u'PE1,PE4', u'PE2,R1', u'Core']
    roles_list = [roles[0] for roles in host_roles if not is_empty(roles[0])]

    # Collapses the comma delimited strings to list
    roles_list = [] if is_empty(roles_list) else ",".join(roles_list).split(',')

    # Make the list unique, then sort it
    roles_list = sorted(list(set(roles_list)))

    rows = []
    for role in roles_list:
        rows.append({'role': role})

    return jsonify(**{'data': rows})
Ejemplo n.º 18
0
Archivo: cco.py Proyecto: smjurcak/csm
def user_preferences():
    db_session = DBSession()
    form = PreferencesForm(request.form)

    user = get_user_by_id(db_session, current_user.id)

    if request.method == 'POST' and form.validate():
        user.preferences[0].cco_username = form.cco_username.data

        if len(form.cco_password.data) > 0:
            user.preferences[0].cco_password = form.cco_password.data

        # All the checked checkboxes (i.e. platforms and releases to exclude).
        values = request.form.getlist('check')
        excluded_platform_list = ','.join(values)

        preferences = Preferences.get(db_session, current_user.id)
        preferences.excluded_platforms_and_releases = excluded_platform_list

        db_session.commit()

        return redirect(url_for('home'))
    else:
        preferences = user.preferences[0]
        form.cco_username.data = preferences.cco_username

        if not is_empty(user.preferences[0].cco_password):
            form.password_placeholder = 'Use Password on File'
        else:
            form.password_placeholder = 'No Password Specified'

    return render_template('cco/preferences.html', form=form,
                           platforms_and_releases=get_platforms_and_releases_dict(db_session))
Ejemplo n.º 19
0
Archivo: sum.py Proyecto: smjurcak/csm
    def dispatch(self):
        db_session = DBSession()

        try:
            # Check if Scheduled Installs are allowed to run.
            if not db_session.query(SystemOption).first().can_install:
                return
                
            install_jobs = db_session.query(InstallJob).filter(
                InstallJob.scheduled_time <= datetime.datetime.utcnow()).order_by(InstallJob.scheduled_time.asc()).all()
            download_job_key_dict = get_download_job_key_dict()

            if len(install_jobs) > 0:
                for install_job in install_jobs:
                    if install_job.status != JobStatus.FAILED:
                        # If there is pending download, don't submit the install job
                        if self.is_pending_on_download(download_job_key_dict, install_job):
                            continue

                        # This install job has a dependency, check if the expected criteria is met
                        if install_job.dependency is not None:
                            dependency_completed = self.get_install_job_dependency_completed(db_session, install_job)
                            # If the dependency has not been completed, don't proceed
                            if len(dependency_completed) == 0:
                                continue

                        self.submit_job(InstallWorkUnit(install_job.host_id, install_job.id))

        except Exception:
            # print(traceback.format_exc())
            # Purpose ignore.  Otherwise, it may generate continue exception
            pass
        finally:
            db_session.close()
Ejemplo n.º 20
0
def get_download_job_key_dict():
    result = {}
    db_session = DBSession()
    download_jobs = db_session.query(DownloadJob).all()
    for download_job in download_jobs:
        download_job_key = "{}{}{}{}".format(download_job.user_id,download_job.cco_filename, download_job.server_id, download_job.server_directory)
        result[download_job_key] = download_job
    return result
Ejemplo n.º 21
0
def api_get_conformance_report(id):
    rows = []
    dt_params = DataTableParams(request)
    db_session = DBSession()

    conformance_report = get_conformance_report_by_id(db_session, id)
    if not conformance_report:
        response = dict()
        response['draw'] = dt_params.draw
        response['recordsTotal'] = 0
        response['recordsFiltered'] = 0
        response['data'] = rows
        return jsonify(**response)

    clauses = []
    if len(dt_params.search_value):
        criteria = '%' + dt_params.search_value + '%'
        clauses.append(ConformanceReportEntry.hostname.like(criteria))
        clauses.append(ConformanceReportEntry.software_platform.like(criteria))
        clauses.append(ConformanceReportEntry.software_version.like(criteria))
        clauses.append(ConformanceReportEntry.conformed.like(criteria))
        clauses.append(ConformanceReportEntry.host_packages.like(criteria))
        clauses.append(ConformanceReportEntry.missing_packages.like(criteria))

    query = db_session.query(ConformanceReportEntry)
    total_count = query.filter(ConformanceReportEntry.conformance_report_id == id).count()
    filtered_count = query.filter(and_(ConformanceReportEntry.conformance_report_id == id), or_(*clauses)).count()

    columns = [getattr(ConformanceReportEntry.hostname, dt_params.sort_order)(),
               getattr(ConformanceReportEntry.software_platform, dt_params.sort_order)(),
               getattr(ConformanceReportEntry.software_version, dt_params.sort_order)(),
               '',
               '',
               getattr(ConformanceReportEntry.conformed, dt_params.sort_order)()]

    entries = query.order_by(columns[dt_params.column_order])\
        .filter(and_(ConformanceReportEntry.conformance_report_id == id), or_(*clauses))\
        .slice(dt_params.start_length, dt_params.start_length + dt_params.display_length).all()

    for entry in entries:
        row = dict()
        row['hostname'] = entry.hostname
        row['software_platform'] = entry.software_platform
        row['software_version'] = entry.software_version
        row['missing_packages'] = entry.missing_packages
        row['host_packages'] = entry.host_packages
        row['conformed'] = entry.conformed
        row['comments'] = entry.comments

        rows.append(row)

    response = dict()
    response['draw'] = dt_params.draw
    response['recordsTotal'] = total_count
    response['recordsFiltered'] = filtered_count
    response['data'] = rows

    return jsonify(**response)
Ejemplo n.º 22
0
 def start(self):
     db_session = DBSession()
     for sql in sql_statements:
         try:
             db_session.execute(sql)
         except:
             # import traceback
             # print(traceback.format_exc())
             pass
Ejemplo n.º 23
0
def api_get_host_software_profile_list(id):
    rows = []
    dt_params = DataTableParams(request)
    db_session = DBSession()

    software_profile = get_software_profile_by_id(db_session, id)
    if not software_profile:
        response = dict()
        response['draw'] = dt_params.draw
        response['recordsTotal'] = 0
        response['recordsFiltered'] = 0
        response['data'] = rows
        return jsonify(**response)

    clauses = []
    if len(dt_params.search_value):
        criteria = '%' + dt_params.search_value + '%'
        clauses.append(Host.hostname.like(criteria))
        clauses.append(Region.name.like(criteria))
        clauses.append(Host.roles.like(criteria))
        clauses.append(Host.platform.like(criteria))
        clauses.append(Host.software_platform.like(criteria))
        clauses.append(Host.software_version.like(criteria))

    query = db_session.query(Host).join(Region, Host.region_id == Region.id)
    total_count = query.filter(Host.software_profile_id == id).count()
    filtered_count = query.filter(and_(Host.software_profile_id == id), or_(*clauses)).count()

    columns = [getattr(Host.hostname, dt_params.sort_order)(),
               getattr(Region.name, dt_params.sort_order)(),
               getattr(Host.roles, dt_params.sort_order)(),
               getattr(Host.platform, dt_params.sort_order)(),
               getattr(Host.software_platform, dt_params.sort_order)(),
               getattr(Host.software_version, dt_params.sort_order)()]

    hosts = query.order_by(columns[dt_params.column_order])\
        .filter(and_(Host.software_profile_id == id), or_(*clauses))\
        .slice(dt_params.start_length, dt_params.start_length + dt_params.display_length).all()

    for host in hosts:
        row = dict()
        row['hostname'] = host.hostname
        row['region'] = '' if host.region is None else host.region.name
        row['roles'] = host.roles
        row['platform'] = host.platform
        row['software_platform'] = host.software_platform
        row['software_version'] = host.software_version

        rows.append(row)

    response = dict()
    response['draw'] = dt_params.draw
    response['recordsTotal'] = total_count
    response['recordsFiltered'] = filtered_count
    response['data'] = rows

    return jsonify(**response)
Ejemplo n.º 24
0
def get_progress():
    db_session = DBSession()
    job_id = request.args.get('job_id')

    tar_job = db_session.query(CreateTarJob).filter(CreateTarJob.id == job_id).first()
    if tar_job is None:
        logger.error('Unable to retrieve Create Tar Job: %s' % job_id)
        return jsonify(status='Unable to retrieve job')

    return jsonify(status='OK',progress= tar_job.status)
Ejemplo n.º 25
0
def api_get_distinct_host_platforms():
    rows = []
    db_session = DBSession()

    platforms = db_session.query(Host.software_platform).order_by(Host.software_platform.asc()).distinct()
    for platform in platforms:
        if platform[0] is not None:
            rows.append({'platform': platform[0]})

    return jsonify(**{'data': rows})
Ejemplo n.º 26
0
def api_get_model_name_summary(region_id):
    """
    Return the model name, in use (count), available (count) summary datatable json data
    """
    db_session = DBSession()

    rows = get_model_name_summary_query_results(db_session, region_id)

    db_session.close()
    return jsonify(**{'data': rows})
Ejemplo n.º 27
0
Archivo: log.py Proyecto: smjurcak/csm
def api_get_log_trace(log_id):
    """
    Returns the log trace of a particular log record.
    """
    db_session = DBSession()

    log = db_session.query(Log).filter(Log.id == log_id).first()
    return jsonify(**{'data': [
        {'severity': log.level, 'message': log.msg, 'trace': log.trace, 'created_time': log.created_time}
    ]})
Ejemplo n.º 28
0
Archivo: base.py Proyecto: smjurcak/csm
    def store_inventory(self, ctx, inventory_data, chassis_indices):
        """
        Store/update the processed inventory data in database
        :param ctx: context object
        :param inventory_data: parsed inventory data as a list of dictionaries
        :param chassis_indices: a list of index/indices of chassis inventory dictionary in inventory_data
        :return: None
        """
        if len(chassis_indices) == 0 or len(chassis_indices) > len(inventory_data):
            logger = get_db_session_logger(ctx.db_session)
            logger.exception('index/indices of chassis found in inventory output is out of range for host ' +
                             '{}.'.format(ctx.host.hostname))
            return

        # Assign the ordering or "position" of inventory in output from show inventory
        # to each inventory entry, but adjust the ordering so that chassis always have
        # negative position(s) (so as to mark corresponding inventory as chassis)
        # and non-chassis always have consecutive non-negative positions in ascending order,
        # It goes like this - if there is only one chassis, its position will be -1,
        # the non-chassis inventories will have positions starting from 0
        # If there are multiple chassis, for example 3 chassis, chassis 0 will have position -3,
        # chassis 1 will have position -2, chassis 2 will have position -1, non-chassis will
        # still have positions starting from 0
        chassis_position = 0 - len(chassis_indices)
        for chassis_idx in chassis_indices:
            inventory_data[chassis_idx]['position'] = chassis_position
            chassis_position += 1

        idx = 0
        position = 0
        rack_number = 0
        while idx < len(inventory_data):
            if rack_number < len(chassis_indices):
                if idx == chassis_indices[rack_number]:
                    rack_number += 1
                else:
                    inventory_data[idx]['position'] = position
                    position += 1
            else:
                inventory_data[idx]['position'] = position
                position += 1
            idx += 1

        db_session = DBSession()
        # this is necessary for now because somewhere in the thread, can be
        # anywhere in the code, the db_session was not closed - to be found out later.
        db_session.close()

        if len(ctx.host.host_inventory) > 0:
            self.compare_and_update(ctx, db_session, inventory_data)
        else:
            self.store_new_inventory(db_session, inventory_data, ctx.host.id)

        db_session.close()
        return
Ejemplo n.º 29
0
    def refresh_all(cls):
        """
        Retrieves all the catalog data and SMU XML file data and updates the database.
        """
        db_session = DBSession()
        
        catalog = SMUInfoLoader.get_catalog_from_cco()
        if len(catalog) > 0:
            system_option = SystemOption.get(db_session)
            try:
                # Remove all rows first
                db_session.query(CCOCatalog).delete()
            
                for platform in catalog:
                    releases = catalog[platform]
                    for release in releases:
                        cco_catalog = CCOCatalog(platform=platform,release=release)
                        db_session.add(cco_catalog)

                        SMUInfoLoader(platform, release)
                
                system_option.cco_lookup_time = datetime.datetime.utcnow()
                db_session.commit()
                return True
            except Exception:
                logger.exception('refresh_all() hit exception')
                db_session.rollback()  
            
        return False
Ejemplo n.º 30
0
def delete_custom_command_profile(profile_name):
    db_session = DBSession()

    command_profile = get_command_profile(db_session, profile_name)
    if command_profile is None:
        abort(404)

    db_session.delete(command_profile)
    db_session.commit()

    return jsonify({'status': 'OK'})
Ejemplo n.º 31
0
def query_add_inventory():
    """
    Provide service for user to:
    1. Query an inventory based on serial number.
    2. Add/Edit the inventory with input serial number, model name(optional), and notes(optional).
    3. Delete the inventory with input serial number.
    Any user can query the inventory, but only Admin user can add, update or delete inventory.
    """
    sn_form = QueryInventoryBySerialNumberForm(request.form)
    update_inventory_form = UpdateInventoryForm(request.form)

    inventory_data_fields = {
        'serial_number_submitted': None,
        'new_inventory': False,
        'inventory_name': None,
        'description': None,
        'hardware_revision': None,
        'hostname': '',
        'region_name': None,
        'chassis': None,
        'platform': None,
        'software': None,
        'last_successful_retrieval': None,
        'inventory_retrieval_status': None
    }

    error_msg = None
    success_msg = None

    if request.method == 'GET':
        init_query_add_inventory_forms(sn_form, update_inventory_form)

    elif request.method == 'POST':
        db_session = DBSession()

        # if user submitted the form with the serial number
        # then we display back all the information for the inventory with this serial number
        if sn_form.hidden_submit_sn.data == 'True':
            get_inventory_data_by_serial_number(db_session, sn_form,
                                                update_inventory_form,
                                                inventory_data_fields)

        # if user submitted the form with the updated inventory info
        # we update/create or delete the inventory
        else:

            if current_user.privilege != UserPrivilege.ADMIN:
                error_msg = "User not authorized to create, update or delete inventory."

            # there is restriction on front end - serial_number submitted cannot be empty string, double check here
            elif update_inventory_form.hidden_serial_number.data:

                inventory_entry = db_session.query(Inventory).filter(
                    Inventory.serial_number ==
                    update_inventory_form.hidden_serial_number.data).first()

                if update_inventory_form.hidden_action.data == "Update":
                    update_or_add_inventory(
                        db_session, inventory_entry,
                        update_inventory_form.hidden_serial_number.data,
                        update_inventory_form.model_name.data,
                        update_inventory_form.notes.data)

                elif update_inventory_form.hidden_action.data == "Delete":
                    success_msg, error_msg = delete_inventory(
                        db_session, inventory_entry)

                else:
                    error_msg = "Unknown request is received."
            else:
                error_msg = "Failed to create inventory because serial number submitted is empty."

            if not error_msg:
                # clear the forms to indicate a successful inventory update/add action!
                init_query_add_inventory_forms(sn_form, update_inventory_form)
            else:
                sn_form.serial_number.data = update_inventory_form.hidden_serial_number.data
                get_inventory_data_by_serial_number(db_session, sn_form,
                                                    update_inventory_form,
                                                    inventory_data_fields)

    return render_template('inventory/query_add_inventory.html',
                           sn_form=sn_form,
                           update_inventory_form=update_inventory_form,
                           success_msg=success_msg,
                           error_msg=error_msg,
                           current_user=current_user,
                           **inventory_data_fields)
Ejemplo n.º 32
0
    def get_smu_info_from_cco(self, platform, release):
        save_to_db = True
        db_session = DBSession()
        platform_release = platform + '_' + release

        try:
            self.smu_meta = SMUMeta(platform_release=platform_release)
            # Load data from the SMU XML file
            self.load()

            # This can happen if the given platform and release is not valid.
            # The load() method calls get_smu_info_from_db and failed.
            if not self.is_valid:
                logger.error(
                    'get_smu_info_from_cco() hit exception, platform_release='
                    + platform_release)
                return

            db_smu_meta = db_session.query(SMUMeta).filter(
                SMUMeta.platform_release == platform_release).first()
            if db_smu_meta:
                if db_smu_meta.created_time == self.smu_meta.created_time:
                    save_to_db = False
                else:
                    # Delete the existing smu_meta and smu_info for this platform and release
                    db_session.delete(db_smu_meta)
                    db_session.commit()

            if save_to_db:
                db_session.add(self.smu_meta)
            else:
                db_smu_meta.retrieval_time = datetime.datetime.utcnow()

            db_session.commit()

        except Exception:
            logger.exception(
                'get_smu_info_from_cco() hit exception, platform_release=' +
                platform_release)
Ejemplo n.º 33
0
def api_create_install_request(request):
    """
    Install Action: Pre-Upgrade, Post-Upgrade, and Commit

        POST: http://localhost:5000/api/v1/install
        BODY:
            [ {'hostname': 'My Host',
               'install_action': 'Post-Upgrade',
               'scheduled_time': '05-02-2016 08:00 AM',
               'command_profile': 'Edge Devices',
               'dependency': 'Add'} ]

    Install Action: Add
        BODY:
            [ {'hostname': 'My Host',
               'install_action': 'Add',
               'scheduled_time': '05-02-2016 08:00 AM',
               'server_repository': 'My FTP Server',
               'software_packages': ['asr9k-px-5.3.3.CSCuz05961.pie, asr9k-px-5.3.3.CSCux89921.pie],
               'dependency': 'Pre-Upgrade'} ]

    Install Action: Activate, Remove, Deactivate
        BODY:
            [ {'hostname': 'My Host',
               'install_action': 'Activate',
               'scheduled_time': '05-02-2016 08:00 AM',
               'software_packages': ['asr9k-px-5.3.3.CSCuz05961.pie, asr9k-px-5.3.3.CSCux89921.pie]
               'dependency': '101'} ]


        RETURN:
            {"api_response": {
                "install_job_list": [ {"status": "SUCCESS", "hostname": "My Host", "id": 101},
                                      {"status": "FAILED", "hostname": "My Host 2", "status_message": "Unable to locate host"} ]

                }
            }
    """
    rows = []
    error_found = False
    db_session = DBSession()
    custom_command_profile_dict = get_custom_command_profile_name_to_id_dict(
        db_session)
    # ----------------------------  first phase is to attempt the data validation ---------------------------- #

    entries = []
    json_list = convert_json_request_to_list(request)

    for data in json_list:
        row = dict()
        try:
            validate_required_keys_in_dict(data, [KEY_INSTALL_ACTION])

            install_action = data[KEY_INSTALL_ACTION]
            if install_action not in supported_install_actions:
                raise ValueError("'{}' is an invalid install action.".format(
                    install_action))

            validate_acceptable_keys_in_dict(data, acceptable_keys)
            validate_required_keys_in_dict(data,
                                           required_keys_dict[install_action])

            hostname = data[KEY_HOSTNAME]
            host = get_host(db_session, hostname)
            if host is None:
                raise ValueError("'{}' is an invalid hostname.".format(
                    data[KEY_HOSTNAME]))

            if KEY_SERVER_REPOSITORY in data.keys():
                server = get_server(db_session, data[KEY_SERVER_REPOSITORY])
                if server is None:
                    raise ValueError(
                        "'{}' is an invalid server repository.".format(
                            data[KEY_SERVER_REPOSITORY]))

            if KEY_CUSTOM_COMMAND_PROFILE in data.keys():
                custom_command_profile_names = convert_value_to_list(
                    data, KEY_CUSTOM_COMMAND_PROFILE)
                for custom_command_profile_name in custom_command_profile_names:
                    custom_command_profile_id = custom_command_profile_dict.get(
                        custom_command_profile_name)
                    if custom_command_profile_id is None:
                        raise ValueError(
                            "'{}' is an invalid custom command profile.".
                            format(custom_command_profile_name))

            if KEY_SOFTWARE_PACKAGES in data.keys() and is_empty(
                    data[KEY_SOFTWARE_PACKAGES]):
                raise ValueError(
                    "Software packages when specified cannot be empty.")

            # Check time fields and validate their values
            if KEY_SCHEDULED_TIME not in data.keys():
                row[KEY_UTC_SCHEDULED_TIME] = datetime.utcnow()
            elif KEY_UTC_OFFSET not in data.keys():
                raise ValueError(
                    "Missing utc_offset. If scheduled_time is submitted, utc_offset is also required."
                )
            elif not verify_utc_offset(data[KEY_UTC_OFFSET]):
                raise ValueError(
                    "Invalid utc_offset: Must be in '<+|->dd:dd' format and be between -12:00 and +14:00."
                )
            else:
                try:
                    time = datetime.strptime(data[KEY_SCHEDULED_TIME],
                                             "%m-%d-%Y %I:%M %p")
                    row[KEY_UTC_SCHEDULED_TIME] = get_utc_time(
                        time, data[KEY_UTC_OFFSET])
                except ValueError:
                    raise ValueError(
                        "Invalid scheduled_time: {} must be in 'mm-dd-yyyy hh:mm AM|PM' format."
                        .format(data[KEY_SCHEDULED_TIME]))

            # Handle duplicate entry.  It is defined by the hostname and install_action pair.
            if (hostname, install_action) not in entries:
                entries.append((hostname, install_action))
            else:
                raise ValueError(
                    "More than one entry with the same hostname: '{}' and install_action: '{}'. "
                    "Remove any duplicate and resubmit.".format(
                        hostname, install_action))

        except Exception as e:
            row[RESPONSE_STATUS] = APIStatus.FAILED
            row[RESPONSE_STATUS_MESSAGE] = e.message
            error_found = True

        # Add the original key value pairs to the new array.
        for key in data.keys():
            row[key] = data[key]

        rows.append(row)

    # End of loop

    if error_found:
        for row in rows:
            if RESPONSE_STATUS not in row.keys():
                row[RESPONSE_STATUS] = APIStatus.FAILED
                row[RESPONSE_STATUS_MESSAGE] = 'Not submitted. Check other jobs for error message.'

            if KEY_UTC_SCHEDULED_TIME in row.keys():
                row.pop(KEY_UTC_SCHEDULED_TIME)

        return jsonify(**{RESPONSE_ENVELOPE: {
            KEY_INSTALL_JOB_LIST: rows
        }}), HTTP_BAD_REQUEST

    # ----------------------------  Second phase is to attempt the job creation ---------------------------- #

    sorted_list = sorted(rows, cmp=get_key)

    rows = []
    error_found = False
    implicit_dependency_list = {}

    for install_request in sorted_list:
        row = dict()
        try:
            hostname = install_request[KEY_HOSTNAME]
            install_action = install_request[KEY_INSTALL_ACTION]

            row[KEY_INSTALL_ACTION] = install_action
            row[KEY_HOSTNAME] = hostname

            host_id = get_host(db_session, hostname).id
            utc_scheduled_time = install_request[
                KEY_UTC_SCHEDULED_TIME].strftime("%m/%d/%Y %I:%M %p")

            server_id = -1
            if KEY_SERVER_REPOSITORY in install_request.keys():
                server = get_server(db_session,
                                    install_request[KEY_SERVER_REPOSITORY])
                if server is not None:
                    server_id = server.id

            server_directory = ''
            if KEY_SERVER_DIRECTORY in install_request.keys():
                server_directory = install_request[KEY_SERVER_DIRECTORY]

            software_packages = []
            if KEY_SOFTWARE_PACKAGES in install_request.keys():
                software_packages = install_request[KEY_SOFTWARE_PACKAGES]

            custom_command_profile_ids = []
            if KEY_CUSTOM_COMMAND_PROFILE in install_request.keys():
                custom_command_profile_names = convert_value_to_list(
                    install_request, KEY_CUSTOM_COMMAND_PROFILE)
                for custom_command_profile_name in custom_command_profile_names:
                    custom_command_profile_id = custom_command_profile_dict.get(
                        custom_command_profile_name)
                    if custom_command_profile_id is not None:
                        custom_command_profile_ids.append(
                            str(custom_command_profile_id))

            install_job = create_or_update_install_job(
                db_session,
                host_id=host_id,
                install_action=install_action,
                scheduled_time=utc_scheduled_time,
                software_packages=software_packages,
                server_id=server_id,
                server_directory=server_directory,
                custom_command_profile_ids=custom_command_profile_ids,
                dependency=get_dependency_id(db_session,
                                             implicit_dependency_list,
                                             install_request, host_id),
                created_by=g.api_user.username)

            row[KEY_ID] = install_job.id

            if install_action in ordered_install_actions:
                if hostname not in implicit_dependency_list:
                    implicit_dependency_list[hostname] = []

                implicit_dependency_list[hostname].append(
                    (install_job.id, install_action,
                     install_request[KEY_UTC_SCHEDULED_TIME]))

            row[RESPONSE_STATUS] = APIStatus.SUCCESS

        except Exception as e:
            row[RESPONSE_STATUS] = APIStatus.FAILED
            row[RESPONSE_STATUS_MESSAGE] = e.message
            row[RESPONSE_TRACE] = traceback.format_exc()
            error_found = True

        rows.append(row)

    return jsonify(**{RESPONSE_ENVELOPE: {
        KEY_INSTALL_JOB_LIST: rows
    }}), (HTTP_OK if not error_found else HTTP_MULTI_STATUS_ERROR)
Ejemplo n.º 34
0
def init():
    db_session = DBSession()
    init_user(db_session)
    init_system_option(db_session)
    init_encrypt(db_session)
    db_session.commit()
Ejemplo n.º 35
0
 def _collect_video_info(self,tid, simple_info, session:DBSession, collect_id = 0):
     uploader_exists = session.query(
         exists().where(database.UploaderInfo.mid == simple_info['author_id'])
     ).scalar()
     if not uploader_exists:
         new_up = database.UploaderInfo(
             mid=simple_info['author_id'],
             name=simple_info['author_name'],
         )
         session.add(new_up)
         session.commit()
     new_video = database.VideoInfo(
         aid=simple_info['id'],
         bvid=simple_info['bvid'],
         tid=tid,
         title=simple_info['name'],
         owner_id=simple_info['author_id'],
         collect_time=datetime.datetime.now(),
         collect_id = collect_id
     )
     try:
         session.add(new_video)
         session.commit()
     except Exception as e:
         logger.error(f"向数据库中写入视频信息出错: {simple_info['name']}")
         session.rollback()
Ejemplo n.º 36
0
def load_user(user_id):
    """Hook for Flask-Login to load a User instance from a user ID."""
    db_session = DBSession()
    return db_session.query(User).get(user_id)
Ejemplo n.º 37
0
def api_host(hostname):
    db_session = DBSession()
    
    host = get_host(db_session, hostname)
    return get_host_json([host], request)
Ejemplo n.º 38
0
def check_if_email_notify_enabled():
    db_session = DBSession()
    system_option = SystemOption.get(db_session)
    db_session.close()
    return jsonify({'email_notify_enabled': system_option.enable_email_notify})
Ejemplo n.º 39
0
class Watcher_Thread(threading.Thread):
    task_queue: Queue
    thread_id: int
    use_proxy: bool

    def __init__(self, thread_id: int, task_queue: Queue, use_proxy=False):
        threading.Thread.__init__(self)
        self.thread_id = thread_id
        self.task_queue = task_queue
        self.use_proxy = use_proxy
        self.session = DBSession()
        self.spider = VideoInfoSpider(use_proxy=use_proxy)

    def _watch_video(self, video):
        if not video.cid:
            try:
                video.cid = self.spider.get_cid(video.aid)
            except Exception as e:
                logger.error(f"T:{self.thread_id} 获取 {video.aid} cid出错:\n {e}")
                time.sleep(2)
                return
            video.first_get_state_time = datetime.datetime.now()
        try:
            online_count, zid = self.spider.get_online_count_and_tid(
                video.aid, video.cid)
        except Exception as e:
            logger.error(f"T:{self.thread_id} 获取 {video.aid} 在线观看人数出错:\n {e}")
            time.sleep(10)
            return
        if not video.zid:
            zone_exists = self.session.query(
                exists().where(ZoneInfo.zid == zid)).scalar()
            if not zone_exists:
                try:
                    new_zone = ZoneInfo(zid=zid, )
                    self.session.add(new_zone)
                    self.session.commit()
                except Exception as e:
                    logger.error(f"写入 {video.aid} zid 出错:\n")
                    self.session.rollback()
                    return
            video.zid = zid
        try:
            total_count = self.spider.get_total_count(video.aid)
        except Exception as e:
            logger.error(f"T: {self.thread_id} 获取 {video.aid} 当前总状态出错:\n{e}")
            time.sleep(10)
            return
        try:
            new_state = VideoState(
                aid=total_count['aid'],
                time=datetime.datetime.now(),
                online=online_count,
                view=total_count['view'],
                danmaku=total_count['danmaku'],
                reply=total_count['reply'],
                favorite=total_count['favorite'],
                coin=total_count['coin'],
                share=total_count['share'],
                like=total_count['like'],
                dislike=total_count['dislike'],
                now_rank=total_count['now_rank'],
                his_rank=total_count['his_rank'],
            )
            self.session.add(new_state)
            video.last_get_state_time = datetime.datetime.now()
            logger.info(
                f"T:{self.thread_id} {video.aid} {video.title} online: {online_count}, total: {new_state.view}"
            )
            self.session.commit()
        except Exception as e:
            logger.error(f"写入数据库出错\n {e}")
            self.session.rollback()

    def run(self):
        while True:
            if self.task_queue.empty():
                logger.info(
                    f"thread:{self.thread_id} Empty task queue, Done!Done!Done!"
                )
                break
            try:
                video = self.task_queue.get()
                self._watch_video(video)
            except Exception as e:
                logger.error(f"T:{self.thread_id} 爬取任务出错:{e}")
Ejemplo n.º 40
0
def api_import_inventory():
    """
    API for importing inventory
    Note: 1. If inventory already exists in db, and the model name has been discovered
         by CSM, we will not overwrite the model name with data from here.
         2. For data with duplicate serial numbers, only the first data entry will be created
         in db or used to update an existing inventory.
    return either status: OK with unimported_inventory: list of unimported data rows
                                                        (for noting duplicated serial numbers)
           or status: errors in the imported data separated by comma's
    """
    if current_user.privilege != UserPrivilege.ADMIN:
        abort(401)
    importable_header = [
        HEADER_FIELD_SERIAL_NUMBER, HEADER_FIELD_MODEL_NAME, HEADER_FIELD_NOTES
    ]
    general_notes = request.form['general_notes']
    data_list = request.form['data_list']

    db_session = DBSession()

    reader = csv.reader(data_list.splitlines(), delimiter=',')
    header_row = next(reader)

    # Check mandatory data fields
    error = []
    if HEADER_FIELD_SERIAL_NUMBER not in header_row:
        error.append('"serial_number" is missing in the header.')

    for header_field in header_row:
        if header_field not in importable_header:
            error.append('"' + header_field + '" is not a valid header field.')

    if error:
        return jsonify({'status': ','.join(error)})

    row = 2
    # already checked that HEADER_FIELD_SERIAL_NUMBER is in header
    serial_number_idx = header_row.index(HEADER_FIELD_SERIAL_NUMBER)

    data_list = list(reader)

    # Check if each row has the same number of data fields as the header
    for row_data in data_list:
        if len(row_data) > 0:
            if len(row_data) != len(header_row):
                error.append('line %d has wrong number of data fields.' % row)
            else:
                if not row_data[serial_number_idx]:
                    error.append('line %d missing serial number value.' % row)
        row += 1

    if error:
        return jsonify({'status': ','.join(error)})

    # Import the data
    unique_serial_numbers = set()
    unimported_inventory = []
    row = 1
    for data in data_list:
        row += 1
        if len(data) == 0:
            continue

        serial_number = ''
        model_name = ''
        notes = general_notes

        for column in range(len(header_row)):

            header_field = header_row[column]
            data_field = data[column].strip()

            if header_field == HEADER_FIELD_SERIAL_NUMBER:
                serial_number = data_field
            elif header_field == HEADER_FIELD_MODEL_NAME:
                model_name = data_field
            elif header_field == HEADER_FIELD_NOTES and data_field:
                notes = data_field

        if serial_number:
            inventory_obj = db_session.query(Inventory).filter(
                Inventory.serial_number == serial_number).first()
            # only create/update inventory data if the serial number is unique among the imported data
            if serial_number not in unique_serial_numbers:
                update_or_add_inventory(db_session,
                                        inventory_obj,
                                        serial_number,
                                        model_name,
                                        notes,
                                        commit=False)
                unique_serial_numbers.add(serial_number)
            else:
                unimported_inventory.append('line %d: ' % row + ','.join(data))
        else:
            return jsonify(
                {'status': 'Serial number data field cannot be empty.'})

    db_session.commit()
    db_session.close()

    if unimported_inventory:
        return jsonify({
            'status': 'OK',
            'unimported_inventory': unimported_inventory
        })

    return jsonify({'status': 'OK', 'unimported_inventory': []})
Ejemplo n.º 41
0
Archivo: log.py Proyecto: smjurcak/csm
def logs():
    return render_template('log.html',
                           system_option=SystemOption.get(DBSession()))
Ejemplo n.º 42
0
Archivo: log.py Proyecto: smjurcak/csm
def host_session_log(hostname, table, id):
    """
    This route is also used by mailer.py for email notification.
    """
    db_session = DBSession()

    record = None
    doc_central_log_file_path = ''

    if table == 'install_job':
        record = db_session.query(InstallJob).filter(
            InstallJob.id == id).first()
    elif table == 'install_job_history':
        record = db_session.query(InstallJobHistory).filter(
            InstallJobHistory.id == id).first()

        doc_central_log_file_path = get_doc_central_log_path(record)
    elif table == 'inventory_job_history':
        record = db_session.query(InventoryJobHistory).filter(
            InventoryJobHistory.id == id).first()

    if record is None:
        abort(404)

    file_path = request.args.get('file_path')
    log_file_path = get_log_directory() + file_path

    if not (os.path.isdir(log_file_path) or os.path.isfile(log_file_path)):
        abort(404)

    file_pairs = {}
    log_file_contents = ''

    file_suffix = '.diff.html'
    if os.path.isdir(log_file_path):
        # Returns all files under the requested directory
        log_file_list = get_file_list(log_file_path)
        diff_file_list = [
            filename for filename in log_file_list if file_suffix in filename
        ]

        for filename in log_file_list:
            diff_file_path = ''
            if file_suffix not in filename:
                if filename + file_suffix in diff_file_list:
                    diff_file_path = os.path.join(file_path,
                                                  filename + file_suffix)
                file_pairs[os.path.join(file_path, filename)] = diff_file_path

        file_pairs = collections.OrderedDict(sorted(file_pairs.items()))
    else:
        with io.open(log_file_path, "rt", encoding='latin-1') as fo:
            log_file_contents = fo.read()

    return render_template('host/session_log.html',
                           hostname=hostname,
                           table=table,
                           record_id=id,
                           file_pairs=file_pairs,
                           log_file_contents=log_file_contents,
                           is_file=os.path.isfile(log_file_path),
                           doc_central_log_file_path=doc_central_log_file_path)
Ejemplo n.º 43
0
 def __enter__(self):
     self.db_session = DBSession()
     return Users(self.db_session)
Ejemplo n.º 44
0
def api_get_nonlocal_servers_by_region_name(region_name):

    db_session = DBSession()
    region = get_region(db_session, region_name)

    return get_nonlocal_servers(db_session, region)
Ejemplo n.º 45
0
def api_get_nonlocal_servers_by_region(region_id):

    db_session = DBSession()
    region = get_region_by_id(db_session, region_id)

    return get_nonlocal_servers(db_session, region)
Ejemplo n.º 46
0
def export_inventory_information():
    """export the inventory search result to cvs, html or excel format."""
    db_session = DBSession()
    export_results_form = ExportInventoryInformationForm(request.form)

    export_data = dict()
    export_data['export_format'] = export_results_form.export_format.data
    export_data['serial_number'] = export_results_form.hidden_serial_number.data \
        if export_results_form.hidden_serial_number.data != "" else None
    export_data['region_ids'] = export_results_form.hidden_region_ids.data.split(',') \
        if export_results_form.hidden_region_ids.data else []
    export_data['chassis_types'] = export_results_form.hidden_chassis_types.data.split(',') \
        if export_results_form.hidden_chassis_types.data else []
    export_data['software_versions'] = export_results_form.hidden_software_versions.data.split(',') \
        if export_results_form.hidden_software_versions.data else []
    export_data['model_names'] = export_results_form.hidden_model_names.data.split(',') \
        if export_results_form.hidden_model_names.data else []
    export_data['partial_model_names'] = export_results_form.hidden_partial_model_names.data.split(',') \
        if export_results_form.hidden_partial_model_names.data else []
    export_data['vid'] = export_results_form.hidden_vid.data \
        if export_results_form.hidden_vid.data != "" else None

    if export_data['region_ids']:
        region_names = db_session.query(Region.name).filter(
            Region.id.in_(map(int, export_data['region_ids']))).order_by(
                Region.name.asc()).all()
        export_data['region_names'] = []
        [
            export_data['region_names'].append(query_tuple[0])
            for query_tuple in region_names
        ]
    else:
        export_data['region_names'] = []

    export_data['available_inventory_iter'] = query_available_inventory(
        db_session, export_data.get('serial_number'),
        export_data.get('model_names'), export_data.get('partial_model_names'),
        export_data.get('vid'))

    export_data['in_use_inventory_iter'] = query_in_use_inventory(
        db_session, export_data)

    export_data['user'] = current_user

    writer = None
    if export_data.get('export_format') == ExportInformationFormat.HTML:
        writer = ExportInventoryInfoHTMLWriter(**export_data)
    elif export_data.get(
            'export_format') == ExportInformationFormat.MICROSOFT_EXCEL:
        writer = ExportInventoryInfoExcelWriter(**export_data)
    elif export_data.get('export_format') == ExportInformationFormat.CSV:
        writer = ExportInventoryInfoCSVWriter(**export_data)

    if writer:
        file_path = writer.write_report()

        if export_results_form.send_email.data:
            email_message = "<html><head></head><body>Please find in the attachment the inventory search results " \
                            "matching the following search criteria: "
            search_criteria_in_html = get_search_filter_in_html(export_data)
            if search_criteria_in_html:
                email_message += search_criteria_in_html + '</body></html>'
            else:
                email_message += '&nbsp;None</body></html>'
            create_email_job_with_attachment_files(
                db_session, email_message, file_path,
                export_results_form.user_email.data)

        return send_file(file_path, as_attachment=True)

    logger.error('inventory: invalid export format "%s" chosen.' %
                 export_data.get('export_format'))
    return
Ejemplo n.º 47
0
def api_host_list():
    db_session = DBSession()
    
    hosts = get_host_list(db_session)
    return get_host_json(hosts, request)
Ejemplo n.º 48
0
def api_create_server_repositories(request):
    """
    [{
        "hostname": "Repository_1",
        "server_type": "TFTP",
        "tftp_server_path": "223.255.254.245",
        "home_directory": "/auto/tftp-sjc-users1"
    },{
        "hostname": "Repository_2",
        "server_type": "FTP",
        "server_address": "172.27.153.150",
        "home_directory": "/tftpboot"
    },{
        "hostname": "Repository_3",
        "server_type": "SFTP",
        "server_address": "nb-server3",
        "home_directory": "/auto/tftp-vista"
    }]
    """
    rows = []
    db_session = DBSession()
    error_found = False

    json_list = convert_json_request_to_list(request)
    for data in json_list:
        row = dict()
        try:
            validate_required_keys_in_dict(data,
                                           [KEY_HOSTNAME, KEY_SERVER_TYPE])

            hostname = get_acceptable_string(data[KEY_HOSTNAME])
            row[KEY_HOSTNAME] = hostname

            if len(hostname) == 0:
                raise ValueError(
                    "Server repository name '{}' is not valid.".format(
                        data[KEY_HOSTNAME]))

            server_type = data.get(KEY_SERVER_TYPE)
            if server_type not in [
                    ServerType.TFTP_SERVER, ServerType.FTP_SERVER,
                    ServerType.SFTP_SERVER, ServerType.LOCAL_SERVER,
                    ServerType.SCP_SERVER
            ]:
                raise ValueError(
                    "'{}' is not a supported server type.".format(server_type))

            row[KEY_SERVER_TYPE] = server_type

            validate_required_keys_in_dict(data,
                                           required_keys_dict[server_type])
            validate_acceptable_keys_in_dict(data,
                                             acceptable_keys_dict[server_type])

            server = get_server(db_session, hostname)
            if server is None:
                # These are the required fields for a new server repository creation.
                validate_required_keys_in_dict(
                    data, required_keys_dict.get(server_type))

            server_url = data.get(server_url_dict[server_type])
            server_url = server_url if server_url is not None else \
                (None if server is None else server.server_url)

            server_directory = data.get(server_directory_dict[server_type])
            server_directory = server_directory if server_directory is not None else \
                (None if server is None else server.server_directory)

            vrf = data.get(KEY_VRF) if data.get(KEY_VRF) is not None else \
                (None if server is None else server.vrf)

            username = data.get(KEY_USERNAME) if data.get(KEY_USERNAME) is not None else \
                (None if server is None else server.username)

            password = data.get(KEY_PASSWORD) if data.get(KEY_PASSWORD) is not None else \
                (None if server is None else server.password)

            destination_on_host = data.get(KEY_DESTINATION_ON_HOST) if data.get(KEY_DESTINATION_ON_HOST) is not None else \
                (None if server is None else server.destination_on_host)

            create_or_update_server_repository(
                db_session,
                hostname=hostname,
                server_type=server_type,
                server_url=server_url,
                username=username,
                password=password,
                vrf=vrf,
                server_directory=server_directory,
                destination_on_host=destination_on_host,
                created_by=g.api_user.username,
                server=get_server(db_session, hostname))

            row[RESPONSE_STATUS] = APIStatus.SUCCESS
        except Exception as e:
            row[RESPONSE_STATUS] = APIStatus.FAILED
            row[RESPONSE_STATUS_MESSAGE] = e.message
            error_found = True

        rows.append(row)

    return jsonify(**{RESPONSE_ENVELOPE: {
        'server_repository_list': rows
    }}), (HTTP_OK if not error_found else HTTP_MULTI_STATUS_ERROR)
Ejemplo n.º 49
0
    def startCollect(self, detail = False):
        collect_id = 0
        popular_spider = PopularSpider()
        video_info_spider = VideoInfoSpider()
        channels = popular_spider.get_all_channels()
        session = DBSession()
        logger.info("开始写入数据库...")
        for channel in channels:
            try:
                channel_exists = session.query(
                    exists().where(database.ChannelInfo.tid == channel['id'])
                ).scalar()
                if not channel_exists:
                    new_channel = database.ChannelInfo(
                        tid=channel['id'],
                        name=channel['name'],
                        cover=channel['cover'],
                        subscribed_count=channel['subscribed_count'],
                        archive_count=channel['archive_count'],
                        featured_count=channel['featured_count'],
                    )
                    session.add(new_channel)
                    session.commit()
            except Exception:
                logger.error("频道写入失败,继续下一个频道")
                session.rollback()

        db_channels = session.query(database.ChannelInfo).all()
        for channel in db_channels:
            try:
                logger.info(f"开始爬取频道 {channel.tid}, {channel.name}"
                            f"top100 video")
                hot_videos = popular_spider.get_channel_hot_video(channel_id=channel.tid, video_num=20)
            except Exception:
                logger.error("频道获取失败,继续下一个频道")
                continue
            for video in hot_videos:  # 遍历所有推荐视频
                collect_id += 1
                if not detail:
                    self._collect_video_info(channel.tid, video, session, collect_id)
                    continue
                info = video_info_spider.get_video_info(video['id'])
                uploader_exists = session.query(
                    exists().where(database.UploaderInfo.mid == info['owner']['mid'])
                ).scalar()
                if not uploader_exists:
                    new_up = database.UploaderInfo(
                        mid=info['owner']['mid'],
                        name=info['owner']['name'],
                        face=info['owner']['face'],
                    )
                    session.add(new_up)
                    session.commit()
                zone_exists = session.query(
                    exists().where(database.ZoneInfo.zid == info['tid'])
                ).scalar()
                if not zone_exists:
                    new_zone = database.ZoneInfo(
                        zid=info['tid'],
                    )
                    session.add(new_zone)
                    session.commit()
                new_video = database.VideoInfo(
                    aid=info['aid'],
                    bvid=info['bvid'],
                    tid=channel.tid,
                    zid=info['tid'],
                    title=info['title'],
                    pubdate=datetime.datetime.fromtimestamp(info['pubdate']),
                    desc=info['desc'],
                    attribute=info['attribute'],
                    duration=info['duration'],
                    owner_id=info['owner']['mid'],
                    collect_time=datetime.datetime.now(),
                    collect_id=collect_id,
                )
                try:
                    session.add(new_video)
                    session.commit()
                except Exception as e:
                    logger.error(f"向数据库中写入视频信息出错: {info['title']}")
                    session.rollback()
        session.close()
Ejemplo n.º 50
0
    def get_smu_info_from_cco(self, platform, release):
        same_as_db = False
        db_session = DBSession()
        platform_release = platform + '_' + release
        try:
            self.smu_meta = SMUMeta(platform_release=platform_release)
            # Load data from the SMU XML file
            self.load()

            # This can happen if the given platform and release is not valid.
            # The load method calls get_smu_info_from_db and failed.
            if self.smu_meta is None:
                return

            db_smu_meta = db_session.query(SMUMeta).filter(
                SMUMeta.platform_release == platform + '_' + release).first()
            if db_smu_meta is not None:
                if db_smu_meta.created_time == self.smu_meta.created_time:
                    same_as_db = True
                else:
                    # Delete the existing smu_meta and smu_info for this platform and release
                    db_session.delete(db_smu_meta)

            if not same_as_db:
                db_session.add(self.smu_meta)
            else:
                db_smu_meta.retrieval_time = datetime.datetime.utcnow()

            # Use Flush to detect concurrent saving condition.  It is
            # possible that another process may perform the same save.
            # If this happens, Duplicate Key may result.
            db_session.flush()
            db_session.commit()

        except IntegrityError:
            db_session.rollback()
        except Exception:
            db_session.rollback()
            logger.exception('get_smu_info_from_cco hit exception')
Ejemplo n.º 51
0
def load_user(user_id):
    session = DBSession()
    user = session.query(User).filter(User.userid == user_id).first()
    session.close()
    return user
Ejemplo n.º 52
0
def insert_or_update_user(user_id, fullname, username):
    session = DBSession()
    target_user = session.query(User).get(user_id)
    if not target_user:
        new_user = User(id=user_id, fullname=fullname, username=username)
        session.add(new_user)
        session.commit()
    elif target_user.fullname != fullname or target_user.username != username:
        target_user.fullname = fullname
        target_user.username = username
        session.commit()
    session.close()
Ejemplo n.º 53
0
def home():
    if current_user.privilege != UserPrivilege.ADMIN:
        abort(401)

    db_session = DBSession()

    smtp_form = SMTPForm(request.form)
    admin_console_form = AdminConsoleForm(request.form)

    smtp_server = get_smtp_server(db_session)
    system_option = SystemOption.get(db_session)

    fill_user_privileges(
        admin_console_form.ldap_default_user_privilege.choices)

    if request.method == 'POST' and \
        smtp_form.validate() and \
        admin_console_form.validate():

        if smtp_server is None:
            smtp_server = SMTPServer()
            db_session.add(smtp_server)

        smtp_server.server = smtp_form.server.data
        smtp_server.server_port = smtp_form.server_port.data if len(
            smtp_form.server_port.data) > 0 else None
        smtp_server.sender = smtp_form.sender.data
        smtp_server.use_authentication = smtp_form.use_authentication.data
        smtp_server.username = smtp_form.username.data
        if len(smtp_form.password.data) > 0:
            smtp_server.password = smtp_form.password.data
        smtp_server.secure_connection = smtp_form.secure_connection.data

        system_option.inventory_threads = admin_console_form.num_inventory_threads.data
        system_option.install_threads = admin_console_form.num_install_threads.data
        system_option.download_threads = admin_console_form.num_download_threads.data
        system_option.can_schedule = admin_console_form.can_schedule.data
        system_option.can_install = admin_console_form.can_install.data
        system_option.enable_email_notify = admin_console_form.enable_email_notify.data
        system_option.enable_inventory = admin_console_form.enable_inventory.data

        # The LDAP UI may be hidden if it is not supported.
        # In this case, the flag is not set.
        if not is_empty(admin_console_form.enable_ldap_auth.data):
            system_option.enable_ldap_auth = admin_console_form.enable_ldap_auth.data
            system_option.ldap_server_url = admin_console_form.ldap_server_url.data
            system_option.ldap_default_user_privilege = admin_console_form.ldap_default_user_privilege.data
            system_option.ldap_server_distinguished_names = admin_console_form.ldap_server_distinguished_names.data.strip(
            )

        system_option.inventory_hour = admin_console_form.inventory_hour.data
        system_option.inventory_history_per_host = admin_console_form.inventory_history_per_host.data
        system_option.download_history_per_user = admin_console_form.download_history_per_user.data
        system_option.install_history_per_host = admin_console_form.install_history_per_host.data
        system_option.total_system_logs = admin_console_form.total_system_logs.data
        system_option.enable_default_host_authentication = admin_console_form.enable_default_host_authentication.data
        system_option.default_host_authentication_choice = admin_console_form.default_host_authentication_choice.data
        system_option.enable_cco_lookup = admin_console_form.enable_cco_lookup.data
        system_option.use_utc_timezone = admin_console_form.use_utc_timezone.data
        system_option.default_host_username = admin_console_form.default_host_username.data

        if len(admin_console_form.default_host_password.data) > 0:
            system_option.default_host_password = admin_console_form.default_host_password.data

        system_option.enable_user_credential_for_host = admin_console_form.enable_user_credential_for_host.data

        db_session.commit()

        return redirect(url_for('home'))
    else:

        admin_console_form.num_inventory_threads.data = system_option.inventory_threads
        admin_console_form.num_install_threads.data = system_option.install_threads
        admin_console_form.num_download_threads.data = system_option.download_threads
        admin_console_form.can_schedule.data = system_option.can_schedule
        admin_console_form.can_install.data = system_option.can_install
        admin_console_form.enable_email_notify.data = system_option.enable_email_notify
        admin_console_form.enable_ldap_auth.data = system_option.enable_ldap_auth
        admin_console_form.ldap_server_url.data = system_option.ldap_server_url
        admin_console_form.ldap_default_user_privilege.data = system_option.ldap_default_user_privilege
        admin_console_form.ldap_server_distinguished_names.data = system_option.ldap_server_distinguished_names
        admin_console_form.enable_inventory.data = system_option.enable_inventory
        admin_console_form.inventory_hour.data = system_option.inventory_hour
        admin_console_form.inventory_history_per_host.data = system_option.inventory_history_per_host
        admin_console_form.download_history_per_user.data = system_option.download_history_per_user
        admin_console_form.install_history_per_host.data = system_option.install_history_per_host
        admin_console_form.total_system_logs.data = system_option.total_system_logs
        admin_console_form.enable_default_host_authentication.data = system_option.enable_default_host_authentication
        admin_console_form.default_host_authentication_choice.data = system_option.default_host_authentication_choice
        admin_console_form.default_host_username.data = system_option.default_host_username
        admin_console_form.enable_cco_lookup.data = system_option.enable_cco_lookup
        admin_console_form.use_utc_timezone.data = system_option.use_utc_timezone
        admin_console_form.cco_lookup_time.data = get_datetime_string(
            system_option.cco_lookup_time)
        admin_console_form.enable_user_credential_for_host.data = system_option.enable_user_credential_for_host

        if not is_empty(system_option.default_host_password):
            admin_console_form.default_host_password_placeholder = 'Use Password on File'
        else:
            admin_console_form.default_host_password_placeholder = 'No Password Specified'

        if smtp_server is not None:
            smtp_form.server.data = smtp_server.server
            smtp_form.server_port.data = smtp_server.server_port
            smtp_form.sender.data = smtp_server.sender
            smtp_form.use_authentication.data = smtp_server.use_authentication
            smtp_form.username.data = smtp_server.username
            smtp_form.secure_connection.data = smtp_server.secure_connection
            if not is_empty(smtp_server.password):
                smtp_form.password_placeholder = 'Use Password on File'
            else:
                smtp_form.password_placeholder = 'No Password Specified'

        return render_template('admin/index.html',
                               admin_console_form=admin_console_form,
                               smtp_form=smtp_form,
                               system_option=SystemOption.get(db_session),
                               is_ldap_supported=is_ldap_supported())
Ejemplo n.º 54
0
def delete_chat_or_do_nothing(chat_id):
    session = DBSession()
    target_chat = session.query(Chat).get(chat_id)
    if target_chat and not target_chat.enable:
        session.delete(target_chat)
        session.commit()
        related_messages = session.query(Message).filter(
            Message.from_chat == chat_id)
        for message in related_messages:
            session.delete(message)
            session.commit()
        msg_text = '成功删除相关记录'
    else:
        msg_text = '此前未停用或未启用'
    session.close()
    return msg_text
Ejemplo n.º 55
0
def host_edit(hostname):
    db_session = DBSession()

    host = get_host(db_session, hostname)
    if host is None:
        abort(404)

    form = HostForm(request.form)
    fill_jump_hosts(db_session, form.jump_host.choices)
    fill_regions(db_session, form.region.choices)
    fill_software_profiles(db_session, form.software_profile.choices)

    if request.method == 'POST' and form.validate():
        if not can_edit(current_user):
            abort(401)

        # Editing a hostname which has already existed in the database.
        if hostname != form.hostname.data and get_host(
                db_session, form.hostname.data) is not None:
            return render_template('host/edit.html',
                                   form=form,
                                   duplicate_error=True)

        create_or_update_host(
            db_session=db_session,
            hostname=form.hostname.data,
            region_id=form.region.data,
            location=form.location.data,
            roles=form.roles.data,
            software_profile_id=form.software_profile.data,
            connection_type=form.connection_type.data,
            host_or_ip=form.host_or_ip.data,
            username=form.username.data,
            password=form.password.data if len(form.password.data) > 0 else
            host.connection_param[0].password,
            enable_password=form.enable_password.data
            if len(form.enable_password.data) > 0 else
            host.connection_param[0].enable_password,
            port_number=form.port_number.data,
            jump_host_id=form.jump_host.data,
            created_by=current_user.username,
            host=host)

        return_url = get_return_url(request, 'home')

        if return_url is None:
            return redirect(url_for('home'))
        else:
            return redirect(url_for(return_url, hostname=hostname))
    else:
        # Assign the values to form fields
        form.hostname.data = host.hostname
        form.region.data = host.region_id
        form.software_profile.data = host.software_profile_id
        form.location.data = host.location
        form.roles.data = host.roles
        form.host_or_ip.data = host.connection_param[0].host_or_ip
        form.username.data = host.connection_param[0].username
        form.jump_host.data = host.connection_param[0].jump_host_id
        form.connection_type.data = host.connection_param[0].connection_type
        form.port_number.data = host.connection_param[0].port_number
        if not is_empty(host.connection_param[0].password):
            form.password_placeholder = 'Use Password on File'
        else:
            form.password_placeholder = 'No Password Specified'

        if not is_empty(host.connection_param[0].enable_password):
            form.enable_password_placeholder = 'Use Password on File'
        else:
            form.enable_password_placeholder = 'No Password Specified'

        return render_template('host/edit.html', form=form)
Ejemplo n.º 56
0
def shutdown_session(exception=None):
    db_session = DBSession()
    db_session.close()
Ejemplo n.º 57
0
        if len(args) >= 1:
            msg = msg % args

        log = Log(level=record.__dict__['levelname'],
                  trace=trace,
                  msg=msg,
                  created_time=datetime.datetime.utcnow())

        self.db_session.add(log)
        self.db_session.commit()


logger = logging.getLogger('logger')
logger.setLevel(logging.DEBUG)
logger.addHandler(LogHandler(DBSession()))


def get_db_session_logger(db_session):
    """
    Return a session specific logger.  This is necessary especially
    if the db_session is from a different process address space.
    """
    session_logger = logging.getLogger('session_logger_%s' %
                                       db_session.hash_key)
    if not hasattr(session_logger, 'initialized'):
        session_logger.setLevel(logging.DEBUG)
        session_logger.addHandler(LogHandler(db_session))
        session_logger.initialized = True

    return session_logger
Ejemplo n.º 58
0
def init_system_option():
    db_session = DBSession()
    if db_session.query(SystemOption).count() == 0:
        db_session.add(SystemOption())
        db_session.commit()