Ejemplo n.º 1
0
    def get(self, **kwargs):
        script_path = request.args.get('file', default=None)
        script_type = request.args.get('script_type', default=None)

        organization = kwargs['organization']
        team = kwargs['team']

        back_scripts_root = get_back_scripts_root(team=team,
                                                  organization=organization)
        user_scripts_root = get_user_scripts_root(team=team,
                                                  organization=organization)

        if script_path:
            if script_type is None:
                return error_message(EINVAL,
                                     'Field script_type is required'), 400
            if script_type == 'user_scripts':
                return send_from_directory(
                    Path(os.getcwd()) / user_scripts_root, script_path)
            elif script_type == 'backing_scripts':
                return send_from_directory(
                    Path(os.getcwd()) / back_scripts_root, script_path)
            else:
                return error_message(EINVAL, 'Unsupported script type ' +
                                     script_type), 400
        elif script_type:
            return error_message(EINVAL, 'Field file is required'), 400

        user_scripts = path_to_dict(user_scripts_root)
        back_scripts = path_to_dict(back_scripts_root)
        return {'user_scripts': user_scripts, 'backing_scripts': back_scripts}
Ejemplo n.º 2
0
    def delete(self, **kwargs):
        """Delete the script file"""
        organization = kwargs['organization']
        team = kwargs['team']

        script = request.json.get('file', None)
        if not script:
            return response_message(EINVAL, 'Field file is required'), 400
        if not is_path_secure(script):
            return response_message(
                EINVAL,
                'Referencing to Upper level directory is not allowed'), 401

        dirname = os.path.dirname(script)
        basename = os.path.basename(script)

        script_type = request.json.get('script_type', None)
        if script_type is None:
            return response_message(EINVAL,
                                    'Field script_type is required'), 400

        if script_type == 'test_scripts':
            root = get_user_scripts_root(team=team, organization=organization)
        elif script_type == 'test_libraries':
            root = get_back_scripts_root(team=team, organization=organization)
        else:
            return response_message(EINVAL, 'Unsupported script type ' +
                                    script_type), 400

        if not os.path.exists(root / script):
            return response_message(ENOENT,
                                    'File/directory doesn\'t exist'), 404

        if os.path.isfile(root / script):
            try:
                os.remove(root / script)
            except OSError as err:
                current_app.logger.exception(err)
                return response_message(
                    EIO, 'Error happened while deleting the file'), 401

            if script_type == 'test_scripts':
                Test.objects(test_suite=os.path.splitext(basename)[0],
                             path=dirname).delete()
        else:
            if script_type == 'test_scripts':
                cnt = Test.objects(path__startswith=dirname).delete()
                if cnt == 0:
                    current_app.logger.error(
                        f'Test suite not found in the database under the path {dirname}'
                    )
            try:
                shutil.rmtree(root / script)
            except OSError as err:
                current_app.logger.exception(err)
                return response_message(
                    EIO, 'Error happened while deleting the directory'), 401

        return response_message(SUCCESS)
Ejemplo n.º 3
0
    def post(self, **kwargs):
        """
        Upload the scripts

        Note: Files in the form request payload are tuples of file name and file content
        which can't be explicitly listed here. Please check out the form data format on the web.
        Usually browser will take care of it.
        """
        found = False

        organization = kwargs['organization']
        team = kwargs['team']
        user = kwargs['user']

        script_type = request.form.get('script_type', None)
        if script_type is None:
            return response_message(EINVAL,
                                    'Field script_type is required'), 400

        if script_type == 'test_scripts':
            root = get_user_scripts_root(team=team, organization=organization)
        elif script_type == 'test_libraries':
            root = get_back_scripts_root(team=team, organization=organization)
        else:
            return response_message(EINVAL, 'Unsupported script type ' +
                                    script_type), 400

        if not os.path.exists(root):
            os.mkdir(root)

        for name, file in request.files.items():
            if not is_path_secure(file.filename):
                return response_message(
                    EINVAL,
                    'Referencing to Upper level directory is not allowed'), 401
            found = True
            filename = root / file.filename
            file.save(str(filename))

        if not found:
            return response_message(EINVAL,
                                    'No files are found in the request'), 401

        if script_type == 'test_scripts':
            for name, file in request.files.items():
                if not file.filename.endswith('.md'):
                    continue
                ret = db_update_test(root, file.filename, user, organization,
                                     team)
                if not ret:
                    return response_message(UNKNOWN_ERROR,
                                            'Failed to update test suite'), 401
Ejemplo n.º 4
0
    def post(self, **kwargs):
        found = False

        organization = kwargs['organization']
        team = kwargs['team']
        user = kwargs['user']

        script_type = request.form.get('script_type', None)
        if script_type is None:
            return error_message(EINVAL, 'Field script_type is required'), 400

        script_type = request.form.get('script_type', None)
        if script_type is None:
            return error_message(EINVAL, 'Field script_type is required'), 400

        if script_type == 'user_scripts':
            root = get_user_scripts_root(team=team, organization=organization)
        elif script_type == 'backing_scripts':
            root = get_back_scripts_root(team=team, organization=organization)
        else:
            return error_message(EINVAL,
                                 'Unsupported script type ' + script_type), 400

        if not os.path.exists(root):
            os.mkdir(root)

        for name, file in request.files.items():
            if '..' in file.filename:
                return error_message(
                    EINVAL,
                    'Referencing to Upper level directory is not allowed'), 401
            found = True
            filename = root / file.filename
            file.save(str(filename))

        if not found:
            return error_message(ENOENT,
                                 'No files are found in the request'), 404

        if script_type == 'user_scripts':
            for name, file in request.files.items():
                ret = db_update_test(scripts_dir=root,
                                     script=file.filename,
                                     user=user.email,
                                     organization=organization,
                                     team=team)
                if ret:
                    return error_message(UNKNOWN_ERROR,
                                         'Failed to update test suite'), 401
Ejemplo n.º 5
0
    def delete(self, **kwargs):
        organization = kwargs['organization']
        team = kwargs['team']

        script = request.json.get('file', None)
        if script is None or script == '':
            return error_message(EINVAL, 'Field file is required'), 400
        if '..' in script:
            return error_message(
                EINVAL,
                'Referencing to Upper level directory is not allowed'), 401

        script_type = request.json.get('script_type', None)
        if script_type is None:
            return error_message(EINVAL, 'Field script_type is required'), 400

        if script_type == 'user_scripts':
            root = get_user_scripts_root(team=team, organization=organization)
        elif script_type == 'backing_scripts':
            root = get_back_scripts_root(team=team, organization=organization)
        else:
            return error_message(EINVAL,
                                 'Unsupported script type ' + script_type), 400

        if not os.path.exists(root / script):
            return error_message(ENOENT, 'file/directory doesn\'t exist'), 404

        if os.path.isfile(root / script):
            try:
                os.remove(root / script)
            except OSError as err:
                print(err)
                return error_message(
                    EIO, 'Error happened while deleting a file: '), 401
        else:
            try:
                shutil.rmtree(root / script)
            except OSError as err:
                print(err)
                return error_message(
                    EIO, 'Error happened while deleting a directory'), 401

        if script_type == 'user_scripts':
            cnt = Test.objects(path=os.path.abspath(root / script)).delete()
            if cnt == 0:
                return error_message(ENOENT, 'Test suite not found'), 404
Ejemplo n.º 6
0
def install_test_suite(package, user, organization, team, pypi_root, proprietary, version=None, new_package=True):
    if version is None:
        version = package.latest_version
        pkg_file = package.files[0]
    else:
        pkg_file = package.get_package_by_version(version)
        if not pkg_file:
            current_app.logger.error(f'package file not found for {package.name} with version {version}')
            return False
    pkg_file = pypi_root / package.package_name / pkg_file

    requires = get_package_requires(str(pkg_file), organization, team, type='Test Suite')
    if requires:
        for pkg, ver in requires:
            ret = install_test_suite(pkg, user, organization, team, pypi_root, proprietary, version=ver, new_package=False)
            if not ret:
                current_app.logger.error(f'Failed to install dependent package {package.name}')
                return False

    scripts_root = get_user_scripts_root(organization=organization, team=team)
    libraries_root = get_back_scripts_root(organization=organization, team=team)
    with zipfile.ZipFile(pkg_file) as zf:
        for f in zf.namelist():
            if f.startswith('EGG-INFO'):
                continue
            dirname = os.path.dirname(f)
            if os.path.exists(scripts_root / dirname):
                shutil.rmtree(scripts_root / dirname)
            if os.path.exists(libraries_root / dirname):
                shutil.rmtree(libraries_root / dirname)

    with zipfile.ZipFile(pkg_file) as zf:
        libraries = (f for f in zf.namelist() if not f.startswith('EGG-INFO') and '/scripts/' not in f)
        for l in libraries:
            zf.extract(l, libraries_root)
        scripts = [f for f in zf.namelist() if '/scripts/' in f]
        for s in scripts:
            zf.extract(s, scripts_root)
        for pkg_name in set((s.split('/', 1)[0] for s in scripts)):
            for f in os.listdir(scripts_root / pkg_name / 'scripts'):
                shutil.move(str(scripts_root / pkg_name / 'scripts' / f), scripts_root / pkg_name)
                db_update_test(scripts_root, os.path.join(pkg_name, f), user, organization, team, package, version)
            shutil.rmtree(scripts_root / pkg_name / 'scripts')
    package.modify(inc__download_times=1)
    return True
Ejemplo n.º 7
0
    def get(self, **kwargs):
        """
        A compound get method for returning script file list or script file content
        
        When field file is None, return the file list as per script_type, otherwise return the specified file.
        """
        script_path = request.args.get('file', default=None)
        script_type = request.args.get('script_type', default=None)

        organization = kwargs['organization']
        team = kwargs['team']

        test_libraries_root = get_back_scripts_root(team=team,
                                                    organization=organization)
        test_scripts_root = get_user_scripts_root(team=team,
                                                  organization=organization)

        if script_path:
            if script_type is None:
                return response_message(EINVAL,
                                        'Field script_type is required'), 400
            if script_type == 'test_scripts':
                return send_from_directory(
                    Path(os.getcwd()) / test_scripts_root, script_path)
            elif script_type == 'test_libraries':
                return send_from_directory(
                    Path(os.getcwd()) / test_libraries_root, script_path)
            else:
                return response_message(
                    EINVAL, 'Unsupported script type ' + script_type), 400
        elif script_type:
            return response_message(EINVAL, 'Field file is required'), 400

        test_scripts = path_to_dict(test_scripts_root)
        test_libraries = path_to_dict(test_libraries_root)
        return {'test_scripts': test_scripts, 'test_libraries': test_libraries}
Ejemplo n.º 8
0
    def post(self, **kwargs):
        """Update the script file content"""
        script = request.json.get('file', None)
        if not script:
            return response_message(EINVAL, 'Field file is required'), 400
        if not is_path_secure(script):
            return response_message(EINVAL, 'Illegal file path'), 401

        new_name = request.json.get('new_name', None)
        if new_name:
            if not is_path_secure(new_name):
                return response_message(
                    EINVAL,
                    'Referencing to Upper level directory is not allowed'), 401

        script_type = request.json.get('script_type', None)
        if script_type is None:
            return response_message(EINVAL,
                                    'Field script_type is required'), 400

        content = request.json.get('content', None)
        if content is None and new_name is None:
            return response_message(EINVAL, 'Field content is required'), 400

        organization = kwargs['organization']
        team = kwargs['team']
        user = kwargs['user']
        package = None

        if script_type == 'test_scripts':
            root = get_user_scripts_root(team=team, organization=organization)
        elif script_type == 'test_libraries':
            root = get_back_scripts_root(team=team, organization=organization)
        else:
            return response_message(EINVAL, 'Unsupported script type ' +
                                    script_type), 400

        dirname = os.path.dirname(script)
        basename = os.path.basename(script)

        if script_type == 'test_scripts' and basename.endswith('.md'):
            test = Test.objects(test_suite=os.path.splitext(basename)[0],
                                path=dirname).first()
        elif script_type == 'test_libraries' and dirname:
            package = Package.objects(py_packages=dirname).first()
            if not package:
                return response_message(ENOENT, 'package not found'), 404

        try:
            os.makedirs(root / dirname)
        except FileExistsError:
            pass

        if content or content == '':
            if script_type == 'test_scripts':
                content = re.sub(r'\\([{}*_\.])', r'\1', content)
            elif script_type == 'test_libraries':
                content = re.sub(r'\r\n', '\n', content)

            if basename:
                with open(root / script, 'w', encoding='utf-8') as f:
                    f.write(content)

        if new_name:
            if basename:
                new_path = os.path.join(dirname, new_name)
                os.rename(root / script, root / new_path)
                if script_type == 'test_scripts' and test:
                    test.modify(test_suite=os.path.splitext(new_name)[0])
            else:
                os.rename(root / script,
                          root / os.path.dirname(dirname) / new_name)

        if basename and script_type == 'test_scripts':
            _script = str(Path(dirname) / new_name) if new_name else script
            if _script.endswith('.md'):
                ret = db_update_test(root, _script, user, organization, team)
                if not ret:
                    return response_message(UNKNOWN_ERROR,
                                            'Failed to update test suite'), 401

        if script_type == 'test_libraries' and package:
            package.modify(modified=True)

        return response_message(SUCCESS)
Ejemplo n.º 9
0
    def post(self, **kwargs):
        script = request.json.get('file', None)
        if script is None or script == '':
            return error_message(EINVAL, 'Field file is required'), 400
        if '..' in script:
            return error_message(
                EINVAL,
                'Referencing to Upper level directory is not allowed'), 401

        new_name = request.json.get('new_name', None)
        if new_name:
            if '..' in new_name:
                return error_message(
                    EINVAL,
                    'Referencing to Upper level directory is not allowed'), 401

        script_type = request.json.get('script_type', None)
        if script_type is None:
            return error_message(EINVAL, 'Field script_type is required'), 400

        content = request.json.get('content', None)
        if content is None and new_name is None:
            return error_message(EINVAL, 'Field content is required'), 400

        organization = kwargs['organization']
        team = kwargs['team']
        user = kwargs['user']

        if script_type == 'user_scripts':
            root = get_user_scripts_root(team=team, organization=organization)
        elif script_type == 'backing_scripts':
            root = get_back_scripts_root(team=team, organization=organization)
        else:
            return error_message(EINVAL,
                                 'Unsupported script type ' + script_type), 400

        if content:
            if script_type == 'user_scripts':
                content = re.sub(r'\\([{}_])', r'\1', content)
            elif script_type == 'backing_scripts':
                content = re.sub(r'\r\n', '\n', content)

            dirname = os.path.dirname(script)
            try:
                os.makedirs(root / dirname)
            except FileExistsError:
                pass

            if dirname != script:
                with open(root / script, 'w') as f:
                    f.write(content)

        if new_name:
            os.rename(root / script, root / os.path.dirname(script) / new_name)

        if script_type == 'user_scripts':
            _script = str(Path(os.path.dirname(script)) /
                          new_name) if new_name else script
            ret = db_update_test(scripts_dir=root,
                                 script=_script,
                                 user=user.email,
                                 organization=organization,
                                 team=team)
            if ret:
                return error_message(UNKNOWN_ERROR,
                                     'Failed to update test suite'), 401
Ejemplo n.º 10
0
def process_task_per_endpoint(app, endpoint, organization=None, team=None):
    global ROBOT_PROCESSES, TASKS_CACHED

    if not organization and not team:
        app.logger.error('Argument organization and team must neither be None')
        return
    room_id = get_room_id(str(organization.id), str(team.id) if team else '')

    taskqueues = TaskQueue.objects(organization=organization,
                                   team=team,
                                   endpoint=endpoint)
    if taskqueues.count() == 0:
        app.logger.error('Taskqueue not found')
        return
    # taskqueues = [q for q in taskqueues]  # query becomes stale if the document it points to gets changed elsewhere, use document instead of query to perform deletion
    taskqueue_first = taskqueues.first()

    endpoint_id = str(endpoint.id)
    endpoint_uid = endpoint.uid
    org_name = team.organization.name + '-' + team.name if team else organization.name

    while True:
        taskqueue_first.reload('to_delete')
        if taskqueue_first.to_delete:
            taskqueues.delete()
            endpoint.delete()
            app.logger.info('Abort the task loop: {} @ {}'.format(
                org_name, endpoint_uid))
            break
        # TODO: lower priority tasks will take precedence if higher priority queue is empty first
        # but filled then when thread is searching for tasks in the lower priority task queues
        for priority in QUEUE_PRIORITY:
            exit_task = False
            for taskqueue in taskqueues:
                taskqueue.reload('to_delete')
                if taskqueue.to_delete:
                    exit_task = True
                    break
                if taskqueue.priority == priority:
                    break
            else:
                app.logger.error('Found task queue with unknown priority')
                continue

            if exit_task:
                break

            # "continue" to search for tasks in the lower priority task queues
            # "break" to start over to search for tasks from the top priority task queue
            task = taskqueue.pop()
            if not task:
                continue
            task_id = str(task.id)
            if isinstance(task, DBRef):
                app.logger.warning(
                    'task {} has been deleted, ignore it'.format(task_id))
                taskqueue.modify(running_task=None)
                break

            if task.kickedoff != 0 and not task.parallelization:
                app.logger.info(
                    'task has been taken over by other threads, do nothing')
                taskqueue.modify(running_task=None)
                break

            task.modify(inc__kickedoff=1)
            if task.kickedoff != 1 and not task.parallelization:
                app.logger.warning('a race condition happened')
                taskqueue.modify(running_task=None)
                break

            app.logger.info('Start to run task {} in the thread {}'.format(
                task_id,
                threading.current_thread().name))

            result_dir = get_test_result_path(task)
            scripts_dir = get_user_scripts_root(task)
            args = [
                'robot', '--loglevel', 'debug', '--outputdir',
                str(result_dir), '--extension', 'md', '--consolecolors', 'on',
                '--consolemarkers', 'on'
            ]
            os.makedirs(result_dir)

            if hasattr(task, 'testcases'):
                for t in task.testcases:
                    args.extend(['-t', t])

            if hasattr(task, 'variables'):
                variable_file = Path(result_dir) / 'variablefile.py'
                convert_json_to_robot_variable(args, task.variables,
                                               variable_file)

            addr, port = '127.0.0.1', 8270
            args.extend([
                '-v', f'address_daemon:{addr}', '-v', f'port_daemon:{port}',
                '-v', f'task_id:{task_id}', '-v',
                f'endpoint_uid:{endpoint_uid}'
            ])
            args.append(
                os.path.join(scripts_dir, task.test.path,
                             task.test.test_suite + '.md'))
            app.logger.info('Arguments: ' + str(args))

            p = subprocess.Popen(
                args,
                stdout=subprocess.PIPE,
                stderr=subprocess.STDOUT,
                bufsize=0,
                creationflags=subprocess.CREATE_NEW_PROCESS_GROUP
                if os.name == 'nt' else 0)
            ROBOT_PROCESSES[task.id] = p

            task.status = 'running'
            task.run_date = datetime.datetime.utcnow()
            task.endpoint_run = endpoint
            task.save()
            RPC_SOCKET.emit('task started', {'task_id': task_id}, room=room_id)

            log_msg = StringIO()
            if room_id not in ROOM_MESSAGES:
                ROOM_MESSAGES[room_id] = {task_id: log_msg}
            else:
                if task_id not in ROOM_MESSAGES[room_id]:
                    ROOM_MESSAGES[room_id][task_id] = log_msg
            ss = b''
            while True:
                c = p.stdout.read(1)
                if not c:
                    break
                try:
                    c = c.decode(encoding=sys.getdefaultencoding())
                except UnicodeDecodeError:
                    ss += c
                else:
                    c = '\r\n' if c == '\n' else c
                    log_msg.write(c)
                    RPC_SOCKET.emit('test report', {
                        'task_id': task_id,
                        'message': c
                    },
                                    room=room_id)
            del ROBOT_PROCESSES[task.id]

            if ss != b'':
                ss = ss.decode(chardet.detect(ss)['encoding'])
                log_msg_all = StringIO()
                log_msg_all.write(ss)
                log_msg_all.write(log_msg.getvalue())
                app.logger.info('\n' + log_msg_all.getvalue())
                RPC_SOCKET.emit('test report', {
                    'task_id': task_id,
                    'message': ss
                },
                                room=room_id)
            else:
                app.logger.info('\n' + log_msg.getvalue())
            #app.logger.info('\n' + log_msg.getvalue().replace('\r\n', '\n'))

            p.wait()
            if p.returncode == 0:
                task.status = 'successful'
            else:
                task.reload('status')
                if task.status != 'cancelled':
                    task.status = 'failed'
            task.save()
            RPC_SOCKET.emit('task finished', {
                'task_id': task_id,
                'status': task.status
            },
                            room=room_id)
            ROOM_MESSAGES[room_id][task_id].close()
            del ROOM_MESSAGES[room_id][task_id]
            if task_id in TASKS_CACHED:
                del TASKS_CACHED[task_id]

            taskqueue.modify(running_task=None)
            endpoint.modify(last_run_date=datetime.datetime.utcnow())

            if task.upload_dir:
                resource_dir_tmp = get_upload_files_root(task)
                if os.path.exists(resource_dir_tmp):
                    make_tarfile_from_dir(str(result_dir / 'resource.tar.gz'),
                                          resource_dir_tmp)

            result_dir_tmp = result_dir / 'temp'
            if os.path.exists(result_dir_tmp):
                shutil.rmtree(result_dir_tmp)

            notification_chain_call(task)
            TASK_LOCK.acquire()
            TASK_THREADS[endpoint_id] = 1
            TASK_LOCK.release()
            break
        else:
            TASK_LOCK.acquire()
            if TASK_THREADS[endpoint_id] != 1:
                TASK_THREADS[endpoint_id] = 1
                TASK_LOCK.release()
                app.logger.info('Run the lately scheduled task')
                continue
            del TASK_THREADS[endpoint_id]
            TASK_LOCK.release()
            break