def upload_ftp(db, path) -> None: """ Make a FTP upload. :param db: database :type db: database cursor :param path: path to sample file :type path: str """ from run import log, config upload_path = str(path) path_parts = upload_path.split(os.path.sep) # We assume /home/{uid}/ as specified in the model user_id = path_parts[2] user = User.query.filter(User.id == user_id).first() if user is None: log.critical(f"Did not find a user for user_id {user_id}!") return filename, uploaded_file_extension = os.path.splitext(path) log.debug(f"Checking if {upload_path} has a forbidden extension") if remove_forbidden_file(uploaded_file_extension, upload_path, user): return uploaded_mime_type = magic.from_file(upload_path, mime=True) forbidden_mime_types = ForbiddenMimeType.query.filter( ForbiddenMimeType.mimetype == uploaded_mime_type).first() if forbidden_mime_types is not None: log.error( f"User {user.name} tried to upload a file with a forbidden MIME type ({uploaded_mime_type})!" ) os.remove(upload_path) return guessed_extension = mimetypes.guess_extension(uploaded_mime_type) log.debug( f"Mimetype {uploaded_mime_type} creates guessed extension {guessed_extension}" ) if guessed_extension is not None and remove_forbidden_file( guessed_extension, upload_path, user): return log.debug("Moving file to temporary folder and changing permissions...") filename = secure_filename(upload_path.replace(f"/home/{user.id}/", '')) intermediate_path = os.path.join(config.get('SAMPLE_REPOSITORY', ''), 'TempFiles', filename) log.debug(f"Copy {upload_path} to {intermediate_path}") shutil.copy(upload_path, intermediate_path) os.remove(upload_path) log.debug(f"Checking hash value for {intermediate_path}") file_hash = create_hash_for_sample(intermediate_path) if sample_already_uploaded(file_hash): log.debug(f"Sample already exists: {intermediate_path}") os.remove(intermediate_path) else: add_sample_to_queue(file_hash, intermediate_path, user.id, db)
def queue_test(db, gh_commit, commit, test_type, branch="master", pr_nr=0): from run import log fork = Fork.query.filter(Fork.github.like( "%/CCExtractor/ccextractor.git")).first() if test_type == TestType.pull_request: branch = "pull_request" # Create Linux test entry linux = Test(TestPlatform.linux, test_type, fork.id, branch, commit, pr_nr) db.add(linux) # Create Windows test entry # windows = Test(TestPlatform.windows, test_type, fork.id, branch, # commit, pr_nr) # db.add(windows) db.commit() # Update statuses on GitHub try: gh_commit.post( state=Status.PENDING, description="Tests queued", context="CI - %s" % linux.platform.value, target_url=url_for( 'test.by_id', test_id=linux.id, _external=True)) # gh_commit.post( # state=Status.PENDING, description="Tests queued", # context="CI - %s" % windows.platform.value, # target_url=url_for( # 'test.by_id', test_id=windows.id, _external=True)) except ApiError as a: log.critical('Could not post to GitHub! Response: %s' % a.response) return # Kick off KVM process start_ci_vm(db)
def queue_test(db, gh_commit, commit, test_type, branch="master", pr_nr=0): """ Function to store test details into Test model for each platform, and post the status to GitHub. :param db: Database connection. :type db: sqlalchemy.orm.scoped_session :param gh_commit: The GitHub API call for the commit. Can be None :type gh_commit: Any :param commit: The commit hash. :type commit: str :param test_type: The type of test :type test_type: TestType :param branch: Branch name :type branch: str :param pr_nr: Pull Request number, if applicable. :type pr_nr: int :return: Nothing :rtype: None """ from run import log fork = Fork.query.filter( Fork.github.like("%/CCExtractor/ccextractor.git")).first() if test_type == TestType.pull_request: branch = "pull_request" linux_test = Test(TestPlatform.linux, test_type, fork.id, branch, commit, pr_nr) db.add(linux_test) windows_test = Test(TestPlatform.windows, test_type, fork.id, branch, commit, pr_nr) db.add(windows_test) db.commit() if gh_commit is not None: status_entries = { linux_test.platform.value: linux_test.id, windows_test.platform.value: windows_test.id } for platform_name, test_id in status_entries.items(): try: gh_commit.post( state=Status.PENDING, description="Tests queued", context="CI - {name}".format(name=platform_name), target_url=url_for('test.by_id', test_id=test_id, _external=True)) except ApiError as a: log.critical( 'Could not post to GitHub! Response: {res}'.format( res=a.response)) log.debug("Created tests, waiting for cron...")
def queue_test(db, repository, gh_commit, commit, test_type, branch="master", pr_nr=0): """ Function to store test details into Test model seperately for various vm. Post status to the github """ from run import log fork = Fork.query.filter( Fork.github.like("%/CCExtractor/ccextractor.git")).first() if test_type == TestType.pull_request: branch = "pull_request" # Create Linux test entry linux = Test(TestPlatform.linux, test_type, fork.id, branch, commit, pr_nr) db.add(linux) # Create Windows test entry windows = Test(TestPlatform.windows, test_type, fork.id, branch, commit, pr_nr) db.add(windows) db.commit() # Update statuses on GitHub try: gh_commit.post(state=Status.PENDING, description="Tests queued", context="CI - %s" % linux.platform.value, target_url=url_for('test.by_id', test_id=linux.id, _external=True)) except ApiError as a: log.critical('Could not post to GitHub! Response: %s' % a.response) try: gh_commit.post(state=Status.PENDING, description="Tests queued", context="CI - %s" % windows.platform.value, target_url=url_for('test.by_id', test_id=windows.id, _external=True)) except ApiError as a: log.critical('Could not post to GitHub! Response: %s' % a.response) # We wait for the cron to kick off the CI VM's log.debug("Created tests, waiting for cron...")
def queue_test(db, gh_commit, commit, test_type, branch="master", pr_nr=0): """ Function to store test details into Test model separately for various vm. Post status to GitHub """ from run import log fork = Fork.query.filter(Fork.github.like( "%/CCExtractor/ccextractor.git")).first() if test_type == TestType.pull_request: branch = "pull_request" # Create Linux test entry linux = Test(TestPlatform.linux, test_type, fork.id, branch, commit, pr_nr) db.add(linux) # Create Windows test entry windows = Test(TestPlatform.windows, test_type, fork.id, branch, commit, pr_nr) db.add(windows) db.commit() # Update statuses on GitHub if gh_commit is not None: test_ids = [linux.id, windows.id] ci_names = [linux.platform.value, windows.platform.value] for idx in range(len(ci_names)): try: gh_commit.post( state=Status.PENDING, description="Tests queued", context="CI - {name}".format(name=ci_names[idx]), target_url=url_for( 'test.by_id', test_id=test_ids[idx], _external=True)) except ApiError as a: log.critical( 'Could not post to GitHub! Response: {res}'.format( res=a.response) ) # We wait for the cron to kick off the CI VM's log.debug("Created tests, waiting for cron...")
def kvm_processor(db, kvm_name, platform, repository, delay): """ Checks whether there is no already running same kvm. Checks whether machine is in maintenance mode or not Launch kvm if not used by any other test Creates testing xml files to test the change in main repo. Creates clone with separate branch and merge pr into it. """ from run import config, log, app log.info("[{platform}] Running kvm_processor".format(platform=platform)) if kvm_name == "": log.critical('[{platform}] KVM name is empty!') return if delay is not None: import time log.debug('[{platform}] Sleeping for {time} seconds'.format( platform=platform, time=delay)) time.sleep(delay) maintenance_mode = MaintenanceMode.query.filter( MaintenanceMode.platform == platform).first() if maintenance_mode is not None and maintenance_mode.disabled: log.debug('[{platform}] In maintenance mode! Waiting...').format( platform=platform) return # Open connection to libvirt conn = libvirt.open("qemu:///system") if conn is None: log.critical( "[{platform}] Couldn't open connection to libvirt!".format( platform=platform)) return try: vm = conn.lookupByName(kvm_name) except libvirt.libvirtError: log.critical("[{platform}] No VM named {name} found!".format( platform=platform, name=kvm_name)) return vm_info = vm.info() if vm_info[0] != libvirt.VIR_DOMAIN_SHUTOFF: # Running, check expiry (2 hours runtime max) status = Kvm.query.filter(Kvm.name == kvm_name).first() max_runtime = config.get("KVM_MAX_RUNTIME", 120) if status is not None: if datetime.datetime.now( ) - status.timestamp >= datetime.timedelta(minutes=max_runtime): # Mark entry as aborted test_progress = TestProgress(status.test.id, TestStatus.canceled, 'Runtime exceeded') db.add(test_progress) db.delete(status) db.commit() # Abort process if vm.destroy() == -1: # Failed to shut down log.critical( "[{platform}] Failed to shut down {name}".format( platform=platform, name=kvm_name)) return else: log.info("[{platform}] Current job not expired yet.".format( platform=platform)) return else: log.warn( "[{platform}] No task, but VM is running! Hard reset necessary" .format(platform=platform)) if vm.destroy() == -1: # Failed to shut down log.critical("[{platform}] Failed to shut down {name}".format( platform=platform, name=kvm_name)) return # Check if there's no KVM status left status = Kvm.query.filter(Kvm.name == kvm_name).first() if status is not None: log.warn( "[{platform}] KVM is powered off, but test {id} still present". format(platform=platform, id=status.test.id)) db.delete(status) db.commit() # Get oldest test for this platform finished_tests = db.query(TestProgress.test_id).filter( TestProgress.status.in_([TestStatus.canceled, TestStatus.completed])).subquery() test = Test.query.filter( and_(Test.id.notin_(finished_tests), Test.platform == platform)).order_by(Test.id.asc()).first() if test is None: log.info('[{platform}] No more tests to run, returning'.format( platform=platform)) return if test.test_type == TestType.pull_request and test.pr_nr == 0: log.warn('[{platform}] Test {id} is invalid, deleting'.format( platform=platform, id=test.id)) db.delete(test) db.commit() return # Reset to snapshot if vm.hasCurrentSnapshot() != 1: log.critical( "[{platform}] VM {name} has no current snapshot set!".format( platform=platform, name=kvm_name)) return snapshot = vm.snapshotCurrent() if vm.revertToSnapshot(snapshot) == -1: log.critical( "[{platform}] Failed to revert to {snapshot} for {name}".format( platform=platform, snapshot=snapshot.getName(), name=kvm_name)) return log.info("[{p}] Reverted to {snap} for {name}".format( p=platform, snap=snapshot.getName(), name=kvm_name)) log.debug('Starting test {id}'.format(id=test.id)) status = Kvm(kvm_name, test.id) # Prepare data # 0) Write url to file with app.app_context(): full_url = url_for('ci.progress_reporter', test_id=test.id, token=test.token, _external=True, _scheme="https") file_path = os.path.join(config.get('SAMPLE_REPOSITORY', ''), 'vm_data', kvm_name, 'reportURL') with open(file_path, 'w') as f: f.write(full_url) # 1) Generate test files base_folder = os.path.join(config.get('SAMPLE_REPOSITORY', ''), 'vm_data', kvm_name, 'ci-tests') categories = Category.query.order_by(Category.id.desc()).all() commit_name = 'fetch_commit_' + platform.value commit_hash = GeneralData.query.filter( GeneralData.key == commit_name).first().value last_commit = Test.query.filter( and_(Test.commit == commit_hash, Test.platform == platform)).first() log.debug("[{p}] We will compare against the results of test {id}".format( p=platform, id=last_commit.id)) # Init collection file multi_test = etree.Element('multitest') for category in categories: # Skip categories without tests if len(category.regression_tests) == 0: continue # Create XML file for test file_name = '{name}.xml'.format(name=category.name) single_test = etree.Element('tests') for regression_test in category.regression_tests: entry = etree.SubElement(single_test, 'entry', id=str(regression_test.id)) command = etree.SubElement(entry, 'command') command.text = regression_test.command input_node = etree.SubElement( entry, 'input', type=regression_test.input_type.value) # Need a path that is relative to the folder we provide inside the CI environment. input_node.text = regression_test.sample.filename output_node = etree.SubElement(entry, 'output') output_node.text = regression_test.output_type.value compare = etree.SubElement(entry, 'compare') last_files = TestResultFile.query.filter( and_(TestResultFile.test_id == last_commit.id, TestResultFile.regression_test_id == regression_test.id)).subquery() for output_file in regression_test.output_files: ignore_file = str(output_file.ignore).lower() file_node = etree.SubElement(compare, 'file', ignore=ignore_file, id=str(output_file.id)) last_commit_files = db.query(last_files.c.got).filter( and_( last_files.c.regression_test_output_id == output_file.id, last_files.c.got.isnot(None))).first() correct = etree.SubElement(file_node, 'correct') # Need a path that is relative to the folder we provide inside the CI environment. if last_commit_files is None: correct.text = output_file.filename_correct else: correct.text = output_file.create_correct_filename( last_commit_files[0]) expected = etree.SubElement(file_node, 'expected') expected.text = output_file.filename_expected( regression_test.sample.sha) # Save XML single_test.getroottree().write(os.path.join(base_folder, file_name), encoding='utf-8', xml_declaration=True, pretty_print=True) # Append to collection file test_file = etree.SubElement(multi_test, 'testfile') location = etree.SubElement(test_file, 'location') location.text = file_name # Save collection file multi_test.getroottree().write(os.path.join(base_folder, 'TestAll.xml'), encoding='utf-8', xml_declaration=True, pretty_print=True) # 2) Create git repo clone and merge PR into it (if necessary) try: repo = Repo( os.path.join(config.get('SAMPLE_REPOSITORY', ''), 'vm_data', kvm_name, 'unsafe-ccextractor')) except InvalidGitRepositoryError: log.critical( "[{platform}] Could not open CCExtractor's repository copy!". format(platform=platform)) return # Return to master repo.heads.master.checkout(True) # Update repository from upstream try: origin = repo.remote('origin') except ValueError: log.critical("[{platform}] Origin remote doesn't exist!".format( platform=platform)) return fetch_info = origin.fetch() if len(fetch_info) == 0: log.info( '[{platform}] Fetch from remote returned no new data...'.format( platform=platform)) # Pull code (finally) pull_info = origin.pull() if len(pull_info) == 0: log.info( "[{platform}] Pull from remote returned no new data...".format( platform=platform)) if pull_info[0].flags > 128: log.critical( "[{platform}] Didn't pull any information from remote: {flags}!". format(platform=platform, flags=pull_info[0].flags)) return # Delete the test branch if it exists, and recreate try: repo.delete_head('CI_Branch', force=True) except GitCommandError: log.warn("[{platform}] Could not delete CI_Branch head".format( platform=platform)) # Remove possible left rebase-apply directory try: shutil.rmtree( os.path.join(config.get('SAMPLE_REPOSITORY', ''), 'unsafe-ccextractor', '.git', 'rebase-apply')) except OSError: log.warn("[{platform}] Could not delete rebase-apply".format( platform=platform)) # If PR, merge, otherwise reset to commit if test.test_type == TestType.pull_request: # Fetch PR (stored under origin/pull/<id>/head pull_info = origin.fetch( 'pull/{id}/head:CI_Branch'.format(id=test.pr_nr)) if len(pull_info) == 0: log.warn( "[{platform}] Didn't pull any information from remote PR!". format(platform=platform)) if pull_info[0].flags > 128: log.critical( "[{platform}] Didn't pull any information from remote PR: {flags}!" .format(platform=platform, flags=pull_info[0].flags)) return try: test_branch = repo.heads['CI_Branch'] except IndexError: log.critical('CI_Branch does not exist') return test_branch.checkout(True) # TODO: check what happens on merge conflicts # Rebase on master # try: # repo.git.rebase('master') # except GitCommandError: # progress = TestProgress( # test.id, TestStatus.preparation, # 'Rebase on master' # ) # db.add(progress) # progress = TestProgress( # test.id, TestStatus.canceled, # 'Merge conflict, please resolve.' # ) # db.add(progress) # db.commit() # # Report back # gh_commit = repository.statuses(test.commit) # # with app.app_context(): # target_url = url_for( # 'test.by_id', test_id=test.id, _external=True) # context = "CI - %s" % test.platform.value # gh_commit.post( # state=Status.ERROR, description='Failed to rebase', # context=context, target_url=target_url) # # Return, so next one can be handled # return else: test_branch = repo.create_head('CI_Branch', 'HEAD') test_branch.checkout(True) try: repo.head.reset(test.commit, working_tree=True) except GitCommandError: log.warn( "[{platform}] Commit {hash} for test {id} does not exist!". format(platform=platform, hash=test.commit, id=test.id)) return # Power on machine try: vm.create() db.add(status) db.commit() except libvirt.libvirtError: log.critical("[{platform}] Failed to launch VM {name}".format( platform=platform, name=kvm_name)) except IntegrityError: log.warn("[{platform}] Duplicate entry for {id}".format( platform=platform, id=test.id))
def process_id(upload_id): """ Process the sample that is uploaded to the platform. :param upload_id: The identity of uploaded file that will be processed :type upload_id: str :return: Process progress in form and queue the sample :rtype: str """ from run import config, log queued_sample = QueuedSample.query.filter( QueuedSample.id == upload_id).first() if queued_sample is not None: if queued_sample.user_id == g.user.id: versions = CCExtractorVersion.query.all() form = FinishQueuedSampleForm(request.form) form.version.choices = [(v.id, v.version) for v in versions] if form.validate_on_submit(): db_committed = False repo_folder = config.get('SAMPLE_REPOSITORY', '') temp_path = os.path.join(repo_folder, 'QueuedFiles', queued_sample.filename) final_path = os.path.join(repo_folder, 'TestFiles', queued_sample.filename) try: extension = queued_sample.extension[1:] if len( queued_sample.extension) > 0 else "" sample = Sample(queued_sample.sha, extension, queued_sample.original_name) g.db.add(sample) g.db.flush([sample]) uploaded = Upload(g.user.id, sample.id, form.version.data, Platform.from_string(form.platform.data), form.parameters.data, form.notes.data) g.db.add(uploaded) g.db.delete(queued_sample) g.db.commit() db_committed = True except Exception: traceback.print_exc() g.db.rollback() if db_committed: if form.report.data == 'y': data = "" try: kvm_name = config.get('KVM_LINUX_NAME', '') repo = Repo( os.path.join(repo_folder, 'vm_data', kvm_name, 'unsafe-ccextractor')) data = repo.git.show( f"{repo.heads.master}:.github/ISSUE_TEMPLATE.md" ) except InvalidGitRepositoryError: log.critical( "Could not open CCExtractor's repository") version = CCExtractorVersion.query.filter( CCExtractorVersion.id == form.version.data).first() data = data.replace("**X.X**", version.version) data = data.replace("[ ] I have read", "[X] I have read") data = data.replace("[ ] I have checked", "[X] I have checked") data = data.replace("[ ] I have used", "[X] I have used") data = data.replace( "[ ] I am an active contributor to CCExtractor.", "[X] I used the platform to submit this issue!") data = data.replace("`-autoprogram`", f"`{form.parameters.data}`") platform = form.platform.data.title() data = data.replace('[ ] ' + platform, '[X] ' + platform) # Remove everything starting from the video links data = data[:data.find('**Video links**')] # Append our own content here sample_link = url_for('sample.sample_by_id', sample_id=sample.id, _external=True) data += f""" **Sample** [Sample {sample.id}]({sample_link}) uploaded on the Sample Platform. **Extra information** *Notes:* {form.notes.data} *Description:* {form.IssueBody.data}""" issue_title = f"[BUG] {form.IssueTitle.data}" issue_data = make_github_issue( issue_title, data, ['bug', f'sample_{sample.id}']) if issue_data != 'ERROR': issue_id = issue_data['number'] issue_title = issue_data['title'] issue_user = issue_data['user']['login'] issue_date = issue_data['created_at'] issue_status = issue_data['state'] issue = Issue(sample.id, issue_id, issue_date, issue_title, issue_user, issue_status) g.db.add(issue) g.db.commit() else: flash( "Could not submit an issue on GitHub (did you revoke permissions for the platform?)." " Please submit it manually.") os.rename(temp_path, final_path) return redirect( url_for('sample.sample_by_id', sample_id=sample.id)) return {'form': form, 'queued_sample': queued_sample} # Raise error raise QueuedSampleNotFoundException()
def process_id(upload_id): """ Process the sample that is uploaded to the platform :param upload_id: The identity of uploaded file that will be processed :type upload_id: str :return: Process progress in form and queue the sample :rtype: str """ from run import config, log # Fetch upload id queued_sample = QueuedSample.query.filter(QueuedSample.id == upload_id).first() if queued_sample is not None: if queued_sample.user_id == g.user.id: # Allowed to process versions = CCExtractorVersion.query.all() form = FinishQueuedSampleForm(request.form) form.version.choices = [(v.id, v.version) for v in versions] if form.validate_on_submit(): # Store in DB db_committed = False repo_folder = config.get('SAMPLE_REPOSITORY', '') temp_path = os.path.join(repo_folder, 'QueuedFiles', queued_sample.filename) final_path = os.path.join(repo_folder, 'TestFiles', queued_sample.filename) try: extension = queued_sample.extension[1:] if len(queued_sample.extension) > 0 else "" sample = Sample(queued_sample.sha, extension, queued_sample.original_name) g.db.add(sample) g.db.flush([sample]) uploaded = Upload( g.user.id, sample.id, form.version.data, Platform.from_string(form.platform.data), form.parameters.data, form.notes.data ) g.db.add(uploaded) g.db.delete(queued_sample) g.db.commit() db_committed = True except Exception: traceback.print_exc() g.db.rollback() # Move file if db_committed: if form.report.data == 'y': data = "" try: kvm_name = config.get('KVM_LINUX_NAME', '') repo = Repo(os.path.join(repo_folder, 'vm_data', kvm_name, 'unsafe-ccextractor')) data = repo.git.show('{branch}:{file}'.format( branch=repo.heads.master, file='.github/ISSUE_TEMPLATE.md') ) except InvalidGitRepositoryError: log.critical(" Could not open CCExtractor's repository ") version = CCExtractorVersion.query.filter(CCExtractorVersion.id == form.version.data).first() data = data.replace('**X.X**', version.version) data = data.replace('[ ] I have read', '[X] I have read') data = data.replace('[ ] I have checked', '[X] I have checked') data = data.replace('[ ] I have used', '[X] I have used') data = data.replace( '[ ] I am an active contributor to CCExtractor.', '[X] I used the platform to submit this issue!' ) data = data.replace( '`-autoprogram`', '`{param}`'.format(param=form.parameters.data, version=form.version.data) ) platform = form.platform.data.title() data = data.replace('[ ] ' + platform, '[X] ' + platform) # Remove everything starting from the video links data = data[:data.find('**Video links**')] # Append our own content here sample_link = url_for('sample.sample_by_id', sample_id=sample.id, _external=True) data += '**Sample**\n\n[Sample {id}]({link}) was uploaded on the sample platform.\n'.format( id=sample.id, link=sample_link) data += '**Extra information**\n\n*Notes:*\n{notes}\n*Description:*\n{desc}'.format( notes=form.notes.data, desc=form.IssueBody.data) issue_title = '[BUG] {data}'.format(data=form.IssueTitle.data) issue_data = make_github_issue(issue_title, data, ['bug', 'sample' + str(sample.id)]) if issue_data != 'ERROR': issue_id = issue_data['number'] issue_title = issue_data['title'] issue_user = issue_data['user']['login'] issue_date = issue_data['created_at'] issue_status = issue_data['state'] issue = Issue(sample.id, issue_id, issue_date, issue_title, issue_user, issue_status) g.db.add(issue) g.db.commit() else: flash('Could not submit an issue on GitHub (did you revoke permissions for the platform?).' ' Please submit it manually.') os.rename(temp_path, final_path) return redirect( url_for('sample.sample_by_id', sample_id=sample.id)) return { 'form': form, 'queued_sample': queued_sample } # Raise error raise QueuedSampleNotFoundException()
def kvm_processor(db, kvm_name, platform, repository, delay): from run import config, log, app if kvm_name == "": log.critical('KVM name is empty!') return if delay is not None: import time log.debug('Sleeping for {time} seconds'.format(time=delay)) time.sleep(delay) # Open connection to libvirt conn = libvirt.open("qemu:///system") if conn is None: log.critical("Couldn't open connection to libvirt!") return try: vm = conn.lookupByName(kvm_name) except libvirt.libvirtError: log.critical("Couldn't find the Linux CI machine named %s" % kvm_name) return vm_info = vm.info() if vm_info[0] != libvirt.VIR_DOMAIN_SHUTOFF: # Running, check expiry (2 hours runtime max) status = Kvm.query.filter(Kvm.name == kvm_name).first() max_runtime = config.get("KVM_MAX_RUNTIME", 120) if status is not None: if datetime.datetime.now() >= status.timestamp + \ datetime.timedelta(minutes=max_runtime): # Mark entry as aborted test_progress = TestProgress(status.test.id, TestStatus.canceled, 'Runtime exceeded') db.add(test_progress) db.delete(status) db.commit() # Abort process if vm.destroy() == -1: # Failed to shut down log.critical("Failed to shut down %s" % kvm_name) return else: log.info("Current job is still running and not expired") return else: log.warn("No currently running task, but VM is running! Hard " "reset necessary") if vm.destroy() == -1: # Failed to shut down log.critical("Failed to shut down %s" % kvm_name) return # Check if there's no KVM status left status = Kvm.query.filter(Kvm.name == kvm_name).first() if status is not None: log.warn("KVM is powered off, but test is still in there: %s" % status.test.id) db.delete(status) db.commit() # Get oldest test for this platform finished_tests = db.query(TestProgress.test_id).filter( TestProgress.status.in_([TestStatus.canceled, TestStatus.completed])).subquery() test = Test.query.filter( and_(Test.id.notin_(finished_tests), Test.platform == platform)).order_by(Test.id.asc()).first() if test is None: log.info('No more tests to run, returning') return if test.test_type == TestType.pull_request and test.pr_nr == 0: log.warn('Got an invalid test with number %s, deleting' % test.id) db.delete(test) db.commit() return # Reset to snapshot if vm.hasCurrentSnapshot() != 1: log.critical("VM %s has no current snapshot set!" % kvm_name) return snapshot = vm.snapshotCurrent() if vm.revertToSnapshot(snapshot) == -1: log.critical("Failed to revert to snapshot %s for VM %s" % (snapshot.getName(), kvm_name)) return log.info('Reverted to snapshot %s for VM %s' % (snapshot.getName(), kvm_name)) log.debug('Starting test %s' % test.id) status = Kvm(kvm_name, test.id) # Prepare data # 0) Write url to file with app.app_context(): full_url = url_for('ci.progress_reporter', test_id=test.id, token=test.token, _external=True, _scheme="https") file_path = os.path.join(config.get('SAMPLE_REPOSITORY', ''), 'reportURL') with open(file_path, 'w') as f: f.write(full_url) # 1) Generate test files base_folder = os.path.join(config.get('SAMPLE_REPOSITORY', ''), 'ci-tests') categories = Category.query.order_by(Category.id.desc()).all() # Init collection file multi_test = etree.Element('multitest') for category in categories: if len(category.regression_tests) == 0: # Skip categories without tests continue # Create XML file for test file_name = '{name}.xml'.format(name=category.name) single_test = etree.Element('tests') for regression_test in category.regression_tests: entry = etree.SubElement(single_test, 'entry', id=str(regression_test.id)) command = etree.SubElement(entry, 'command') command.text = regression_test.command input_node = etree.SubElement( entry, 'input', type=regression_test.input_type.value) # Need a path that is relative to the folder we provide # inside the CI environment. input_node.text = regression_test.sample.filename output_node = etree.SubElement(entry, 'output') output_node.text = regression_test.output_type.value compare = etree.SubElement(entry, 'compare') for output_file in regression_test.output_files: file_node = etree.SubElement( compare, 'file', ignore='true' if output_file.ignore else 'false', id=str(output_file.id)) correct = etree.SubElement(file_node, 'correct') # Need a path that is relative to the folder we provide # inside the CI environment. correct.text = output_file.filename_correct expected = etree.SubElement(file_node, 'expected') expected.text = output_file.filename_expected( regression_test.sample.sha) # Save XML single_test.getroottree().write(os.path.join(base_folder, file_name), encoding='utf-8', xml_declaration=True, pretty_print=True) # Append to collection file test_file = etree.SubElement(multi_test, 'testfile') location = etree.SubElement(test_file, 'location') location.text = file_name # Save collection file multi_test.getroottree().write(os.path.join(base_folder, 'TestAll.xml'), encoding='utf-8', xml_declaration=True, pretty_print=True) # 2) Create git repo clone and merge PR into it (if necessary) try: repo = Repo( os.path.join(config.get('SAMPLE_REPOSITORY', ''), 'unsafe-ccextractor')) except InvalidGitRepositoryError: log.critical('Could not open CCExtractor\'s repository copy!') return # Return to master repo.heads.master.checkout(True) # Update repository from upstream try: origin = repo.remote('origin') except ValueError: log.critical('Origin remote doesn\'t exist!') return fetch_info = origin.fetch() if len(fetch_info) == 0: log.warn('No info fetched from remote!') # Pull code (finally) pull_info = origin.pull() if len(pull_info) == 0: log.warn('Didn\'t pull any information from remote!') if pull_info[0].flags > 128: log.critical('Didn\'t pull any information from remote: %s!' % pull_info[0].flags) return # Delete the test branch if it exists, and recreate try: repo.delete_head('CI_Branch', force=True) except GitCommandError: log.warn('Could not delete CI_Branch head') # Remove possible left rebase-apply directory try: shutil.rmtree( os.path.join(config.get('SAMPLE_REPOSITORY', ''), 'unsafe-ccextractor', '.git', 'rebase-apply')) except OSError: log.warn('Could not delete rebase-apply') # If PR, merge, otherwise reset to commit if test.test_type == TestType.pull_request: # Fetch PR (stored under origin/pull/<id>/head pull_info = origin.fetch( 'pull/{id}/head:CI_Branch'.format(id=test.pr_nr)) if len(pull_info) == 0: log.warn('Didn\'t pull any information from remote PR!') if pull_info[0].flags > 128: log.critical('Didn\'t pull any information from remote PR: %s!' % pull_info[0].flags) return try: test_branch = repo.heads['CI_Branch'] except IndexError: log.critical('CI_Branch does not exist') return # Check out branch test_branch.checkout(True) # Rebase on master # try: # repo.git.rebase('master') # except GitCommandError: # progress = TestProgress( # test.id, TestStatus.preparation, # 'Rebase on master' # ) # db.add(progress) # progress = TestProgress( # test.id, TestStatus.canceled, # 'Merge conflict, please resolve.' # ) # db.add(progress) # db.commit() # # Report back # gh_commit = repository.statuses(test.commit) # # with app.app_context(): # target_url = url_for( # 'test.by_id', test_id=test.id, _external=True) # context = "CI - %s" % test.platform.value # gh_commit.post( # state=Status.ERROR, description='Failed to rebase', # context=context, target_url=target_url) # # Return, so next one can be handled # return # TODO: check what happens on merge conflicts else: test_branch = repo.create_head('CI_Branch', 'HEAD') # Check out branch for test purposes test_branch.checkout(True) try: repo.head.reset(test.commit, working_tree=True) except GitCommandError: log.warn('Git commit %s (test %s) does not exist!' % (test.commit, test.id)) return # Power on machine try: vm.create() db.add(status) db.commit() except libvirt.libvirtError: log.critical("Failed to launch VM %s" % kvm_name) except IntegrityError: log.warn("Duplicate entry for %s" % test.id)