def remove_forbidden_file(file_extension: str, file_path: str, user: User) -> bool: """ Remove the given file if the extension is not allowed. :param file_extension: The file extension to check. :type file_extension: str :param file_path: The path to the uploaded file. :type file_path: str :param user: The user that uploaded the file. :type user: User :return: True if the file was removed, False otherwise. :rtype: bool """ from run import log dotless_extension = file_extension[1:] log.debug(f"Retrieved {dotless_extension} from {file_extension}") forbidden = ForbiddenExtension.query.filter( ForbiddenExtension.extension == dotless_extension).first() if forbidden is not None: log.error( f"User {user.name} tried to upload a file with a forbidden extension ({dotless_extension})!" ) os.remove(file_path) return True return False
def upload_ftp(db, path) -> None: """ Make a FTP upload. :param db: database :type db: database cursor :param path: path to sample file :type path: str """ from run import log, config upload_path = str(path) path_parts = upload_path.split(os.path.sep) # We assume /home/{uid}/ as specified in the model user_id = path_parts[2] user = User.query.filter(User.id == user_id).first() if user is None: log.critical(f"Did not find a user for user_id {user_id}!") return filename, uploaded_file_extension = os.path.splitext(path) log.debug(f"Checking if {upload_path} has a forbidden extension") if remove_forbidden_file(uploaded_file_extension, upload_path, user): return uploaded_mime_type = magic.from_file(upload_path, mime=True) forbidden_mime_types = ForbiddenMimeType.query.filter( ForbiddenMimeType.mimetype == uploaded_mime_type).first() if forbidden_mime_types is not None: log.error( f"User {user.name} tried to upload a file with a forbidden MIME type ({uploaded_mime_type})!" ) os.remove(upload_path) return guessed_extension = mimetypes.guess_extension(uploaded_mime_type) log.debug( f"Mimetype {uploaded_mime_type} creates guessed extension {guessed_extension}" ) if guessed_extension is not None and remove_forbidden_file( guessed_extension, upload_path, user): return log.debug("Moving file to temporary folder and changing permissions...") filename = secure_filename(upload_path.replace(f"/home/{user.id}/", '')) intermediate_path = os.path.join(config.get('SAMPLE_REPOSITORY', ''), 'TempFiles', filename) log.debug(f"Copy {upload_path} to {intermediate_path}") shutil.copy(upload_path, intermediate_path) os.remove(upload_path) log.debug(f"Checking hash value for {intermediate_path}") file_hash = create_hash_for_sample(intermediate_path) if sample_already_uploaded(file_hash): log.debug(f"Sample already exists: {intermediate_path}") os.remove(intermediate_path) else: add_sample_to_queue(file_hash, intermediate_path, user.id, db)
def start_new_test(db, repository, delay): """ Function to start a new test based on kvm table. """ from run import log finished_tests = db.query(TestProgress.test_id).filter( TestProgress.status.in_([TestStatus.canceled, TestStatus.completed])).subquery() test = Test.query.filter(and_(Test.id.notin_(finished_tests))).order_by( Test.id.asc()).first() if test is None: return elif test.platform is TestPlatform.windows: kvm_processor_windows(db, repository, delay) elif test.platform is TestPlatform.linux: kvm_processor_linux(db, repository, delay) else: log.error("Unsupported CI platform: {platform}".format( platform=test.platform)) return
def progress_reporter(test_id, token): """ Handle the progress of a certain test after validating the token. If necessary, update the status on GitHub. :param test_id: The id of the test to update. :type test_id: int :param token: The token to check the validity of the request. :type token: str :return: Nothing. :rtype: None """ from run import config, log # Verify token test = Test.query.filter(Test.id == test_id).first() if test is not None and test.token == token: repo_folder = config.get('SAMPLE_REPOSITORY', '') if 'type' in request.form: if request.form['type'] == 'progress': # Progress, log status = TestStatus.from_string(request.form['status']) # Check whether test is not running previous status again istatus = TestStatus.progress_step(status) message = request.form['message'] if len(test.progress) != 0: laststatus = TestStatus.progress_step( test.progress[-1].status) if laststatus in [ TestStatus.completed, TestStatus.canceled ]: return "FAIL" if laststatus > istatus: status = TestStatus.canceled message = "Duplicate Entries" progress = TestProgress(test.id, status, message) g.db.add(progress) g.db.commit() gh = GitHub(access_token=g.github['bot_token']) repository = gh.repos(g.github['repository_owner'])( g.github['repository']) # Store the test commit for testing in case of commit if status == TestStatus.completed: commit_name = 'fetch_commit_' + test.platform.value commit = GeneralData.query.filter( GeneralData.key == commit_name).first() fetch_commit = Test.query.filter( and_(Test.commit == commit.value, Test.platform == test.platform)).first() if test.test_type == TestType.commit and test.id > fetch_commit.id: commit.value = test.commit g.db.commit() # If status is complete, remove the Kvm entry if status in [TestStatus.completed, TestStatus.canceled]: log.debug("Test {id} has been {status}".format( id=test_id, status=status)) var_average = 'average_time_' + test.platform.value current_average = GeneralData.query.filter( GeneralData.key == var_average).first() average_time = 0 total_time = 0 if current_average is None: platform_tests = g.db.query(Test.id).filter( Test.platform == test.platform).subquery() finished_tests = g.db.query( TestProgress.test_id).filter( and_( TestProgress.status.in_([ TestStatus.canceled, TestStatus.completed ]), TestProgress.test_id.in_( platform_tests))).subquery() in_progress_statuses = [ TestStatus.preparation, TestStatus.completed, TestStatus.canceled ] finished_tests_progress = g.db.query( TestProgress).filter( and_( TestProgress.test_id.in_(finished_tests), TestProgress.status.in_( in_progress_statuses))).subquery() times = g.db.query( finished_tests_progress.c.test_id, label( 'time', func.group_concat( finished_tests_progress.c.timestamp)) ).group_by(finished_tests_progress.c.test_id).all() for p in times: parts = p.time.split(',') start = datetime.datetime.strptime( parts[0], '%Y-%m-%d %H:%M:%S') end = datetime.datetime.strptime( parts[-1], '%Y-%m-%d %H:%M:%S') total_time += (end - start).total_seconds() if len(times) != 0: average_time = total_time // len(times) new_avg = GeneralData(var_average, average_time) g.db.add(new_avg) g.db.commit() else: all_results = TestResult.query.count() regression_test_count = RegressionTest.query.count() number_test = all_results / regression_test_count updated_average = float( current_average.value) * (number_test - 1) pr = test.progress_data() end_time = pr['end'] start_time = pr['start'] if end_time.tzinfo is not None: end_time = end_time.replace(tzinfo=None) if start_time.tzinfo is not None: start_time = start_time.replace(tzinfo=None) last_running_test = end_time - start_time updated_average = updated_average + last_running_test.total_seconds( ) current_average.value = updated_average // number_test g.db.commit() kvm = Kvm.query.filter(Kvm.test_id == test_id).first() if kvm is not None: log.debug("Removing KVM entry") g.db.delete(kvm) g.db.commit() # Post status update state = Status.PENDING target_url = url_for('test.by_id', test_id=test.id, _external=True) context = "CI - {name}".format(name=test.platform.value) if status == TestStatus.canceled: state = Status.ERROR message = 'Tests aborted due to an error; please check' elif status == TestStatus.completed: # Determine if success or failure # It fails if any of these happen: # - A crash (unexpected exit code) # - A not None value on the "got" of a TestResultFile ( # meaning the hashes do not match) crashes = g.db.query(count(TestResult.exit_code)).filter( and_(TestResult.test_id == test.id, TestResult.exit_code != TestResult.expected_rc)).scalar() results_zero_rc = g.db.query(RegressionTest.id).filter( RegressionTest.expected_rc == 0).subquery() results = g.db.query(count(TestResultFile.got)).filter( and_( TestResultFile.test_id == test.id, TestResultFile.regression_test_id.in_( results_zero_rc), TestResultFile.got.isnot(None))).scalar() log.debug( 'Test {id} completed: {crashes} crashes, {results} results' .format(id=test.id, crashes=crashes, results=results)) if crashes > 0 or results > 0: state = Status.FAILURE message = 'Not all tests completed successfully, please check' else: state = Status.SUCCESS message = 'Tests completed' update_build_badge(state, test) else: message = progress.message gh_commit = repository.statuses(test.commit) try: gh_commit.post(state=state, description=message, context=context, target_url=target_url) except ApiError as a: log.error( 'Got an exception while posting to GitHub! Message: {message}' .format(message=a.message)) if status in [TestStatus.completed, TestStatus.canceled]: # Start next test if necessary, on the same platform process = Process(target=start_platform, args=(g.db, repository, 60)) process.start() elif request.form['type'] == 'equality': log.debug('Equality for {t}/{rt}/{rto}'.format( t=test_id, rt=request.form['test_id'], rto=request.form['test_file_id'])) rto = RegressionTestOutput.query.filter( RegressionTestOutput.id == request.form['test_file_id']).first() if rto is None: # Equality posted on a file that's ignored presumably log.info('No rto for {test_id}: {test}'.format( test_id=test_id, test=request.form['test_id'])) else: result_file = TestResultFile(test.id, request.form['test_id'], rto.id, rto.correct) g.db.add(result_file) g.db.commit() elif request.form['type'] == 'logupload': log.debug("Received log file for test {id}".format(id=test_id)) # File upload, process if 'file' in request.files: uploaded_file = request.files['file'] filename = secure_filename(uploaded_file.filename) if filename is '': return 'EMPTY' temp_path = os.path.join(repo_folder, 'TempFiles', filename) # Save to temporary location uploaded_file.save(temp_path) final_path = os.path.join( repo_folder, 'LogFiles', '{id}{ext}'.format(id=test.id, ext='.txt')) os.rename(temp_path, final_path) log.debug("Stored log file") elif request.form['type'] == 'upload': log.debug('Upload for {t}/{rt}/{rto}'.format( t=test_id, rt=request.form['test_id'], rto=request.form['test_file_id'])) # File upload, process if 'file' in request.files: uploaded_file = request.files['file'] filename = secure_filename(uploaded_file.filename) if filename is '': return 'EMPTY' temp_path = os.path.join(repo_folder, 'TempFiles', filename) # Save to temporary location uploaded_file.save(temp_path) # Get hash and check if it's already been submitted hash_sha256 = hashlib.sha256() with open(temp_path, "rb") as f: for chunk in iter(lambda: f.read(4096), b""): hash_sha256.update(chunk) file_hash = hash_sha256.hexdigest() filename, file_extension = os.path.splitext(filename) final_path = os.path.join( repo_folder, 'TestResults', '{hash}{ext}'.format(hash=file_hash, ext=file_extension)) os.rename(temp_path, final_path) rto = RegressionTestOutput.query.filter( RegressionTestOutput.id == request.form['test_file_id']).first() result_file = TestResultFile(test.id, request.form['test_id'], rto.id, rto.correct, file_hash) g.db.add(result_file) g.db.commit() elif request.form['type'] == 'finish': log.debug('Finish for {t}/{rt}'.format( t=test_id, rt=request.form['test_id'])) regression_test = RegressionTest.query.filter( RegressionTest.id == request.form['test_id']).first() result = TestResult(test.id, regression_test.id, request.form['runTime'], request.form['exitCode'], regression_test.expected_rc) g.db.add(result) try: g.db.commit() except IntegrityError as e: log.error('Could not save the results: {msg}'.format( msg=e.message)) return "OK" return "FAIL"
def upload_ftp(db, path): from run import log, config temp_path = str(path) path_parts = temp_path.split(os.path.sep) # We assume /home/{uid}/ as specified in the model user_id = path_parts[2] user = User.query.filter(User.id == user_id).first() filename, file_extension = os.path.splitext(path) # FIRST, check extension. We can't limit extensions on FTP as we can on the web interface. log.debug('Checking if {path} has a forbidden extension'.format(path=temp_path)) forbidden = ForbiddenExtension.query.filter(ForbiddenExtension.extension == file_extension[1:]).first() if forbidden is not None: log.error('User {name} tried to upload a file with a forbidden extension ({extension})!'.format( name=user.name, extension=file_extension[1:] )) os.remove(temp_path) return mimetype = magic.from_file(temp_path, mime=True) # Check for permitted mimetype forbidden_mime = ForbiddenMimeType.query.filter(ForbiddenMimeType.mimetype == mimetype).first() if forbidden_mime is not None: log.error('User {name} tried to upload a file with a forbidden mimetype ({mimetype})!'.format( name=user.name, mimetype=mimetype )) os.remove(temp_path) return # Check for permitted extension extension = mimetypes.guess_extension(mimetype) if extension is not None: forbidden_real = ForbiddenExtension.query.filter(ForbiddenExtension.extension == extension[1:]).first() if forbidden_real is not None: log.error('User {name} tried to upload a file with a forbidden extension ({extension})!'.format( name=user.name, extension=extension[1:] )) os.remove(temp_path) return log.debug('Moving file to temporary folder and changing permissions...') # Move the file to a temporary location filename = secure_filename(temp_path.replace('/home/' + user_id + '/', '')) intermediate_path = os.path.join(config.get('SAMPLE_REPOSITORY', ''), 'TempFiles', filename) # Save to temporary location log.debug('Copy {old} to {new}'.format(old=temp_path, new=intermediate_path)) shutil.copy(temp_path, intermediate_path) os.remove(temp_path) log.debug('Checking hash value for {path}'.format(path=intermediate_path)) file_hash = create_hash_for_sample(intermediate_path) if sample_already_uploaded(file_hash): # Remove existing file log.debug('Sample already exists: {path}'.format( path=intermediate_path)) os.remove(intermediate_path) else: add_sample_to_queue(file_hash, intermediate_path, user.id, db)
def kvm_processor(db, kvm_name, platform, repository, delay): """ Checks whether there is no already running same kvm. Checks whether machine is in maintenance mode or not Launch kvm if not used by any other test Creates testing xml files to test the change in main repo. Creates clone with separate branch and merge pr into it. """ from run import config, log, app log.info("[{platform}] Running kvm_processor".format(platform=platform)) if kvm_name == "": log.critical('[{platform}] KVM name is empty!') return if delay is not None: import time log.debug('[{platform}] Sleeping for {time} seconds'.format( platform=platform, time=delay)) time.sleep(delay) maintenance_mode = MaintenanceMode.query.filter( MaintenanceMode.platform == platform).first() if maintenance_mode is not None and maintenance_mode.disabled: log.debug('[{platform}] In maintenance mode! Waiting...').format( platform=platform) return # Open connection to libvirt conn = libvirt.open("qemu:///system") if conn is None: log.critical( "[{platform}] Couldn't open connection to libvirt!".format( platform=platform)) return try: vm = conn.lookupByName(kvm_name) except libvirt.libvirtError: log.critical("[{platform}] No VM named {name} found!".format( platform=platform, name=kvm_name)) return vm_info = vm.info() if vm_info[0] != libvirt.VIR_DOMAIN_SHUTOFF: # Running, check expiry (2 hours runtime max) status = Kvm.query.filter(Kvm.name == kvm_name).first() max_runtime = config.get("KVM_MAX_RUNTIME", 120) if status is not None: if datetime.datetime.now( ) - status.timestamp >= datetime.timedelta(minutes=max_runtime): # Mark entry as aborted test_progress = TestProgress(status.test.id, TestStatus.canceled, 'Runtime exceeded') db.add(test_progress) db.delete(status) db.commit() # Abort process if vm.destroy() == -1: # Failed to shut down log.critical( "[{platform}] Failed to shut down {name}".format( platform=platform, name=kvm_name)) return else: log.info("[{platform}] Current job not expired yet.".format( platform=platform)) return else: log.warn( "[{platform}] No task, but VM is running! Hard reset necessary" .format(platform=platform)) if vm.destroy() == -1: # Failed to shut down log.critical("[{platform}] Failed to shut down {name}".format( platform=platform, name=kvm_name)) return # Check if there's no KVM status left status = Kvm.query.filter(Kvm.name == kvm_name).first() if status is not None: log.warn( "[{platform}] KVM is powered off, but test {id} still present". format(platform=platform, id=status.test.id)) db.delete(status) db.commit() # Get oldest test for this platform finished_tests = db.query(TestProgress.test_id).filter( TestProgress.status.in_([TestStatus.canceled, TestStatus.completed])).subquery() test = Test.query.filter( and_(Test.id.notin_(finished_tests), Test.platform == platform)).order_by(Test.id.asc()).first() if test is None: log.info('[{platform}] No more tests to run, returning'.format( platform=platform)) return if test.test_type == TestType.pull_request and test.pr_nr == 0: log.warn('[{platform}] Test {id} is invalid, deleting'.format( platform=platform, id=test.id)) db.delete(test) db.commit() return # Reset to snapshot if vm.hasCurrentSnapshot() != 1: log.critical( "[{platform}] VM {name} has no current snapshot set!".format( platform=platform, name=kvm_name)) return snapshot = vm.snapshotCurrent() if vm.revertToSnapshot(snapshot) == -1: log.critical( "[{platform}] Failed to revert to {snapshot} for {name}".format( platform=platform, snapshot=snapshot.getName(), name=kvm_name)) return log.info("[{p}] Reverted to {snap} for {name}".format( p=platform, snap=snapshot.getName(), name=kvm_name)) log.debug('Starting test {id}'.format(id=test.id)) status = Kvm(kvm_name, test.id) # Prepare data # 0) Write url to file with app.app_context(): full_url = url_for('ci.progress_reporter', test_id=test.id, token=test.token, _external=True, _scheme="https") file_path = os.path.join(config.get('SAMPLE_REPOSITORY', ''), 'vm_data', kvm_name, 'reportURL') with open(file_path, 'w') as f: f.write(full_url) # 1) Generate test files base_folder = os.path.join(config.get('SAMPLE_REPOSITORY', ''), 'vm_data', kvm_name, 'ci-tests') categories = Category.query.order_by(Category.id.desc()).all() commit_name = 'fetch_commit_' + platform.value commit_hash = GeneralData.query.filter( GeneralData.key == commit_name).first().value last_commit = Test.query.filter( and_(Test.commit == commit_hash, Test.platform == platform)).first() log.debug("[{p}] We will compare against the results of test {id}".format( p=platform, id=last_commit.id)) # Init collection file multi_test = etree.Element('multitest') for category in categories: # Skip categories without tests if len(category.regression_tests) == 0: continue # Create XML file for test file_name = '{name}.xml'.format(name=category.name) single_test = etree.Element('tests') for regression_test in category.regression_tests: entry = etree.SubElement(single_test, 'entry', id=str(regression_test.id)) command = etree.SubElement(entry, 'command') command.text = regression_test.command input_node = etree.SubElement( entry, 'input', type=regression_test.input_type.value) # Need a path that is relative to the folder we provide inside the CI environment. input_node.text = regression_test.sample.filename output_node = etree.SubElement(entry, 'output') output_node.text = regression_test.output_type.value compare = etree.SubElement(entry, 'compare') last_files = TestResultFile.query.filter( and_(TestResultFile.test_id == last_commit.id, TestResultFile.regression_test_id == regression_test.id)).subquery() for output_file in regression_test.output_files: ignore_file = str(output_file.ignore).lower() file_node = etree.SubElement(compare, 'file', ignore=ignore_file, id=str(output_file.id)) last_commit_files = db.query(last_files.c.got).filter( and_( last_files.c.regression_test_output_id == output_file.id, last_files.c.got.isnot(None))).first() correct = etree.SubElement(file_node, 'correct') # Need a path that is relative to the folder we provide inside the CI environment. if last_commit_files is None: correct.text = output_file.filename_correct else: correct.text = output_file.create_correct_filename( last_commit_files[0]) expected = etree.SubElement(file_node, 'expected') expected.text = output_file.filename_expected( regression_test.sample.sha) # Save XML single_test.getroottree().write(os.path.join(base_folder, file_name), encoding='utf-8', xml_declaration=True, pretty_print=True) # Append to collection file test_file = etree.SubElement(multi_test, 'testfile') location = etree.SubElement(test_file, 'location') location.text = file_name # Save collection file multi_test.getroottree().write(os.path.join(base_folder, 'TestAll.xml'), encoding='utf-8', xml_declaration=True, pretty_print=True) # 2) Create git repo clone and merge PR into it (if necessary) try: repo = Repo( os.path.join(config.get('SAMPLE_REPOSITORY', ''), 'vm_data', kvm_name, 'unsafe-ccextractor')) except InvalidGitRepositoryError: log.critical( "[{platform}] Could not open CCExtractor's repository copy!". format(platform=platform)) return # Return to master repo.heads.master.checkout(True) # Update repository from upstream try: origin = repo.remote('origin') except ValueError: log.critical("[{platform}] Origin remote doesn't exist!".format( platform=platform)) return fetch_info = origin.fetch() if len(fetch_info) == 0: log.info( '[{platform}] Fetch from remote returned no new data...'.format( platform=platform)) # Pull code (finally) pull_info = origin.pull() if len(pull_info) == 0: log.info( "[{platform}] Pull from remote returned no new data...".format( platform=platform)) if pull_info[0].flags > 128: log.critical( "[{platform}] Didn't pull any information from remote: {flags}!". format(platform=platform, flags=pull_info[0].flags)) return # Delete the test branch if it exists, and recreate try: repo.delete_head('CI_Branch', force=True) except GitCommandError: log.warn("[{platform}] Could not delete CI_Branch head".format( platform=platform)) # Remove possible left rebase-apply directory try: shutil.rmtree( os.path.join(config.get('SAMPLE_REPOSITORY', ''), 'unsafe-ccextractor', '.git', 'rebase-apply')) except OSError: log.warn("[{platform}] Could not delete rebase-apply".format( platform=platform)) # If PR, merge, otherwise reset to commit if test.test_type == TestType.pull_request: # Fetch PR (stored under origin/pull/<id>/head pull_info = origin.fetch( 'pull/{id}/head:CI_Branch'.format(id=test.pr_nr)) if len(pull_info) == 0: log.warn( "[{platform}] Didn't pull any information from remote PR!". format(platform=platform)) if pull_info[0].flags > 128: log.critical( "[{platform}] Didn't pull any information from remote PR: {flags}!" .format(platform=platform, flags=pull_info[0].flags)) return try: test_branch = repo.heads['CI_Branch'] except IndexError: log.critical('CI_Branch does not exist') return test_branch.checkout(True) try: pull = repository.pulls('{pr_nr}'.format(pr_nr=test.pr_nr)).get() except ApiError as a: log.error( 'Got an exception while fetching the PR payload! Message: {message}' .format(message=a.message)) return if pull['mergeable'] is False: progress = TestProgress(test.id, TestStatus.canceled, "Commit could not be merged", datetime.datetime.now()) db.add(progress) db.commit() try: with app.app_context(): repository.statuses(test.commit).post( state=Status.FAILURE, description="Tests canceled due to merge conflict", context="CI - {name}".format(name=test.platform.value), target_url=url_for('test.by_id', test_id=test.id, _external=True)) except ApiError as a: log.error( 'Got an exception while posting to GitHub! Message: {message}' .format(message=a.message)) return # Merge on master if no conflict repo.git.merge('master') else: test_branch = repo.create_head('CI_Branch', 'HEAD') test_branch.checkout(True) try: repo.head.reset(test.commit, working_tree=True) except GitCommandError: log.warn( "[{platform}] Commit {hash} for test {id} does not exist!". format(platform=platform, hash=test.commit, id=test.id)) return # Power on machine try: vm.create() db.add(status) db.commit() except libvirt.libvirtError: log.critical("[{platform}] Failed to launch VM {name}".format( platform=platform, name=kvm_name)) except IntegrityError: log.warn("[{platform}] Duplicate entry for {id}".format( platform=platform, id=test.id))