def store_victims_db(self, victims_db_dir): """ Zip victims_db_dir/* and store to S3 as VICTIMS_DB_ARCHIVE""" with tempdir() as temp_archive_dir: temp_archive_path = os.path.join(temp_archive_dir, self.VICTIMS_DB_ARCHIVE) with cwd(victims_db_dir): Archive.zip_file('.', temp_archive_path) self.store_file(temp_archive_path, self.VICTIMS_DB_ARCHIVE)
def retrieve_victims_db_if_exists(self, victims_db_dir): """ Retrieve VICTIMS_DB_ARCHIVE from S3 and extract into victims_db_dir """ if self.object_exists(self.VICTIMS_DB_ARCHIVE): with tempdir() as temp_archive_dir: temp_archive_path = os.path.join(temp_archive_dir, self.VICTIMS_DB_ARCHIVE) self.retrieve_file(self.VICTIMS_DB_ARCHIVE, temp_archive_path) Archive.extract_zip(temp_archive_path, victims_db_dir) return True return False
def retrieve_depcheck_db_if_exists(self, data_dir): """ Retrieve zipped CVE DB file as stored on S3 and extract""" if self.object_exists(self.DEPCHECK_DB_ARCHIVE): with tempdir() as archive_dir: archive_path = os.path.join(archive_dir, self.DEPCHECK_DB_ARCHIVE) self.retrieve_file(self.DEPCHECK_DB_ARCHIVE, archive_path) Archive.extract_zip(archive_path, data_dir) return True return False
def test_for_archive_create_by_root(self): """Test extracting archives created by root.""" response = requests.get( "https://registry.npmjs.org/ajax-worker/-/ajax-worker-1.2.3.tgz") with tempfile.NamedTemporaryFile(suffix=".tgz") as package, tempfile.TemporaryDirectory() \ as extracted: package.write(response.content) Archive.extract(package.name, extracted) with pytest.raises(PermissionError): shutil.rmtree(extracted) Archive.fix_permissions(extracted + "/package")
def store_depcheck_db(self, data_dir): """ Zip CVE DB file and store to S3 """ with tempdir() as archive_dir: archive_path = os.path.join(archive_dir, self.DEPCHECK_DB_ARCHIVE) db_file_path = os.path.join(data_dir, self.DEPCHECK_DB_FILENAME) try: Archive.zip_file(db_file_path, archive_path, junk_paths=True) except TaskError: pass else: self.store_file(archive_path, self.DEPCHECK_DB_ARCHIVE)
def retrieve_index_if_exists(self, target_dir): """ Retrieve central-index.zip from S3 and extract into target_dir/central-index""" if self.object_exists(self._INDEX_ARCHIVE): with tempdir() as temp_dir: archive_path = os.path.join(temp_dir, self._INDEX_ARCHIVE) central_index_dir = os.path.join(target_dir, self._INDEX_DIRNAME) self.retrieve_file(self._INDEX_ARCHIVE, archive_path) Archive.extract_zip(archive_path, central_index_dir, mkdest=True) return True return False
def store_index(self, target_dir): """ Zip files in target_dir/central-index dir and store to S3 """ with tempdir() as temp_dir: central_index_dir = os.path.join(target_dir, self._INDEX_DIRNAME) archive_path = os.path.join(temp_dir, self._INDEX_ARCHIVE) try: Archive.zip_file(central_index_dir, archive_path, junk_paths=True) except TaskError: pass else: self.store_file(archive_path, self._INDEX_ARCHIVE)
def test_empty_archive(self, archive_name): """Test extracting an empty archive. Nothing should fail and the destination directory should exist afterwards. """ archive_path = Path(__file__).resolve().parent / Path( 'data/archives/' + archive_name) with tempfile.TemporaryDirectory() as temp_dir: dest_dir = Path(temp_dir) / Path('extracted_jar') assert not dest_dir.exists() Archive.extract(str(archive_path), str(dest_dir)) assert dest_dir.exists()
def test_for_archive_create_by_root(self): """Test extracting archives created by root.""" response = requests.get( "https://registry.npmjs.org/ajax-worker/-/ajax-worker-1.2.3.tgz") with tempfile.NamedTemporaryFile(suffix=".tgz") as package, tempfile.TemporaryDirectory() \ as extracted: dest_dir = Path(extracted) / Path('dest_dir') package.write(response.content) Archive.extract(package.name, str(dest_dir)) assert Path(str(dest_dir)).exists() shutil.rmtree(str(dest_dir)) assert not Path(str(dest_dir)).exists()
def get_extracted_source_jar(self): """Get extracted package source jar file. :return: path to extracted source jar file """ if not os.path.isdir(self._extracted_source_jar_dir): source_jar_path = self.get_source_jar() try: Archive.extract(source_jar_path, self._extracted_source_jar_dir) except Exception: # remove in case of failure so if one catches the exception, # the extraction code is correctly called again shutil.rmtree(self._extracted_source_jar_dir, ignore_errors=True) raise return self._extracted_source_jar_dir
def get_extracted_source_tarball(self): """Get path to the extracted package tarball. :return: path to the extracted package tarball """ self._construct_source_tarball_names() if not os.path.isdir(self._extracted_tarball_dir): source_tarball_path = self.get_source_tarball() os.makedirs(self._extracted_tarball_dir) try: Archive.extract(source_tarball_path, self._extracted_tarball_dir) except Exception: # remove in case of failure so if one catches the exception, # the extraction code is correctly called again shutil.rmtree(self._extracted_tarball_dir, ignore_errors=True) raise return self._extracted_tarball_dir
def test_bad_permissions(self, archive_name): """Test working with an archive which contains files with bad permissions. Bad permissions = 000. All extracted files need to be readable so they can be processed by various tools later. """ archive_path = Path(__file__).resolve().parent / Path( 'data/archives/' + archive_name) with tempfile.TemporaryDirectory() as temp_dir: dest_dir = Path(temp_dir) / Path('dest_dir') Archive.extract(str(archive_path), str(dest_dir)) assert dest_dir.exists() file_path = dest_dir / Path('cant_touch_me') # is the file readable? assert file_path.stat().st_mode & stat.S_IRUSR shutil.rmtree(str(dest_dir), ignore_errors=True) assert not dest_dir.exists()