def store_victims_db(self, victims_db_dir):
""" Zip victims_db_dir/* and store to S3 as VICTIMS_DB_ARCHIVE"""
with tempdir() as temp_archive_dir:
temp_archive_path = os.path.join(temp_archive_dir,
self.VICTIMS_DB_ARCHIVE)
with cwd(victims_db_dir):
Archive.zip_file('.', temp_archive_path)
self.store_file(temp_archive_path, self.VICTIMS_DB_ARCHIVE)
def retrieve_victims_db_if_exists(self, victims_db_dir):
""" Retrieve VICTIMS_DB_ARCHIVE from S3 and extract into victims_db_dir """
if self.object_exists(self.VICTIMS_DB_ARCHIVE):
with tempdir() as temp_archive_dir:
temp_archive_path = os.path.join(temp_archive_dir,
self.VICTIMS_DB_ARCHIVE)
self.retrieve_file(self.VICTIMS_DB_ARCHIVE, temp_archive_path)
Archive.extract_zip(temp_archive_path, victims_db_dir)
return True
return False
def retrieve_depcheck_db_if_exists(self, data_dir):
""" Retrieve zipped CVE DB file as stored on S3 and extract"""
if self.object_exists(self.DEPCHECK_DB_ARCHIVE):
with tempdir() as archive_dir:
archive_path = os.path.join(archive_dir,
self.DEPCHECK_DB_ARCHIVE)
self.retrieve_file(self.DEPCHECK_DB_ARCHIVE, archive_path)
Archive.extract_zip(archive_path, data_dir)
return True
return False
def test_for_archive_create_by_root(self):
"""Test extracting archives created by root."""
response = requests.get(
"https://registry.npmjs.org/ajax-worker/-/ajax-worker-1.2.3.tgz")
with tempfile.NamedTemporaryFile(suffix=".tgz") as package, tempfile.TemporaryDirectory() \
as extracted:
package.write(response.content)
Archive.extract(package.name, extracted)
with pytest.raises(PermissionError):
shutil.rmtree(extracted)
Archive.fix_permissions(extracted + "/package")
def store_depcheck_db(self, data_dir):
""" Zip CVE DB file and store to S3 """
with tempdir() as archive_dir:
archive_path = os.path.join(archive_dir, self.DEPCHECK_DB_ARCHIVE)
db_file_path = os.path.join(data_dir, self.DEPCHECK_DB_FILENAME)
try:
Archive.zip_file(db_file_path, archive_path, junk_paths=True)
except TaskError:
pass
else:
self.store_file(archive_path, self.DEPCHECK_DB_ARCHIVE)
def retrieve_index_if_exists(self, target_dir):
""" Retrieve central-index.zip from S3 and extract into target_dir/central-index"""
if self.object_exists(self._INDEX_ARCHIVE):
with tempdir() as temp_dir:
archive_path = os.path.join(temp_dir, self._INDEX_ARCHIVE)
central_index_dir = os.path.join(target_dir, self._INDEX_DIRNAME)
self.retrieve_file(self._INDEX_ARCHIVE, archive_path)
Archive.extract_zip(archive_path, central_index_dir, mkdest=True)
return True
return False
def store_index(self, target_dir):
""" Zip files in target_dir/central-index dir and store to S3 """
with tempdir() as temp_dir:
central_index_dir = os.path.join(target_dir, self._INDEX_DIRNAME)
archive_path = os.path.join(temp_dir, self._INDEX_ARCHIVE)
try:
Archive.zip_file(central_index_dir, archive_path, junk_paths=True)
except TaskError:
pass
else:
self.store_file(archive_path, self._INDEX_ARCHIVE)
def test_empty_archive(self, archive_name):
"""Test extracting an empty archive.
Nothing should fail and the destination directory should exist afterwards.
"""
archive_path = Path(__file__).resolve().parent / Path(
'data/archives/' + archive_name)
with tempfile.TemporaryDirectory() as temp_dir:
dest_dir = Path(temp_dir) / Path('extracted_jar')
assert not dest_dir.exists()
Archive.extract(str(archive_path), str(dest_dir))
assert dest_dir.exists()
def test_for_archive_create_by_root(self):
"""Test extracting archives created by root."""
response = requests.get(
"https://registry.npmjs.org/ajax-worker/-/ajax-worker-1.2.3.tgz")
with tempfile.NamedTemporaryFile(suffix=".tgz") as package, tempfile.TemporaryDirectory() \
as extracted:
dest_dir = Path(extracted) / Path('dest_dir')
package.write(response.content)
Archive.extract(package.name, str(dest_dir))
assert Path(str(dest_dir)).exists()
shutil.rmtree(str(dest_dir))
assert not Path(str(dest_dir)).exists()
def get_extracted_source_jar(self):
"""Get extracted package source jar file.
:return: path to extracted source jar file
"""
if not os.path.isdir(self._extracted_source_jar_dir):
source_jar_path = self.get_source_jar()
try:
Archive.extract(source_jar_path, self._extracted_source_jar_dir)
except Exception:
# remove in case of failure so if one catches the exception,
# the extraction code is correctly called again
shutil.rmtree(self._extracted_source_jar_dir, ignore_errors=True)
raise
return self._extracted_source_jar_dir
def get_extracted_source_tarball(self):
"""Get path to the extracted package tarball.
:return: path to the extracted package tarball
"""
self._construct_source_tarball_names()
if not os.path.isdir(self._extracted_tarball_dir):
source_tarball_path = self.get_source_tarball()
os.makedirs(self._extracted_tarball_dir)
try:
Archive.extract(source_tarball_path, self._extracted_tarball_dir)
except Exception:
# remove in case of failure so if one catches the exception,
# the extraction code is correctly called again
shutil.rmtree(self._extracted_tarball_dir, ignore_errors=True)
raise
return self._extracted_tarball_dir
def test_bad_permissions(self, archive_name):
"""Test working with an archive which contains files with bad permissions.
Bad permissions = 000.
All extracted files need to be readable so they can be processed by various tools later.
"""
archive_path = Path(__file__).resolve().parent / Path(
'data/archives/' + archive_name)
with tempfile.TemporaryDirectory() as temp_dir:
dest_dir = Path(temp_dir) / Path('dest_dir')
Archive.extract(str(archive_path), str(dest_dir))
assert dest_dir.exists()
file_path = dest_dir / Path('cant_touch_me')
# is the file readable?
assert file_path.stat().st_mode & stat.S_IRUSR
shutil.rmtree(str(dest_dir), ignore_errors=True)
assert not dest_dir.exists()
