def __download_accidents_type_files(self, accidents_type, start_year):
current_year, s3_bucket, local_directory = (
self.current_year,
self.s3_bucket,
self.local_files_directory,
)
download_from_s3_callback = self.download_from_s3_callback
accidents_type_directory = f"{ACCIDENTS_TYPE_PREFIX}_{accidents_type}"
for year in range(start_year, current_year + 1):
s3_files_directory = f"{accidents_type_directory}/{year}"
for s3_object in s3_bucket.objects.filter(
Prefix=s3_files_directory):
local_dir_path = join_path(local_directory,
accidents_type_directory)
if not does_path_exist(local_dir_path):
mkdir(local_dir_path)
local_dir_path = join_path(local_dir_path, str(year))
if not does_path_exist(local_dir_path):
mkdir(local_dir_path)
object_key = s3_object.key
s3_filename = basename(object_key)
local_file_path = join_path(local_dir_path, s3_filename)
download_from_s3_callback(Bucket=ANYWAY_BUCKET,
Key=object_key,
Filename=local_file_path)
def create_entry(name, path=None):
if not path:
path = get_default_venvs_path()
elif not does_path_exist(path):
raise VenvDirBaseError("Base path '{}' does not exist.".format(path))
env_path = os.path.join(path, name)
if does_path_exist(env_path):
raise VenvDirBaseError(
"Virtual environment '{}' already exists.".format(env_path)
)
create_venv(env_path, with_pip=True)
config_parser.create_entry(name, path)
def local_files_directory(self):
if self.__local_files_directory is None:
temp_directory = self.local_temp_directory
files_directory = join_path(temp_directory, LOCAL_CBS_DIRECTORY)
if not does_path_exist(files_directory):
makedirs(files_directory)
self.__local_files_directory = files_directory
return self.__local_files_directory
def add_entry(name, path):
if not does_path_exist(path):
raise VenvDirBaseError("Venv path '{}' does not exist.".format(path))
config_parser.create_entry(name, path)