def validate_args(arguments: dict) -> dict:
"""
Check if the manifest file exists, and
if a target directory is specified check if it is valid, or
if no target directory is specified, use the parent dir
of the manifest file
:param arguments: The arguments
:return: The updated and checked arguments
"""
manifest_file = get_full_path(arguments["manifest"])
if not is_valid_path(str(manifest_file)):
sys.exit(
"The path specified for the manifest file is invalid\n"
"Please verify that it exists and/or that you have the right permissions"
)
else:
print("The path specified for the manifest file is valid")
if arguments["directory"] is not None:
target_dir = get_full_path(arguments["directory"])
else:
print("Download directory not specified, using manifest parent directory")
target_dir = manifest_file.parent
arguments["manifest"] = manifest_file
arguments["directory"] = target_dir
arguments["mods_folder"] = target_dir.joinpath("mods")
if not is_valid_path(str(arguments["mods_folder"])):
print("Creating folder to store mods in")
arguments["mods_folder"].mkdir(parents=True)
return arguments
def store_in_local_data_directory(remote_URL, dataset_name):
global local_data_directory_path
local_data_path = os.path.join(local_data_directory_path, dataset_name)
if not is_valid_path(local_data_directory_path):
os.mkdir(local_data_directory_path)
try:
print(f'\nLoading {dataset_name} from a Remote URL...')
resp = requests.get(remote_URL)
csv_file = open(local_data_path, 'wb')
print(
f'\n\nMaking a local copy of the {dataset_name} for future use...')
csv_file.write(resp.content)
print(f'\n\nLocal copy of {dataset_name} saved successfully!!')
except:
print(
f'\n\nSome error occured while loading {dataset_name} from Remote URL...status code {resp.status_code}!!'
)
def isin_local_data_directory(file_name):
global local_data_directory_path
print('\nSearching data locally...')
complete_file_path = os.path.join(local_data_directory_path, file_name)
if is_valid_path(local_data_directory_path):
if is_valid_path(complete_file_path):
print('FOUND!!')
return True
print('NOT FOUND!!')
return False
def get_dataset_from_user():
global dataset_path
global user_dataset_class_attribute
while True:
dataset_path = input('\nPlease provide the path of the dataset: ')
if is_valid_path(dataset_path):
user_dataset_class_attribute = input(
'\nPlease provide the class label attribute name: ')
return extract_filename(dataset_path)
print("\nPath doesn't exist! Please provide a valid path!")
def download(self) -> None:
"""
The main download action
:return: Nothing
"""
# self.url_full = "https://httpbin.org/delay/5"
# self.url_full = "https://httpbin.org/status/404"
# self.url_full = "https://httpbin.org/status/503"
# self.url_full = "https://httpbin.org/get"
self.url_full = "https://i.ytimg.com/vi/0KEv38tAWm4/maxresdefault.jpg"
# TODO: Figure out how to handle already existing jar file
if not is_valid_path(self.path_full, strict=True):
print("The file already exists at the specified location, removing")
pathlib.Path(self.path_full).unlink()
response = ""
status = 0
attempt = 1
retry = [500, 503, 504]
print(f"Downlading {self.jar} to: {self.path}")
print(self.url_full)
while status != requests.codes.ok:
response = self.handle_request()
status = response.status_code
if status not in retry:
break
attempt += 1
if attempt > 3:
print("All download attempts have failed, aborting")
break
else:
print(f"Retrying...")
time.sleep(2.5)
if status == requests.codes.ok:
self.write_to_disk(response.raw)
print("File succesfully downloaded")
response.close()
def prepare(self):
upload_file = self.request.files['file'][0]
self.file_content = upload_file['body']
self.file_content_type = upload_file['content_type']
self.filename = upload_file['filename']
file_path = self.request.body_arguments.get('path', ['/upload'])[0]
if is_valid_path(file_path):
self.file_path = file_path
else:
self.finish({'Error': 'file path not valid, path: %s' % file_path})
return
self.file_ext = splitext(self.filename)[1]
content_bytes = BytesIO(self.file_content)
f_md5 = hashlib.md5(self.file_content).hexdigest()
self.upload_path = join(self.file_path,
'%s%s' % (f_md5, self.file_ext))
self.upload_file = File(content_bytes,
name=join(self.file_path,
'%s%s' % (f_md5, self.file_ext)))
def download_prepare(self):
global dataset_directory
make_data_home(dataset_directory)
self._file_path = make_file_path(dataset_directory, self.name)
if not is_valid_path(self._file_path):
try:
data_response = requests.get(self._remote_URL)
data_response.raise_for_status()
csvfile = open(self._file_path, 'wb')
csvfile.write(data_response.content)
except:
print(
f'\nError occurred!!Status code: {data_response.status_code}'
)
def test_verify_if_a_path_is_not_valid(self):
path = "testing?"
self.assertFalse(utils.is_valid_path(path))
def test_verify_if_a_path_is_valid(self):
path = "/home/carledriss/unit_test"
self.assertTrue(utils.is_valid_path(path))
def test_verify_if_a_path_is_not_valid(self):
path = "testing?"
self.assertFalse(utils.is_valid_path(path))
def test_verify_if_a_path_is_valid(self):
path = "/home/carledriss/unit_test"
self.assertTrue(utils.is_valid_path(path))
