def main(file_path,
upload_path,
overwrite=False,
storage_account_name=None,
settings_path=None):
try:
azure_data_lake = AzureStorageAccount()
upload_results = azure_data_lake.upload_file(file_path, upload_path,
overwrite,
storage_account_name,
settings_path)
results_to_return = {
'date': upload_results.get('date'),
'error_code': upload_results.get('error_code'),
'upload_path': upload_path
}
click.echo(
json.dumps(results_to_return,
indent=4,
sort_keys=True,
default=str)
) # date formats get messed up when converting, this converts the dates to strings
return True
except Exception as e:
click.echo({'error_code': e})
return False
def main(file_path, storage_account_name=None, settings_path=None):
try:
azure_data_lake = AzureStorageAccount()
results = azure_data_lake.delete_file(file_path, storage_account_name,
settings_path)
click.echo(f'{results}')
return True
except Exception as e:
click.echo({'error_code': e})
return False
def main(directory='', storage_account_name=None, settings_path=None):
try:
azure_data_lake = AzureStorageAccount()
file_iterator = azure_data_lake.list(directory, storage_account_name,
settings_path)
listed_files = [{
'size': x['size'],
'name': x['name']
} for x in file_iterator]
listed_files = json.dumps({'error_code': None, 'files': listed_files})
click.echo(listed_files)
return listed_files
except Exception as e:
click.echo({'error_code': e})
raise Exception(e)
def main(adls_path, save_path, storage_account_name=None, settings_path=None):
try:
azure_data_lake = AzureStorageAccount()
save_path = azure_data_lake.download(
adls_path,
save_path,
storage_account_name=storage_account_name,
settings_path=settings_path)
response = {'save_path': save_path, 'error_code': None}
click.echo(json.dumps(response))
return True
except Exception as e:
click.echo({'error_code': e})
return False
def test_get_container_name_from_string_colon(self):
# setup
upload_path = 'testcontainername:test.txt'
# exercise
container_name = AzureStorageAccount._get_container_name(upload_path)
# validate
expected_container_name_container = 'testcontainername'
self.assertEqual(expected_container_name_container, container_name)
def test_get_default_container_name(self):
# setup
upload_path = 'test.txt'
# exercise
container_name = AzureStorageAccount._get_container_name(upload_path)
# validate
expected_container_name_container = 'default'
self.assertEqual(expected_container_name_container, container_name)
def test_get_container_name_long(self):
# setup
upload_path = 'test.txt'
really_long_container_name = 'x' * 64
os.environ['DataLakeContainerName'] = really_long_container_name
# exercise
container_name = AzureStorageAccount._get_container_name(upload_path)
# validate
self.assertEqual(really_long_container_name, container_name)
def test_get_container_name_from_string_override(self):
# setup
upload_path = 'testcontainername:test.txt'
os.environ['DataLakeContainerName'] = 'envcontainername'
# exercise
container_name = AzureStorageAccount._get_container_name(upload_path)
# validate
expected_container_name_container = 'testcontainername'
self.assertEqual(expected_container_name_container, container_name)
def test_get_container_name_invalid_name_special_chars(self):
# setup
upload_path = 'test.txt'
capitalized_container_name = 'container_name'
os.environ['DataLakeContainerName'] = capitalized_container_name
# exercise
container_name = AzureStorageAccount._get_container_name(upload_path)
# validate
expected_container_name = 'default'
self.assertEqual(expected_container_name, container_name)