def getNumpy(self):
exists, error = self.existsWithError()
if not exists:
raise DataApiError('unable to get file {} - {}'.format(
self.path, error))
np_loader = pkgutil.find_loader('numpy')
if np_loader is not None:
import numpy as np
payload = self.client.getHelper(self.url).json()
return np.array(payload)
else:
raise DataApiError(
"Attempted to .getNumpy() file without numpy available, please install numpy."
)
def getString(self):
exists, error = self.existsWithError()
if not exists:
raise DataApiError('unable to get file {} - {}'.format(
self.path, error))
with open(self.path, 'r') as f:
return f.read()
def getJson(self):
exists, error = self.existsWithError()
if not exists:
raise DataApiError('unable to get file {} - {}'.format(
self.path, error))
# Make HTTP get request
return self.client.getHelper(self.url).json()
def _get_directory_iterator(self, type_filter=None):
marker = None
first = True
while first or (marker is not None and len(marker) > 0):
first = False
url = self.url
query_params = {}
if marker:
query_params['marker'] = marker
response = self.client.getHelper(url, **query_params)
if response.status_code != 200:
raise DataApiError("Directory iteration failed: " +
str(response.content))
responseContent = response.content
if isinstance(responseContent, six.binary_type):
responseContent = responseContent.decode()
content = json.loads(responseContent)
if 'marker' in content:
marker = content['marker']
else:
marker = None
if type_filter is DataObjectType.directory or type_filter is None:
for d in self._iterate_directories(content):
yield d
if type_filter is DataObjectType.file or type_filter is None:
for f in self._iterate_files(content):
yield f
def update_permissions(self, acl):
params = {'acl': acl.to_api_param()}
response = self.client.patchHelper(self.url, params)
if response.status_code != 200:
raise DataApiError('Unable to update permissions: ' +
response.json()['error']['message'])
return True
def delete(self):
# Delete from data api
result = self.client.deleteHelper(self.url)
if 'error' in result:
raise DataApiError(result['error']['message'])
else:
return True
def putFile(self, path):
# Post file to data api
with open(path, 'rb') as f:
result = self.client.putHelper(self.url, f)
if 'error' in result:
raise DataApiError(result['error']['message'])
else:
return self
def putJson(self, data):
# Post to data api
jsonElement = json.dumps(data)
result = self.client.putHelper(self.url, jsonElement)
if 'error' in result:
raise DataApiError(result['error']['message'])
else:
return self
def getBytes(self):
exists, error = self.existsWithError()
if not exists:
raise DataApiError('unable to get file {} - {}'.format(
self.path, error))
f = open(self.path, 'rb')
bts = f.read()
f.close()
return bts
def delete(self, force=False):
# Delete from data api
url = self.url
if force:
url += '?force=true'
result = self.client.deleteHelper(url)
if 'error' in result:
raise DataApiError(result['error']['message'])
else:
return True
def create(self, acl=None):
'''Creates a directory, optionally include Acl argument to set permissions'''
parent, name = getParentAndBase(self.path)
json = {'name': name}
if acl is not None:
json['acl'] = acl.to_api_param()
response = self.client.postJsonHelper(DataDirectory._getUrl(parent),
json, False)
if (response.status_code != 200):
raise DataApiError("Directory creation failed: " +
str(response.content))
def putNumpy(self, array):
# Post numpy array as json payload
np_loader = pkgutil.find_loader('numpy')
if np_loader is not None:
import numpy as np
encoded_array = array.tolist()
self.putJson(encoded_array)
return self
else:
raise DataApiError(
"Attempted to .putNumpy() a file without numpy available, please install numpy."
)
def ls(self, path, client, longlist=False):
# by default list user's hosted data
listing = ""
if path is None:
path = "data://"
file = path.split('/')
if file[-1] != '':
# path is a file, list parent
directory = path[:-len(file[-1])]
f = client.dir(directory)
response = client.getHelper(f.url, **{})
if response.status_code != 200:
raise DataApiError("failed to get file info: " + str(response.content))
responseContent = response.content
if isinstance(responseContent, six.binary_type):
responseContent = responseContent.decode()
content = json.loads(responseContent)
if 'files' in content:
f = client.file(path)
for file_info in content['files']:
if file_info['filename'] == file[-1]:
f.set_attributes(file_info)
if longlist:
listing += f.last_modified.strftime("%Y-%m-%d %H:%M:%S") + ' '
listing += str(f.size) + ' '
listing += f.path + "\n"
else:
listing += f.path + "\n"
else:
# path is a directory
if longlist:
listingDir = client.dir(path)
for f in listingDir.dirs():
listing += f.path + "/\n"
for f in listingDir.files():
listing += f.last_modified.strftime("%Y-%m-%d %H:%M:%S") + ' '
listing += str(f.size) + ' '
listing += f.path + "\n"
else:
listingDir = client.dir(path)
for f in listingDir.dirs():
listing += f.path + "/\n"
for f in listingDir.files():
listing += f.path + "\n"
return listing
def get_permissions(self):
'''
Returns permissions for this directory or None if it's a special collection such as
.session or .algo
'''
response = self.client.getHelper(self.url, acl='true')
if response.status_code != 200:
raise DataApiError('Unable to get permissions:' +
str(response.content))
content = response.json()
if 'acl' in content:
return Acl.from_acl_response(content['acl'])
else:
return None
def getFile(self):
exists, error = self.existsWithError()
if not exists:
raise DataApiError('unable to get file {} - {}'.format(
self.path, error))
# Make HTTP get request
response = self.client.getHelper(self.url)
with tempfile.NamedTemporaryFile(delete=False) as f:
for block in response.iter_content(1024):
if not block:
break
f.write(block)
f.flush()
return open(f.name)
def put(self, data):
# Post to data api
# First turn the data to bytes if we can
if isinstance(data, six.string_types) and not isinstance(
data, six.binary_type):
data = bytes(data.encode())
if isinstance(data, six.binary_type):
result = self.client.putHelper(self.url, data)
if 'error' in result:
raise DataApiError(result['error']['message'])
else:
return self
else:
raise TypeError(
"Must put strings or binary data. Use putJson instead")
def delete(self):
try:
os.remove(self.path)
return True
except:
raise DataApiError('Failed to delete local file ' + self.path)
def putFile(self, path):
result = localPutHelper(path, self.path)
if 'error' in result: raise DataApiError(result['error']['message'])
else: return self
def getJson(self):
exists, error = self.existsWithError()
if not exists:
raise DataApiError('unable to get file {} - {}'.format(
self.path, error))
return json.loads(open(self.path, 'r').read())
def getFile(self):
exists, error = self.existsWithError()
if not exists:
raise DataApiError('unable to get file {} - {}'.format(
self.path, error))
return open(self.path)