def getNumpy(self): exists, error = self.existsWithError() if not exists: raise DataApiError('unable to get file {} - {}'.format( self.path, error)) np_loader = pkgutil.find_loader('numpy') if np_loader is not None: import numpy as np payload = self.client.getHelper(self.url).json() return np.array(payload) else: raise DataApiError( "Attempted to .getNumpy() file without numpy available, please install numpy." )
def getString(self): exists, error = self.existsWithError() if not exists: raise DataApiError('unable to get file {} - {}'.format( self.path, error)) with open(self.path, 'r') as f: return f.read()
def getJson(self): exists, error = self.existsWithError() if not exists: raise DataApiError('unable to get file {} - {}'.format( self.path, error)) # Make HTTP get request return self.client.getHelper(self.url).json()
def _get_directory_iterator(self, type_filter=None): marker = None first = True while first or (marker is not None and len(marker) > 0): first = False url = self.url query_params = {} if marker: query_params['marker'] = marker response = self.client.getHelper(url, **query_params) if response.status_code != 200: raise DataApiError("Directory iteration failed: " + str(response.content)) responseContent = response.content if isinstance(responseContent, six.binary_type): responseContent = responseContent.decode() content = json.loads(responseContent) if 'marker' in content: marker = content['marker'] else: marker = None if type_filter is DataObjectType.directory or type_filter is None: for d in self._iterate_directories(content): yield d if type_filter is DataObjectType.file or type_filter is None: for f in self._iterate_files(content): yield f
def update_permissions(self, acl): params = {'acl': acl.to_api_param()} response = self.client.patchHelper(self.url, params) if response.status_code != 200: raise DataApiError('Unable to update permissions: ' + response.json()['error']['message']) return True
def delete(self): # Delete from data api result = self.client.deleteHelper(self.url) if 'error' in result: raise DataApiError(result['error']['message']) else: return True
def putFile(self, path): # Post file to data api with open(path, 'rb') as f: result = self.client.putHelper(self.url, f) if 'error' in result: raise DataApiError(result['error']['message']) else: return self
def putJson(self, data): # Post to data api jsonElement = json.dumps(data) result = self.client.putHelper(self.url, jsonElement) if 'error' in result: raise DataApiError(result['error']['message']) else: return self
def getBytes(self): exists, error = self.existsWithError() if not exists: raise DataApiError('unable to get file {} - {}'.format( self.path, error)) f = open(self.path, 'rb') bts = f.read() f.close() return bts
def delete(self, force=False): # Delete from data api url = self.url if force: url += '?force=true' result = self.client.deleteHelper(url) if 'error' in result: raise DataApiError(result['error']['message']) else: return True
def create(self, acl=None): '''Creates a directory, optionally include Acl argument to set permissions''' parent, name = getParentAndBase(self.path) json = {'name': name} if acl is not None: json['acl'] = acl.to_api_param() response = self.client.postJsonHelper(DataDirectory._getUrl(parent), json, False) if (response.status_code != 200): raise DataApiError("Directory creation failed: " + str(response.content))
def putNumpy(self, array): # Post numpy array as json payload np_loader = pkgutil.find_loader('numpy') if np_loader is not None: import numpy as np encoded_array = array.tolist() self.putJson(encoded_array) return self else: raise DataApiError( "Attempted to .putNumpy() a file without numpy available, please install numpy." )
def ls(self, path, client, longlist=False): # by default list user's hosted data listing = "" if path is None: path = "data://" file = path.split('/') if file[-1] != '': # path is a file, list parent directory = path[:-len(file[-1])] f = client.dir(directory) response = client.getHelper(f.url, **{}) if response.status_code != 200: raise DataApiError("failed to get file info: " + str(response.content)) responseContent = response.content if isinstance(responseContent, six.binary_type): responseContent = responseContent.decode() content = json.loads(responseContent) if 'files' in content: f = client.file(path) for file_info in content['files']: if file_info['filename'] == file[-1]: f.set_attributes(file_info) if longlist: listing += f.last_modified.strftime("%Y-%m-%d %H:%M:%S") + ' ' listing += str(f.size) + ' ' listing += f.path + "\n" else: listing += f.path + "\n" else: # path is a directory if longlist: listingDir = client.dir(path) for f in listingDir.dirs(): listing += f.path + "/\n" for f in listingDir.files(): listing += f.last_modified.strftime("%Y-%m-%d %H:%M:%S") + ' ' listing += str(f.size) + ' ' listing += f.path + "\n" else: listingDir = client.dir(path) for f in listingDir.dirs(): listing += f.path + "/\n" for f in listingDir.files(): listing += f.path + "\n" return listing
def get_permissions(self): ''' Returns permissions for this directory or None if it's a special collection such as .session or .algo ''' response = self.client.getHelper(self.url, acl='true') if response.status_code != 200: raise DataApiError('Unable to get permissions:' + str(response.content)) content = response.json() if 'acl' in content: return Acl.from_acl_response(content['acl']) else: return None
def getFile(self): exists, error = self.existsWithError() if not exists: raise DataApiError('unable to get file {} - {}'.format( self.path, error)) # Make HTTP get request response = self.client.getHelper(self.url) with tempfile.NamedTemporaryFile(delete=False) as f: for block in response.iter_content(1024): if not block: break f.write(block) f.flush() return open(f.name)
def put(self, data): # Post to data api # First turn the data to bytes if we can if isinstance(data, six.string_types) and not isinstance( data, six.binary_type): data = bytes(data.encode()) if isinstance(data, six.binary_type): result = self.client.putHelper(self.url, data) if 'error' in result: raise DataApiError(result['error']['message']) else: return self else: raise TypeError( "Must put strings or binary data. Use putJson instead")
def delete(self): try: os.remove(self.path) return True except: raise DataApiError('Failed to delete local file ' + self.path)
def putFile(self, path): result = localPutHelper(path, self.path) if 'error' in result: raise DataApiError(result['error']['message']) else: return self
def getJson(self): exists, error = self.existsWithError() if not exists: raise DataApiError('unable to get file {} - {}'.format( self.path, error)) return json.loads(open(self.path, 'r').read())
def getFile(self): exists, error = self.existsWithError() if not exists: raise DataApiError('unable to get file {} - {}'.format( self.path, error)) return open(self.path)