class WritableTextFilesADLSource(base.DataSource): """ A Azure Data Lake source that also implements a "write" method to write files to data lake. """ name = 'writableadltext' partition_access = True version = '0.0.1dev' def __init__(self, tenant_id, client_id, client_secret, store_name, metadata=None): token = lib.auth(tenant_id=tenant_id, client_id=client_id, client_secret=client_secret) self.adl = AzureDLFileSystem(store_name=store_name, token=token) super(WritableTextFilesADLSource, self).__init__(metadata=metadata) def write(self, local_path, remote_path): print('writing') self.adl.put(local_path, remote_path) def read(self, path): with self.adl.open(path, 'rb') as f: data = f.read() print(data)
def open(self, path, mode='rb'): adl_path = self._trim_filename(path) f = AzureDLFileSystem.open(self, adl_path, mode=mode) return f