def export(self, data, filename, format=None, overwrite=False, varname=None): """ Export local array data to a variety of formats. Can write to a local file sytem or S3 (destination inferred from filename schema). S3 writing useful for persisting arrays when working in an environment without accessible local storage. Parameters ---------- data : array-like The data to export filename : str Output location (path/to/file.ext) format : str, optional, default = None Ouput format ("npy", "mat", or "txt"), if not provided will try to infer from file extension. overwrite : boolean, optional, default = False Whether to overwrite if directory or file already exists varname : str, optional, default = None Variable name for writing "mat" formatted files """ from numpy import save, savetxt from scipy.io import savemat from StringIO import StringIO from thunder.rdds.fileio.writers import getFileWriterForPath path, file, format = handleFormat(filename, format) checkParams(format, ["npy", "mat", "txt"]) clazz = getFileWriterForPath(filename) writer = clazz(path, file, overwrite=overwrite, awsCredentialsOverride=self._credentials) stream = StringIO() if format == "mat": varname = os.path.splitext(file)[0] if varname is None else varname savemat(stream, mdict={varname: data}, oned_as='column', do_compression='true') if format == "npy": save(stream, data) if format == "txt": savetxt(stream, data) stream.seek(0) writer.writeFile(stream.buf)
def writeSeriesConfig(outputdirname, nkeys, nvalues, dims=None, keytype='int16', valuetype='int16', confname="conf.json", overwrite=True): """Helper function to write out a conf.json file with required information to load Series binary data. """ import json from thunder.rdds.fileio.writers import getFileWriterForPath filewriterclass = getFileWriterForPath(outputdirname) # write configuration file conf = { 'input': outputdirname, 'nkeys': nkeys, 'nvalues': nvalues, 'valuetype': str(valuetype), 'keytype': str(keytype) } if dims: conf["dims"] = dims confwriter = filewriterclass(outputdirname, confname, overwrite=overwrite) confwriter.writeFile(json.dumps(conf, indent=2)) # touch "SUCCESS" file as final action successwriter = filewriterclass(outputdirname, "SUCCESS", overwrite=overwrite) successwriter.writeFile('')
def export(self, data, filename, outputFormat=None, overwrite=False, varname=None): """ Export local array data to a variety of formats. Can write to a local file sytem or S3 (destination inferred from filename schema). S3 writing useful for persisting arrays when working in an environment without accessible local storage. Parameters ---------- data : array-like The data to export filename : str Output location (path/to/file.ext) outputFormat : str, optional, default = None Ouput format ("npy", "mat", or "txt"), if not provided will try to infer from file extension. overwrite : boolean, optional, default = False Whether to overwrite if directory or file already exists varname : str, optional, default = None Variable name for writing "mat" formatted files """ from numpy import save, savetxt, asarray from scipy.io import savemat from StringIO import StringIO from thunder.rdds.fileio.writers import getFileWriterForPath path, file, outputFormat = handleFormat(filename, outputFormat) checkParams(outputFormat, ["npy", "mat", "txt"]) clazz = getFileWriterForPath(filename) writer = clazz(path, file, overwrite=overwrite, awsCredentialsOverride=self._credentials) stream = StringIO() if outputFormat == "mat": varname = os.path.splitext(file)[0] if varname is None else varname savemat(stream, mdict={varname: data}, oned_as='column', do_compression='true') if outputFormat == "npy": save(stream, data) if outputFormat == "txt": if asarray(data).ndim > 2: raise Exception("Cannot write data with more than two dimensions to text") savetxt(stream, data) stream.seek(0) writer.writeFile(stream.buf)
def writeBinaryImagesConfig(outputDirPath, dims, dtype='int16', confFilename="conf.json", overwrite=True, awsCredentialsOverride=None): """ Helper function to write out a conf.json file with required information to load binary Image data. """ from thunder.rdds.fileio.writers import getFileWriterForPath filewriterClass = getFileWriterForPath(outputDirPath) # write configuration file conf = {'dims': dims, 'dtype': dtype} confWriter = filewriterClass(outputDirPath, confFilename, overwrite=overwrite, awsCredentialsOverride=awsCredentialsOverride) confWriter.writeFile(json.dumps(conf, indent=2)) # touch "SUCCESS" file as final action successWriter = filewriterClass(outputDirPath, "SUCCESS", overwrite=overwrite, awsCredentialsOverride=awsCredentialsOverride) successWriter.writeFile('')
def writeSeriesConfig(outputdirname, nkeys, nvalues, dims=None, keytype='int16', valuetype='int16', confname="conf.json", overwrite=True): """Helper function to write out a conf.json file with required information to load Series binary data. """ import json from thunder.rdds.fileio.writers import getFileWriterForPath filewriterclass = getFileWriterForPath(outputdirname) # write configuration file conf = {'input': outputdirname, 'nkeys': nkeys, 'nvalues': nvalues, 'valuetype': str(valuetype), 'keytype': str(keytype)} if dims: conf["dims"] = dims confwriter = filewriterclass(outputdirname, confname, overwrite=overwrite) confwriter.writeFile(json.dumps(conf, indent=2)) # touch "SUCCESS" file as final action successwriter = filewriterclass(outputdirname, "SUCCESS", overwrite=overwrite) successwriter.writeFile('')
def writeSeriesConfig(outputDirPath, nkeys, nvalues, keyType='int16', valueType='int16', confFilename="conf.json", overwrite=True, awsCredentialsOverride=None): """Helper function to write out a conf.json file with required information to load Series binary data. """ import json from thunder.rdds.fileio.writers import getFileWriterForPath filewriterClass = getFileWriterForPath(outputDirPath) # write configuration file # config JSON keys are lowercased "valuetype", "keytype", not valueType, keyType conf = {'input': outputDirPath, 'nkeys': nkeys, 'nvalues': nvalues, 'valuetype': str(valueType), 'keytype': str(keyType)} confWriter = filewriterClass(outputDirPath, confFilename, overwrite=overwrite, awsCredentialsOverride=awsCredentialsOverride) confWriter.writeFile(json.dumps(conf, indent=2)) # touch "SUCCESS" file as final action successWriter = filewriterClass(outputDirPath, "SUCCESS", overwrite=overwrite, awsCredentialsOverride=awsCredentialsOverride) successWriter.writeFile('')
def writeSeriesConfig(outputDirPath, nkeys, nvalues, keyType='int16', valueType='int16', confFilename="conf.json", overwrite=True, awsCredentialsOverride=None): """ Helper function to write out a conf.json file with required information to load Series binary data. """ import json from thunder.rdds.fileio.writers import getFileWriterForPath filewriterClass = getFileWriterForPath(outputDirPath) # write configuration file # config JSON keys are lowercased "valuetype", "keytype", not valueType, keyType conf = {'input': outputDirPath, 'nkeys': nkeys, 'nvalues': nvalues, 'valuetype': str(valueType), 'keytype': str(keyType)} confWriter = filewriterClass(outputDirPath, confFilename, overwrite=overwrite, awsCredentialsOverride=awsCredentialsOverride) confWriter.writeFile(json.dumps(conf, indent=2)) # touch "SUCCESS" file as final action successWriter = filewriterClass(outputDirPath, "SUCCESS", overwrite=overwrite, awsCredentialsOverride=awsCredentialsOverride) successWriter.writeFile('')