def export(self,
data,
filename,
format=None,
overwrite=False,
varname=None):
"""
Export local array data to a variety of formats.
Can write to a local file sytem or S3 (destination inferred from filename schema).
S3 writing useful for persisting arrays when working in an environment without
accessible local storage.
Parameters
----------
data : array-like
The data to export
filename : str
Output location (path/to/file.ext)
format : str, optional, default = None
Ouput format ("npy", "mat", or "txt"), if not provided will
try to infer from file extension.
overwrite : boolean, optional, default = False
Whether to overwrite if directory or file already exists
varname : str, optional, default = None
Variable name for writing "mat" formatted files
"""
from numpy import save, savetxt
from scipy.io import savemat
from StringIO import StringIO
from thunder.rdds.fileio.writers import getFileWriterForPath
path, file, format = handleFormat(filename, format)
checkParams(format, ["npy", "mat", "txt"])
clazz = getFileWriterForPath(filename)
writer = clazz(path,
file,
overwrite=overwrite,
awsCredentialsOverride=self._credentials)
stream = StringIO()
if format == "mat":
varname = os.path.splitext(file)[0] if varname is None else varname
savemat(stream,
mdict={varname: data},
oned_as='column',
do_compression='true')
if format == "npy":
save(stream, data)
if format == "txt":
savetxt(stream, data)
stream.seek(0)
writer.writeFile(stream.buf)
def writeSeriesConfig(outputdirname,
nkeys,
nvalues,
dims=None,
keytype='int16',
valuetype='int16',
confname="conf.json",
overwrite=True):
"""Helper function to write out a conf.json file with required information to load Series binary data.
"""
import json
from thunder.rdds.fileio.writers import getFileWriterForPath
filewriterclass = getFileWriterForPath(outputdirname)
# write configuration file
conf = {
'input': outputdirname,
'nkeys': nkeys,
'nvalues': nvalues,
'valuetype': str(valuetype),
'keytype': str(keytype)
}
if dims:
conf["dims"] = dims
confwriter = filewriterclass(outputdirname, confname, overwrite=overwrite)
confwriter.writeFile(json.dumps(conf, indent=2))
# touch "SUCCESS" file as final action
successwriter = filewriterclass(outputdirname,
"SUCCESS",
overwrite=overwrite)
successwriter.writeFile('')
def export(self, data, filename, outputFormat=None, overwrite=False, varname=None):
"""
Export local array data to a variety of formats.
Can write to a local file sytem or S3 (destination inferred from filename schema).
S3 writing useful for persisting arrays when working in an environment without
accessible local storage.
Parameters
----------
data : array-like
The data to export
filename : str
Output location (path/to/file.ext)
outputFormat : str, optional, default = None
Ouput format ("npy", "mat", or "txt"), if not provided will
try to infer from file extension.
overwrite : boolean, optional, default = False
Whether to overwrite if directory or file already exists
varname : str, optional, default = None
Variable name for writing "mat" formatted files
"""
from numpy import save, savetxt, asarray
from scipy.io import savemat
from StringIO import StringIO
from thunder.rdds.fileio.writers import getFileWriterForPath
path, file, outputFormat = handleFormat(filename, outputFormat)
checkParams(outputFormat, ["npy", "mat", "txt"])
clazz = getFileWriterForPath(filename)
writer = clazz(path, file, overwrite=overwrite, awsCredentialsOverride=self._credentials)
stream = StringIO()
if outputFormat == "mat":
varname = os.path.splitext(file)[0] if varname is None else varname
savemat(stream, mdict={varname: data}, oned_as='column', do_compression='true')
if outputFormat == "npy":
save(stream, data)
if outputFormat == "txt":
if asarray(data).ndim > 2:
raise Exception("Cannot write data with more than two dimensions to text")
savetxt(stream, data)
stream.seek(0)
writer.writeFile(stream.buf)
def writeBinaryImagesConfig(outputDirPath, dims, dtype='int16',
confFilename="conf.json", overwrite=True, awsCredentialsOverride=None):
"""
Helper function to write out a conf.json file with required information to load binary Image data.
"""
from thunder.rdds.fileio.writers import getFileWriterForPath
filewriterClass = getFileWriterForPath(outputDirPath)
# write configuration file
conf = {'dims': dims, 'dtype': dtype}
confWriter = filewriterClass(outputDirPath, confFilename, overwrite=overwrite,
awsCredentialsOverride=awsCredentialsOverride)
confWriter.writeFile(json.dumps(conf, indent=2))
# touch "SUCCESS" file as final action
successWriter = filewriterClass(outputDirPath, "SUCCESS", overwrite=overwrite,
awsCredentialsOverride=awsCredentialsOverride)
successWriter.writeFile('')
def writeSeriesConfig(outputdirname, nkeys, nvalues, dims=None, keytype='int16', valuetype='int16',
confname="conf.json", overwrite=True):
"""Helper function to write out a conf.json file with required information to load Series binary data.
"""
import json
from thunder.rdds.fileio.writers import getFileWriterForPath
filewriterclass = getFileWriterForPath(outputdirname)
# write configuration file
conf = {'input': outputdirname,
'nkeys': nkeys, 'nvalues': nvalues,
'valuetype': str(valuetype), 'keytype': str(keytype)}
if dims:
conf["dims"] = dims
confwriter = filewriterclass(outputdirname, confname, overwrite=overwrite)
confwriter.writeFile(json.dumps(conf, indent=2))
# touch "SUCCESS" file as final action
successwriter = filewriterclass(outputdirname, "SUCCESS", overwrite=overwrite)
successwriter.writeFile('')
def writeSeriesConfig(outputDirPath, nkeys, nvalues, keyType='int16', valueType='int16',
confFilename="conf.json", overwrite=True, awsCredentialsOverride=None):
"""Helper function to write out a conf.json file with required information to load Series binary data.
"""
import json
from thunder.rdds.fileio.writers import getFileWriterForPath
filewriterClass = getFileWriterForPath(outputDirPath)
# write configuration file
# config JSON keys are lowercased "valuetype", "keytype", not valueType, keyType
conf = {'input': outputDirPath,
'nkeys': nkeys, 'nvalues': nvalues,
'valuetype': str(valueType), 'keytype': str(keyType)}
confWriter = filewriterClass(outputDirPath, confFilename, overwrite=overwrite,
awsCredentialsOverride=awsCredentialsOverride)
confWriter.writeFile(json.dumps(conf, indent=2))
# touch "SUCCESS" file as final action
successWriter = filewriterClass(outputDirPath, "SUCCESS", overwrite=overwrite,
awsCredentialsOverride=awsCredentialsOverride)
successWriter.writeFile('')
def writeSeriesConfig(outputDirPath, nkeys, nvalues, keyType='int16', valueType='int16',
confFilename="conf.json", overwrite=True, awsCredentialsOverride=None):
"""
Helper function to write out a conf.json file with required information to load Series binary data.
"""
import json
from thunder.rdds.fileio.writers import getFileWriterForPath
filewriterClass = getFileWriterForPath(outputDirPath)
# write configuration file
# config JSON keys are lowercased "valuetype", "keytype", not valueType, keyType
conf = {'input': outputDirPath,
'nkeys': nkeys, 'nvalues': nvalues,
'valuetype': str(valueType), 'keytype': str(keyType)}
confWriter = filewriterClass(outputDirPath, confFilename, overwrite=overwrite,
awsCredentialsOverride=awsCredentialsOverride)
confWriter.writeFile(json.dumps(conf, indent=2))
# touch "SUCCESS" file as final action
successWriter = filewriterClass(outputDirPath, "SUCCESS", overwrite=overwrite,
awsCredentialsOverride=awsCredentialsOverride)
successWriter.writeFile('')
