def csv2fbe(filein, stem, configJson):
# Sometimes data comes in the DTS-style (depth as rows, time as columns), so we catch that and pass it on
time_row = configJson.get('time_row', -1)
if time_row > -1:
return csv2dts(filein, stem, configJson)
# output directory name
head_tail = os.path.split(filein)
# Assumption: original suffix is .csv or other 3 letter suffix.
stage = (head_tail[1])[:-4]
dirout = os.path.join(stem, stage)
if not os.path.exists(dirout):
os.makedirs(dirout)
print('writing to ' + dirout)
# recover local variables from the config
TIME_FORMAT = configJson.get('time_format', '%Y-%m-%d %H:%M:%S')
csv_labels = configJson.get('csv_labels', [])
curves = configJson.get('curves', [])
if not (len(csv_labels) == len(curves)):
print(
'Different number of csv_labels to curves in the output file, please check the config.'
)
return
if (len(csv_labels) == 0) or (len(curves) == 0):
print('Either no csv_labels or curves found, please check the config.')
return
depth_unit = configJson.get('depth_unit', 'm')
data_style = configJson.get('data_style', 'UNKNOWN')
# Read the CSV and sort by time-stamps
df = pandas.read_csv(filein)
uniquevals = numpy.unique(df[configJson['time_label']].values)
for id in uniquevals:
# extract all the data at the timestamp and sort by depth ready for output
newdf = df[df[configJson['time_label']] == id]
newdf = newdf.sort_values(by=[configJson['depth_label']])
xaxis = numpy.array(newdf[configJson['depth_label']].tolist())
timeval_curr = dtime.strptime(id, TIME_FORMAT)
fileout = str(int(timeval_curr.timestamp()))
datestring = dtime.fromtimestamp(
int(fileout)).strftime("%Y-%m-%dT%H:%M:%S+00:00")
# Convert the data to numpy for writing
data = numpy.zeros((xaxis.shape[0], len(csv_labels)))
for a in range(len(csv_labels)):
data[:, a] = numpy.array(newdf[csv_labels[a]].tolist())
root = witsmlfbe.witsml_fbe(datestring,
xaxis,
curves,
data,
data_style=data_style,
depth_unit=depth_unit)
# The datestring has characters that we do not want in a filename...
# so we remo
witsmlfbe.writeFBE(dirout, fileout, root)
def write2witsml(dirout,
fileout,
datestring,
xaxis,
band00,
data,
low_freq,
high_freq,
prf,
data_style='UNKNOWN',
label_list=[],
unit_list=[]):
# handle the case of a transposed trace....
if (data.shape[0] == 1):
data = numpy.transpose(data)
band00 = numpy.transpose(band00)
# simple axis replaces the actual axis - deferred better axis handling to a future version
xaxis = numpy.linspace(0,
data.shape[0],
data.shape[0],
endpoint=False,
dtype=numpy.double)
curves = witsmlfbe.generic_curves(low_freq, high_freq, prf)
if (len(label_list) > 0):
icurve = 0
for label in label_list:
curves[icurve]['mnemonic'] = label
curves[icurve]['description'] = label
icurve += 1
if (len(unit_list) > 0):
icurve = 0
for unit in unit_list:
curves[icurve]['unit'] = unit
icurve += 1
dataOutsize = len(low_freq)
useB0 = True
if band00 is None:
useB0 = False
if useB0:
dataOutsize = dataOutsize + 1
dataOut = numpy.zeros((data.shape[0], dataOutsize), dtype=numpy.double)
if useB0:
dataOut[:, 0] = numpy.squeeze(band00)
if (len(data.shape)) < 2:
dataOut[:, 1] = data
else:
dataOut[:, 1:] = data
else:
dataOut = data
curves = curves[1:]
root = witsmlfbe.witsml_fbe(datestring,
xaxis,
curves,
dataOut,
data_style=data_style)
# The datestring has characters that we do not want in a filename...
# so we remo
witsmlfbe.writeFBE(dirout, fileout, root)
def csv2dts(filein, stem, configJson):
# output directory name
output_format = configJson.get('output_format', 'witsml')
head_tail = os.path.split(filein)
# Assumption: original suffix is .csv or other 3 letter suffix.
stage = (head_tail[1])[:-4]
dirout = os.path.join(stem, stage)
if not os.path.exists(dirout):
os.makedirs(dirout)
print('writing to ' + dirout)
# recover local variables from the config
TIME_FORMAT = configJson.get('time_format', '%Y-%m-%d %H:%M:%S')
# time_row must exist or we wouldn't assume this data format
time_row = configJson['time_row']
first_data = configJson.get('first_data', 1)
curves = configJson.get('curves',
[{
"mnemonic": "T",
"unit": "F",
"description": "DTS temperature in Fahrenheit"
}])
depth_unit = configJson.get('depth_unit', 'm')
data_style = configJson.get('data_style', 'UNKNOWN')
skiprows = []
for a in range(first_data):
if not a == time_row:
skiprows.append(a)
# Read the CSV
df = pandas.read_csv(filein, header=time_row, skiprows=skiprows)
if output_format == 'witsml':
firstTime = True
for column in df:
if firstTime == True:
xaxis = numpy.array(df[column].tolist())
else:
timeval = dtime.strptime(column, TIME_FORMAT)
fileout = str(int(timeval.timestamp()))
datestring = dtime.fromtimestamp(
int(fileout)).strftime("%Y-%m-%dT%H:%M:%S+00:00")
data = numpy.array(df[column].tolist())
root = witsmlfbe.witsml_fbe(datestring,
xaxis,
curves,
data,
data_style=data_style,
depth_unit=depth_unit)
# The datestring has characters that we do not want in a filename...
# so we remo
witsmlfbe.writeFBE(dirout, fileout, root)
firstTime = False
else:
xaxis = numpy.squeeze(numpy.array(df.values[:, 0]))
cols = list(df.columns.values)
taxis_dates = numpy.array(
list(
map(dtime.strptime, cols[1:],
itertools.repeat(TIME_FORMAT, len(cols[1:])))))
taxis = numpy.array([ts.timestamp() for ts in taxis_dates])
data = df.values[:, 1:]
xaxisfilename = os.path.join(dirout, 'measured_depth.npy')
numpy.save(xaxisfilename, xaxis)
taxisfilename = os.path.join(dirout, 'time.npy')
numpy.save(taxisfilename, taxis)
datafilename = os.path.join(dirout, str(int(taxis[0])) + '.npy')
numpy.save(datafilename, data)
def convert(infile, outfile):
root = witsmlfbe.las2fbe(infile)
witsmlfbe.writeFBE(outpath, outfile, root)