def __init__(self, *args, **kwargs):
super(ClientTestCase, self).__init__(*args, **kwargs)
# Create a in memory database only once for test suite
url = 'sqlite:///:memory:'
self.client = Client(url)
# add paths
session = self.client.session()
path1 = WaveformPath({'path': '/path/to/1'})
path2 = WaveformPath({'path': '/path/to/2'})
session.add_all([path1, path2])
# add files
file1 = WaveformFile({
'file': 'file_001.mseed',
'size': 2000,
'mtime': UTCDateTime('20120101').timestamp,
'format': 'MSEED'
})
file2 = WaveformFile({
'file': 'file_002.mseed',
'size': 2000,
'mtime': UTCDateTime('20120102').timestamp,
'format': 'MSEED'
})
file3 = WaveformFile({
'file': 'file_001.gse2',
'size': 2000,
'mtime': UTCDateTime('20120102').timestamp,
'format': 'GSE2'
})
path1.files.append(file1)
path1.files.append(file2)
path2.files.append(file3)
session.add_all([file1, file2, file3])
# add channels
channel1 = WaveformChannel({
'network':
'BW',
'station':
'MANZ',
'location':
'',
'channel':
'EHZ',
'starttime':
UTCDateTime('2012-01-01 00:00:00.000000').datetime,
'endtime':
UTCDateTime('2012-01-01 23:59:59.999999').datetime,
'npts':
3000,
'sampling_rate':
100.0
})
channel2 = WaveformChannel({
'network':
'BW',
'station':
'MANZ',
'location':
'',
'channel':
'EHZ',
'starttime':
UTCDateTime('2012-01-02 01:00:00.000000').datetime,
'endtime':
UTCDateTime('2012-01-02 23:59:59.999999').datetime,
'npts':
3000,
'sampling_rate':
100.0
})
# create a channel with preview
header = {
'network': 'GE',
'station': 'FUR',
'location': '00',
'channel': 'BHZ',
'starttime': UTCDateTime('2012-01-01 00:00:00.000000'),
'sampling_rate': 100.0
}
# linear trend
data = np.linspace(0, 1, 3000000)
# some peaks
data[20000] = 15
data[20001] = -15
data[1000000] = 22
data[1000001] = -22
data[2000000] = 14
data[2000001] = -14
tr = Trace(data=data, header=header)
self.preview = createPreview(tr, 30).data
header = dict(tr.stats)
header['starttime'] = tr.stats.starttime.datetime
header['endtime'] = tr.stats.endtime.datetime
channel3 = WaveformChannel(header)
channel3.preview = self.preview.dumps()
file1.channels.append(channel1)
file2.channels.append(channel2)
file3.channels.append(channel3)
session.add_all([channel1, channel2, channel3])
session.commit()
session.close()
def _update_or_insert(self, dataset):
"""
Add a new file into or modifies existing file in database.
"""
if len(dataset) < 1:
return
session = self.session()
data = dataset[0]
# check for duplicates
if self.options.check_duplicates:
query = session.query(WaveformFile, WaveformChannel, WaveformPath)
query = query.filter(WaveformPath.id == WaveformFile.path_id)
query = query.filter(WaveformFile.id == WaveformChannel.file_id)
query = query.filter(WaveformPath.path != data['path'])
query = query.filter(WaveformFile.file == data['file'])
query = query.filter(WaveformChannel.network == data['network'])
query = query.filter(WaveformChannel.station == data['station'])
query = query.filter(WaveformChannel.location == data['location'])
query = query.filter(WaveformChannel.channel == data['channel'])
query = query.filter(
WaveformChannel.starttime == data['starttime'])
query = query.filter(WaveformChannel.endtime == data['endtime'])
if query.count() > 0:
msg = "Duplicate entry '%s' in '%s'."
self.log.error(msg % (data['file'], data['path']))
return
# fetch or create path
try:
# search for existing path
query = session.query(WaveformPath)
path = query.filter_by(path=data['path']).one()
except Exception:
# create new path entry
path = WaveformPath(data)
session.add(path)
# search and delete existing file entry
msg = "Inserted"
if path.id is not None:
# search for existing file
query = session.query(WaveformFile)
files = query.filter_by(path_id=path.id, file=data['file']).all()
if files:
msg = "Updated"
# delete existing file entry and all related information
for file in files:
session.delete(file)
# create new file entry
file = WaveformFile(data)
path.files.append(file)
# add channel entries
for data in dataset:
# create new channel entry
channel = WaveformChannel(data)
file.channels.append(channel)
# add gaps
for gap in data['gaps']:
channel.gaps.append(WaveformGaps(gap))
# add features
for feature in data['features']:
channel.features.append(WaveformFeatures(feature))
try:
session.commit()
except Exception as e:
session.rollback()
self.log.error(str(e))
else:
self.log.debug("%s '%s' in '%s'" %
(msg, data['file'], data['path']))
session.close()
