def __init__(self, *args, **kwargs):
super(ClientTestCase, self).__init__(*args, **kwargs)
# Create a in memory database only once for test suite
url = 'sqlite:///:memory:'
self.client = Client(url)
# add paths
session = self.client.session()
path1 = WaveformPath({'path': '/path/to/1'})
path2 = WaveformPath({'path': '/path/to/2'})
session.add_all([path1, path2])
# add files
file1 = WaveformFile(
{'file': 'file_001.mseed', 'size': 2000,
'mtime': UTCDateTime('20120101').timestamp, 'format': 'MSEED'})
file2 = WaveformFile(
{'file': 'file_002.mseed', 'size': 2000,
'mtime': UTCDateTime('20120102').timestamp, 'format': 'MSEED'})
file3 = WaveformFile(
{'file': 'file_001.gse2', 'size': 2000,
'mtime': UTCDateTime('20120102').timestamp, 'format': 'GSE2'})
path1.files.append(file1)
path1.files.append(file2)
path2.files.append(file3)
session.add_all([file1, file2, file3])
# add channels
channel1 = WaveformChannel(
{'network': 'BW', 'station': 'MANZ',
'location': '', 'channel': 'EHZ',
'starttime':
UTCDateTime('2012-01-01 00:00:00.000000').datetime,
'endtime': UTCDateTime('2012-01-01 23:59:59.999999').datetime,
'npts': 3000, 'sampling_rate': 100.0})
channel2 = WaveformChannel(
{'network': 'BW', 'station': 'MANZ',
'location': '', 'channel': 'EHZ',
'starttime':
UTCDateTime('2012-01-02 01:00:00.000000').datetime,
'endtime':
UTCDateTime('2012-01-02 23:59:59.999999').datetime,
'npts': 3000,
'sampling_rate': 100.0})
# create a channel with preview
header = {'network': 'GE', 'station': 'FUR',
'location': '00', 'channel': 'BHZ',
'starttime': UTCDateTime('2012-01-01 00:00:00.000000'),
'sampling_rate': 100.0}
# linear trend
data = np.linspace(0, 1, 3000000)
# some peaks
data[20000] = 15
data[20001] = -15
data[1000000] = 22
data[1000001] = -22
data[2000000] = 14
data[2000001] = -14
tr = Trace(data=data, header=header)
self.preview = createPreview(tr, 30).data
header = dict(tr.stats)
header['starttime'] = tr.stats.starttime.datetime
header['endtime'] = tr.stats.endtime.datetime
channel3 = WaveformChannel(header)
channel3.preview = self.preview.dumps()
file1.channels.append(channel1)
file2.channels.append(channel2)
file3.channels.append(channel3)
session.add_all([channel1, channel2, channel3])
session.commit()
session.close()
def _update_or_insert(self, dataset):
"""
Add a new file into or modifies existing file in database.
"""
if len(dataset) < 1:
return
session = self.session()
data = dataset[0]
# check for duplicates
if self.options.check_duplicates:
query = session.query(WaveformFile, WaveformChannel, WaveformPath)
query = query.filter(WaveformPath.id == WaveformFile.path_id)
query = query.filter(WaveformFile.id == WaveformChannel.file_id)
query = query.filter(WaveformPath.path != data['path'])
query = query.filter(WaveformFile.file == data['file'])
query = query.filter(WaveformChannel.network == data['network'])
query = query.filter(WaveformChannel.station == data['station'])
query = query.filter(WaveformChannel.location == data['location'])
query = query.filter(WaveformChannel.channel == data['channel'])
query = query.filter(
WaveformChannel.starttime == data['starttime'])
query = query.filter(WaveformChannel.endtime == data['endtime'])
if query.count() > 0:
msg = "Duplicate entry '%s' in '%s'."
self.log.error(msg % (data['file'], data['path']))
return
# fetch or create path
try:
# search for existing path
query = session.query(WaveformPath)
path = query.filter_by(path=data['path']).one()
except Exception:
# create new path entry
path = WaveformPath(data)
session.add(path)
# search and delete existing file entry
msg = "Inserted"
if path.id is not None:
# search for existing file
query = session.query(WaveformFile)
files = query.filter_by(path_id=path.id, file=data['file']).all()
if files:
msg = "Updated"
# delete existing file entry and all related information
for file in files:
session.delete(file)
# create new file entry
file = WaveformFile(data)
path.files.append(file)
# add channel entries
for data in dataset:
# create new channel entry
channel = WaveformChannel(data)
file.channels.append(channel)
# add gaps
for gap in data['gaps']:
channel.gaps.append(WaveformGaps(gap))
# add features
for feature in data['features']:
channel.features.append(WaveformFeatures(feature))
try:
session.commit()
except Exception as e:
session.rollback()
self.log.error(str(e))
else:
self.log.debug("%s '%s' in '%s'" %
(msg, data['file'], data['path']))
session.close()
def __init__(self, *args, **kwargs):
super(ClientTestCase, self).__init__(*args, **kwargs)
# Create a in memory database only once for test suite
url = 'sqlite:///:memory:'
self.client = Client(url)
# add paths
session = self.client.session()
path1 = WaveformPath({'path': '/path/to/1'})
path2 = WaveformPath({'path': '/path/to/2'})
session.add_all([path1, path2])
# add files
file1 = WaveformFile({
'file': 'file_001.mseed',
'size': 2000,
'mtime': UTCDateTime('20120101').timestamp,
'format': 'MSEED'
})
file2 = WaveformFile({
'file': 'file_002.mseed',
'size': 2000,
'mtime': UTCDateTime('20120102').timestamp,
'format': 'MSEED'
})
file3 = WaveformFile({
'file': 'file_001.gse2',
'size': 2000,
'mtime': UTCDateTime('20120102').timestamp,
'format': 'GSE2'
})
path1.files.append(file1)
path1.files.append(file2)
path2.files.append(file3)
session.add_all([file1, file2, file3])
# add channels
channel1 = WaveformChannel({
'network':
'BW',
'station':
'MANZ',
'location':
'',
'channel':
'EHZ',
'starttime':
UTCDateTime('2012-01-01 00:00:00.000000').datetime,
'endtime':
UTCDateTime('2012-01-01 23:59:59.999999').datetime,
'npts':
3000,
'sampling_rate':
100.0
})
channel2 = WaveformChannel({
'network':
'BW',
'station':
'MANZ',
'location':
'',
'channel':
'EHZ',
'starttime':
UTCDateTime('2012-01-02 01:00:00.000000').datetime,
'endtime':
UTCDateTime('2012-01-02 23:59:59.999999').datetime,
'npts':
3000,
'sampling_rate':
100.0
})
# create a channel with preview
header = {
'network': 'GE',
'station': 'FUR',
'location': '00',
'channel': 'BHZ',
'starttime': UTCDateTime('2012-01-01 00:00:00.000000'),
'sampling_rate': 100.0
}
# linear trend
data = np.linspace(0, 1, 3000000)
# some peaks
data[20000] = 15
data[20001] = -15
data[1000000] = 22
data[1000001] = -22
data[2000000] = 14
data[2000001] = -14
tr = Trace(data=data, header=header)
self.preview = createPreview(tr, 30).data
header = dict(tr.stats)
header['starttime'] = tr.stats.starttime.datetime
header['endtime'] = tr.stats.endtime.datetime
channel3 = WaveformChannel(header)
channel3.preview = self.preview.dumps()
file1.channels.append(channel1)
file2.channels.append(channel2)
file3.channels.append(channel3)
session.add_all([channel1, channel2, channel3])
session.commit()
session.close()
def setUpClass(cls):
# Create a in memory database only once for test suite
url = "sqlite:///:memory:"
cls.client = Client(url)
# add paths
session = cls.client.session()
path1 = WaveformPath({"path": "/path/to/1"})
path2 = WaveformPath({"path": "/path/to/2"})
session.add_all([path1, path2])
# add files
file1 = WaveformFile(
{"file": "file_001.mseed", "size": 2000, "mtime": UTCDateTime("20120101").timestamp, "format": "MSEED"}
)
file2 = WaveformFile(
{"file": "file_002.mseed", "size": 2000, "mtime": UTCDateTime("20120102").timestamp, "format": "MSEED"}
)
file3 = WaveformFile(
{"file": "file_001.gse2", "size": 2000, "mtime": UTCDateTime("20120102").timestamp, "format": "GSE2"}
)
path1.files.append(file1)
path1.files.append(file2)
path2.files.append(file3)
session.add_all([file1, file2, file3])
# add channels
channel1 = WaveformChannel(
{
"network": "BW",
"station": "MANZ",
"location": "",
"channel": "EHZ",
"starttime": UTCDateTime("2012-01-01 00:00:00.000000").datetime,
"endtime": UTCDateTime("2012-01-01 23:59:59.999999").datetime,
"npts": 3000,
"sampling_rate": 100.0,
}
)
channel2 = WaveformChannel(
{
"network": "BW",
"station": "MANZ",
"location": "",
"channel": "EHZ",
"starttime": UTCDateTime("2012-01-02 01:00:00.000000").datetime,
"endtime": UTCDateTime("2012-01-02 23:59:59.999999").datetime,
"npts": 3000,
"sampling_rate": 100.0,
}
)
# create a channel with preview
header = {
"network": "GE",
"station": "FUR",
"location": "00",
"channel": "BHZ",
"starttime": UTCDateTime("2012-01-01 00:00:00.000000"),
"sampling_rate": 100.0,
}
# linear trend
data = np.linspace(0, 1, 3000000)
# some peaks
data[20000] = 15
data[20001] = -15
data[1000000] = 22
data[1000001] = -22
data[2000000] = 14
data[2000001] = -14
tr = Trace(data=data, header=header)
cls.preview = create_preview(tr, 30).data
header = dict(tr.stats)
header["starttime"] = tr.stats.starttime.datetime
header["endtime"] = tr.stats.endtime.datetime
channel3 = WaveformChannel(header)
channel3.preview = cls.preview.dumps()
file1.channels.append(channel1)
file2.channels.append(channel2)
file3.channels.append(channel3)
session.add_all([channel1, channel2, channel3])
session.commit()
session.close()