def update_vaps(cls, vaps_records, *args):
cursor = DbUtils().get_cursor(args)
progress_message = seis_utils.progress_message_generator(
f'populate database for table: {cls.table_vaps} '
)
sql_string = (
f'INSERT INTO {cls.table_vaps} ('
f'file_id, line, point, fleet_nr, vibrator, drive, '
f'avg_phase, peak_phase, avg_dist, peak_dist, avg_force, peak_force, '
f'avg_stiffness, avg_viscosity, easting, northing, elevation, '
f'time_break, hdop, tb_date, positioning, geom) '
f'VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, '
f'%s, %s, %s, %s, %s, %s, ST_SetSRID(%s::geometry, %s));')
for vaps_record in vaps_records:
point = Point(vaps_record.easting, vaps_record.northing)
cursor.execute(
sql_string,
(vaps_record.file_id, vaps_record.line, vaps_record.station,
vaps_record.fleet_nr, vaps_record.vibrator, vaps_record.drive,
vaps_record.avg_phase, vaps_record.peak_phase,
vaps_record.avg_dist, vaps_record.peak_dist,
vaps_record.avg_force, vaps_record.peak_force,
vaps_record.avg_stiffness, vaps_record.avg_viscosity,
vaps_record.easting, vaps_record.northing,
vaps_record.elevation, vaps_record.time_break,
vaps_record.hdop, vaps_record.tb_date,
vaps_record.positioning, point.wkb_hex, EPSG_PSD93))
next(progress_message)
def update_vp(cls, vp_records, *args, link_vaps=False):
cursor = DbUtils().get_cursor(args)
progress_message = seis_utils.progress_message_generator(
f'populate database for table: {cls.table_vp} ')
sql_vp_record = (
f'INSERT INTO {cls.table_vp} ('
f'file_id, vaps_id, line, station, vibrator, time_break, '
f'planned_easting, planned_northing, easting, northing, elevation, _offset, '
f'peak_force, avg_force, peak_dist, avg_dist, peak_phase, avg_phase, '
f'qc_flag, geom) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, '
f'%s, %s, %s, %s, %s, %s, %s, %s, %s, ST_SetSRID(%s::geometry, %s));'
)
for vp_record in vp_records:
point = Point(vp_record.easting, vp_record.northing)
if link_vaps:
vp_record = cls.get_vaps_id(vp_record)
cursor.execute(
sql_vp_record,
(vp_record.file_id, vp_record.vaps_id, vp_record.line,
vp_record.station, vp_record.vibrator, vp_record.time_break,
vp_record.planned_easting, vp_record.planned_northing,
vp_record.easting, vp_record.northing, vp_record.elevation,
vp_record.offset, vp_record.peak_force, vp_record.avg_force,
vp_record.peak_dist, vp_record.avg_dist, vp_record.peak_phase,
vp_record.avg_phase, vp_record.qc_flag, point.wkb_hex,
EPSG_PSD93))
next(progress_message)
def read_rcv(cls):
for foldername, _, filenames in os.walk(cls.rcv_base_folder):
for filename in filenames:
if filename[-5:] not in ['.noon', '.NOON']:
continue
rcv_file = FilesRcvTable(*[None] * 2)
abs_filename = os.path.abspath(
os.path.join(foldername, filename))
rcv_file.file_name = abs_filename
rcv_file.file_date = (datetime.fromtimestamp(
os.stat(abs_filename).st_mtime))
id_file = cls.rcv_db.update_rcv_file(rcv_file)
if id_file == -1:
continue
progress_message = seis_utils.progress_message_generator(
f'reading receiver data from {rcv_file.file_name} ')
rcv_records = []
with open(abs_filename, mode='rt') as rcv:
for rcv_line in rcv.readlines():
# if rcv_line[0:7].strip() != 'FDU-428':
# continue
rcv_record = cls.parse_rcv_line(rcv_line)
if rcv_record.fdu_sn:
rcv_record.id_file = id_file
rcv_records.append(rcv_record)
next(progress_message)
cls.rcv_db.update_rcv(rcv_records)
def update_rcv(cls, rcv_records, *args):
cursor = DbUtils().get_cursor(args)
progress_message = seis_utils.progress_message_generator(
f'populate database for table: {cls.table_points} '
)
for rcv_record in rcv_records:
rcv_record.geom = cls.create_point_record(cursor, rcv_record)
cls.create_attr_record(cursor, rcv_record)
next(progress_message)
def populate_vps_by_second(self):
progress_message = seis_utils.progress_message_generator(
f'process records for {self.production_date.strftime("%d-%b-%Y")}')
for vib in range(1, FLEETS + 1):
vib_data = self.vp_records_df[self.vp_records_df['vibrator'] ==
vib]['time_break'].to_list()
for vp_time in vib_data:
vp_seconds = int(vp_time.time().hour * 3600 +
vp_time.time().minute * 60 +
vp_time.time().second)
self.vps_by_second[vp_seconds].append(vib)
next(progress_message)
def read_vaps(cls):
for foldername, _, filenames in os.walk(cls.vaps_base_folder):
for filename in filenames:
if not (filename[-5:] in ['.vaps', '.VAPS'] or
filename[-4:] in ['.txt', '.TXT']):
continue
vaps_file = FilesVapsTable(*[None]*3)
abs_filename = os.path.abspath(os.path.join(foldername, filename))
vaps_file.file_name = abs_filename
vaps_file.file_date = (
datetime.datetime.fromtimestamp(os.stat(abs_filename).st_mtime))
file_id = cls.vp_db.update_vaps_file(vaps_file)
if file_id == -1:
continue
progress_message = seis_utils.progress_message_generator(
f'reading vaps from {vaps_file.file_name} ')
vaps_records = []
vaps_signatures = np.array([])
count = 0
with open(abs_filename, mode='rt') as vaps:
for vaps_line in vaps.readlines():
if vaps_line[0] != 'A':
continue
vaps_record = cls.parse_vaps_line(vaps_line, file_id)
vaps_records, vaps_signatures = seis_utils.update_records(
vaps_records, vaps_signatures, vaps_record)
next(progress_message)
count += 1
print(f'\n{count - len(vaps_records)} duplicates have been deleted ...')
if vaps_records:
cls.vp_db.update_vaps(vaps_records)
_date = vaps_records[0].time_break.date()
cls.vp_db.patch_add_distance_column('VAPS', _date, _date)
print()
def read_vp(cls):
for foldername, _, filenames in os.walk(cls.vp_base_folder):
for filename in filenames:
if filename[-4:] not in ['.txt', '.TXT']:
continue
vp_file = FilesVpTable(*[None]*3)
abs_filename = os.path.abspath(os.path.join(foldername, filename))
vp_file.file_name = abs_filename
vp_file.file_date = (
datetime.datetime.fromtimestamp(os.stat(abs_filename).st_mtime))
file_id = cls.vp_db.update_vp_file(vp_file)
if file_id == -1:
continue
progress_message = seis_utils.progress_message_generator(
f'reading vp from {vp_file.file_name} ')
vp_records = []
vp_signatures = np.array([])
count = 0
with open(abs_filename, mode='rt') as vp:
for vp_line in vp.readlines():
if vp_line[0:9].strip() == 'Line':
continue
vp_record = cls.parse_vp_line(vp_line, file_id)
vp_records, vp_signatures = seis_utils.update_records(
vp_records, vp_signatures, vp_record)
next(progress_message)
count += 1
print(f'\n{count - len(vp_records)} duplicates have been deleted ...')
if vp_records:
cls.vp_db.update_vp(vp_records, link_vaps=LINK_VP_TO_VAPS)
_date = vp_records[0].time_break.date()
cls.vp_db.patch_add_distance_column('VP', _date, _date)
print()
def patch_add_distance_column(cls, database_table, start_date, end_date,
*args):
''' patch to add column distance to vaps table
arguments:
database_table: 'VAPS' or 'VP'
state_date: datetime.date object start date
end_date: datetime.date object end date
'''
cursor = DbUtils().get_cursor(args)
if database_table == 'VAPS':
table = cls.table_vaps
else:
table = cls.table_vp
sql_string = (f'UPDATE {table} '
f'SET'
f' distance = %s, '
f' time = %s, '
f' velocity = %s, '
f' dense_flag = %s '
f'WHERE id = %s;')
_date = start_date
while _date <= end_date:
progress_message = seis_utils.progress_message_generator(
f'add dist, time, vel, dense_flag to {table} for '
f'{_date.strftime("%d-%m-%Y")} ')
vp_records_df = cls.get_data_by_date(database_table, _date)
for vib in range(1, FLEETS):
vib_df = vp_records_df[vp_records_df['vibrator'] == vib]
vp_pts = [
(id_xy[0], Point(id_xy[1], id_xy[2]), id_xy[3])
for id_xy in zip( #pylint: disable=line-too-long
vib_df['id'].to_list(),
vib_df['easting'].to_list(),
vib_df['northing'].to_list(),
vib_df['time_break'].to_list(),
)
]
dense_flag_1 = False
for i in range(len(vp_pts) - 1):
index = vp_pts[i][0]
dx = vp_pts[i + 1][1].x - vp_pts[i][1].x
dy = vp_pts[i + 1][1].y - vp_pts[i][1].y
dist = np.sqrt(dx * dx + dy * dy)
time = max(0, (vp_pts[i + 1][2] - vp_pts[i][2]).seconds -
SWEEP_TIME - PAD_DOWN_TIME)
try:
velocity = dist / time
except ZeroDivisionError:
velocity = 0
if dist < DENSE_CRITERIUM or dense_flag_1:
dense_flag = True
else:
dense_flag = False
cursor.execute(sql_string, (
dist,
time,
velocity,
dense_flag,
index,
))
if dist < DENSE_CRITERIUM:
dense_flag_1 = True
else:
dense_flag_1 = False
next(progress_message)
# handle data for last element if there is one
if vp_pts:
index = vp_pts[-1][0]
cursor.execute(sql_string, (
np.nan,
np.nan,
np.nan,
dense_flag_1,
index,
))
_date += datetime.timedelta(days=+1)