def parse_time(time_str, role):
# If there is no argument, assume unbounded.
# But we do need to use something in the query.
if time_str is None:
return {'start': (-2**31, 0), 'end':(2**31-1, 999999999)}[role]
# Try parsing the time string as different supported formats.
for fmt in TIME_FORMATS:
try:
dt = datetime.datetime.strptime(time_str, fmt)
break
except ValueError:
continue
else:
raise ValueError('The {0} time argument is not understood. Supported formats are: {1}'.format(role, TIME_FORMATS))
# Convert to the format for the query.
return pb_timestamp.dt_to_carchive(dt)
def parse_time(time_str, role):
# If there is no argument, assume unbounded.
# But we do need to use something in the query.
if time_str is None:
return {'start': (-2**31, 0), 'end': (2**31 - 1, 999999999)}[role]
# Try parsing the time string as different supported formats.
for fmt in TIME_FORMATS:
try:
dt = datetime.datetime.strptime(time_str, fmt)
break
except ValueError:
continue
else:
raise ValueError(
'The {0} time argument is not understood. Supported formats are: {1}'
.format(role, TIME_FORMATS))
# Convert to the format for the query.
return pb_timestamp.dt_to_carchive(dt)
def cmd(archive=None, opt=None, args=None, conf=None, **kws):
archs = opt.archive
# Get out dir.
if opt.export_out_dir is None:
raise PbExportError('Output directory not specified!')
out_dir = opt.export_out_dir
# Get granularity.
if opt.export_granularity is None:
raise PbExportError('Export granularity not specified!')
gran = pb_granularity.get_granularity(opt.export_granularity)
if gran is None:
raise PbExportError('Export granularity is not understood!')
appliance_name = 'appliance0'
if opt.appliance_name is not None:
appliance_name = opt.appliance_name
mysql_write_connected = False
if opt.mysql_write_connected is not None:
mysql_write_connected = True
# Collect PV name delimiters.
delimiters = ([] if opt.export_no_default_delimiters else [':', '-']) + \
([] if opt.export_delimiter is None else opt.export_delimiter)
# Collect PVs to archive...
pvs = set()
# Add explicit PVs.
pvs.update(args)
# Sort PVs.
pvs = sorted(pvs)
# Check we have any PVs.
if len(pvs) == 0:
raise PbExportError('No PVs found to export data!')
# Parse start/end times. These give us the native format for the query.
start_ca_t = parse_time(opt.start, 'start')
end_ca_t = parse_time(opt.end, 'end')
# Print some info.
_log.info('Will export data of these PVs: {0}'.format(', '.join(pvs)))
# Keep PV-specific logs.
pv_logs = []
mysql_writer = pb_mysql.MySqlWriter(out_dir, appliance_name, delimiters,
mysql_write_connected)
# Archive PVs one by one.
for pv in pvs:
_log.info('Exporting data for PV: {0}'.format(pv))
# Create and remember a PvLog object.
pvlog = pb_pvlog.PvLog(pv)
pv_logs.append(pvlog)
# Find the last sample timestamp for this PV.
# This is used as-is as a lower bound filter after the query.
last_timestamp = pb_last.find_last_sample_timestamp(
pv, out_dir, gran, delimiters)
pvlog.info('Last timestamp: {0}'.format(last_timestamp))
# We don't want samples <=last_timestamp, we can't write those out.
# Due to conversion errors, we limit the query conservatively, and filter out any
# initial samples we get that we don't want.
if last_timestamp is not None:
low_limit_dt = pb_timestamp.pb_to_dt(
*last_timestamp) - datetime.timedelta(seconds=1)
query_start_ca_t = max(start_ca_t,
pb_timestamp.dt_to_carchive(low_limit_dt))
else:
query_start_ca_t = start_ca_t
pvlog.info('Query low limit: {0}'.format(query_start_ca_t))
# Create exporter instance.
with pb_exporter.Exporter(pv, gran, out_dir, delimiters,
last_timestamp, pvlog,
mysql_writer) as the_exporter:
try:
# Ask for samples.
segment_data = yield archive.fetchraw(pv,
the_exporter,
archs=archs,
cbArgs=(),
T0=query_start_ca_t,
Tend=end_ca_t,
chunkSize=opt.chunk,
enumAsInt=True,
displayMeta=True,
rawTimes=True)
except pb_exporter.SkipPvError as e:
#report error and continue with the next PV
_log.error('PV ERROR: {0}: {1}'.format(pv, e))
pvlog.error(str(e))
#Write the pv info to mysql
mysql_writer.write_pv_info()
mysql_writer.close()
_log.info('ALL DONE, REPORT FOLLOWS\n')
# Print out logs.
for pvlog in pv_logs:
report = pvlog.build_report()
_log.info(report)
defer.returnValue(0)
def cmd(archive=None, opt=None, args=None, conf=None, **kws):
archs = opt.archive
# Get out dir.
if opt.export_out_dir is None:
raise PbExportError('Output directory not specified!')
out_dir = opt.export_out_dir
# Get granularity.
if opt.export_granularity is None:
raise PbExportError('Export granularity not specified!')
gran = pb_granularity.get_granularity(opt.export_granularity)
if gran is None:
raise PbExportError('Export granularity is not understood!')
appliance_name = 'appliance0'
if opt.appliance_name is not None:
appliance_name = opt.appliance_name
mysql_write_connected = False
if opt.mysql_write_connected is not None:
mysql_write_connected = True
# Collect PV name delimiters.
delimiters = ([] if opt.export_no_default_delimiters else [':', '-']) + \
([] if opt.export_delimiter is None else opt.export_delimiter)
# Collect PVs to archive...
pvs = set()
# Add explicit PVs.
pvs.update(args)
# Sort PVs.
pvs = sorted(pvs)
# Check we have any PVs.
if len(pvs)==0:
raise PbExportError('No PVs found to export data!')
# Parse start/end times. These give us the native format for the query.
start_ca_t = parse_time(opt.start, 'start')
end_ca_t = parse_time(opt.end, 'end')
# Print some info.
_log.info('Will export data of these PVs: {0}'.format(', '.join(pvs)))
# Keep PV-specific logs.
pv_logs = []
mysql_writer = pb_mysql.MySqlWriter(out_dir,appliance_name,delimiters,mysql_write_connected)
# Archive PVs one by one.
for pv in pvs:
_log.info('Exporting data for PV: {0}'.format(pv))
# Create and remember a PvLog object.
pvlog = pb_pvlog.PvLog(pv)
pv_logs.append(pvlog)
# Find the last sample timestamp for this PV.
# This is used as-is as a lower bound filter after the query.
last_timestamp = pb_last.find_last_sample_timestamp(pv, out_dir, gran, delimiters)
pvlog.info('Last timestamp: {0}'.format(last_timestamp))
# We don't want samples <=last_timestamp, we can't write those out.
# Due to conversion errors, we limit the query conservatively, and filter out any
# initial samples we get that we don't want.
if last_timestamp is not None:
low_limit_dt = pb_timestamp.pb_to_dt(*last_timestamp) - datetime.timedelta(seconds=1)
query_start_ca_t = max(start_ca_t, pb_timestamp.dt_to_carchive(low_limit_dt))
else:
query_start_ca_t = start_ca_t
pvlog.info('Query low limit: {0}'.format(query_start_ca_t))
# Create exporter instance.
with pb_exporter.Exporter(pv, gran, out_dir, delimiters, last_timestamp, pvlog, mysql_writer) as the_exporter:
try:
# Ask for samples.
segment_data = yield archive.fetchraw(
pv, the_exporter, archs=archs, cbArgs=(),
T0=query_start_ca_t, Tend=end_ca_t, chunkSize=opt.chunk,
enumAsInt=True, displayMeta=True, rawTimes=True
)
except pb_exporter.SkipPvError as e:
#report error and continue with the next PV
_log.error('PV ERROR: {0}: {1}'.format(pv, e))
pvlog.error(str(e))
#Write the pv info to mysql
mysql_writer.write_pv_info()
mysql_writer.close()
_log.info('ALL DONE, REPORT FOLLOWS\n')
# Print out logs.
for pvlog in pv_logs:
report = pvlog.build_report()
_log.info(report)
defer.returnValue(0)
