def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('broker',
type=str,
help='Hostname or IP and port of Kafka broker.')
parser.add_argument('topic',
type=str,
help='Name of Kafka topic to listen to.')
parser.add_argument('interval', type=int, help='Print every Nth alert.')
parser.add_argument('--group',
type=str,
help='Globally unique name of the consumer group. '
'Consumers in the same group will share messages '
'(i.e., only one consumer will receive a message, '
'as in a queue). Default is value of $HOSTNAME.')
parser.add_argument('--stampDir',
type=str,
help='Output directory for writing postage stamp'
'cutout files. **THERE ARE NO STAMPS RIGHT NOW.**')
args = parser.parse_args()
# Configure consumer connection to Kafka broker
conf = {
'bootstrap.servers': args.broker,
'default.topic.config': {
'auto.offset.reset': 'smallest'
}
}
if args.group:
conf['group.id'] = args.group
else:
conf['group.id'] = os.environ['HOSTNAME']
# Start consumer and print alert stream
with alertConsumer.AlertConsumer(args.topic, **conf) as streamReader:
msg_count = 0
while True:
try:
schema, msg = streamReader.poll()
if msg is None:
continue
else:
msg_count += 1
if msg_count % args.interval == 0:
# Apply filter to each alert
alert_filter(msg, args.stampDir)
except alertConsumer.EopError as e:
# Write when reaching end of partition
sys.stderr.write(e.message)
except IndexError:
sys.stderr.write('%% Data cannot be decoded\n')
except UnicodeDecodeError:
sys.stderr.write('%% Unexpected data format received\n')
except KeyboardInterrupt:
sys.stderr.write('%% Aborted by user\n')
sys.exit()
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('host',
type=str,
help='Hostname or IP of Kafka host to connect to.')
parser.add_argument('topic',
type=str,
help='Name of Kafka topic to listen to.')
parser.add_argument('--group',
type=str,
help='Globally unique name of the consumer group. '
'Consumers in the same group will share messages '
'(i.e., only one consumer will receive a message, '
'as in a queue). Default is value of $HOSTNAME.')
args = parser.parse_args()
# Configure consumer connection to Kafka broker
conf = {
'bootstrap.servers':
'{}:9092,{}:9093,{}:9094'.format(args.host, args.host, args.host),
'default.topic.config': {
'auto.offset.reset': 'smallest'
}
}
if args.group:
conf['group.id'] = args.group
else:
conf['group.id'] = os.environ['HOSTNAME']
# Start consumer and monitor alert stream
with alertConsumer.AlertConsumer(args.topic, **conf) as streamWatcher:
while True:
try:
msg = streamWatcher.poll(decode=False, verbose=False)
if msg is None:
continue
else:
print(msg)
except alertConsumer.EopError as e:
# Write when reaching end of partition
sys.stderr.write(e.message)
except KeyboardInterrupt:
sys.stderr.write('%% Aborted by user\n')
sys.exit()
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('host',
type=str,
help='Hostname or IP of Kafka host to connect to.')
parser.add_argument('topic',
type=str,
help='Name of Kafka topic to listen to.')
parser.add_argument('--group',
type=str,
help='Globally unique name of the consumer group. '
'Consumers in the same group will share messages '
'(i.e., only one consumer will receive a message, '
'as in a queue). Default is value of $HOSTNAME.')
parser.add_argument('--stampDir',
type=str,
help='Output directory for writing postage stamp'
'cutout files.')
avrogroup = parser.add_mutually_exclusive_group()
avrogroup.add_argument('--decode',
dest='avroFlag',
action='store_true',
help='Decode from Avro format. (default)')
avrogroup.add_argument('--decode-off',
dest='avroFlag',
action='store_false',
help='Do not decode from Avro format.')
parser.set_defaults(avroFlag=True)
args = parser.parse_args()
# Configure consumer connection to Kafka broker
conf = {
'bootstrap.servers':
'{}:9092,{}:9093,{}:9094'.format(args.host, args.host, args.host),
'default.topic.config': {
'auto.offset.reset': 'smallest'
}
}
if args.group:
conf['group.id'] = args.group
else:
conf['group.id'] = os.environ['HOSTNAME']
# Configure Avro reader schema
schema_files = [
"../ztf-avro-alert/schema/candidate.avsc",
"../ztf-avro-alert/schema/cutout.avsc",
"../ztf-avro-alert/schema/prv_candidate.avsc",
"../ztf-avro-alert/schema/alert.avsc"
]
# Start consumer and print alert stream
with alertConsumer.AlertConsumer(args.topic, schema_files,
**conf) as streamReader:
while True:
try:
msg = streamReader.poll(decode=args.avroFlag)
if msg is None:
continue
else:
for record in msg:
# Apply filter to each alert
alert_filter(record, args.stampDir)
except alertConsumer.EopError as e:
# Write when reaching end of partition
sys.stderr.write(e.message)
except IndexError:
sys.stderr.write('%% Data cannot be decoded\n')
except UnicodeDecodeError:
sys.stderr.write('%% Unexpected data format received\n')
except KeyboardInterrupt:
sys.stderr.write('%% Aborted by user\n')
sys.exit()
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('topic',
type=str,
help='Name of Kafka topic to listen to.')
parser.add_argument('--group',
type=str,
help='Globally unique name of the consumer group. '
'Consumers in the same group will share messages '
'(i.e., only one consumer will receive a message, '
'as in a queue). Default is value of $HOSTNAME.')
parser.add_argument('--tarName', type=str, help='Name of tar file.')
avrogroup = parser.add_mutually_exclusive_group()
avrogroup.add_argument('--decode',
dest='avroFlag',
action='store_true',
help='Decode from Avro format. (default)')
avrogroup.add_argument('--decode-off',
dest='avroFlag',
action='store_false',
help='Do not decode from Avro format.')
parser.set_defaults(avroFlag=True)
args = parser.parse_args()
# Configure consumer connection to Kafka broker
conf = {
'bootstrap.servers':
'epyc.astro.washington.edu:9092,epyc.astro.washington.edu:9093,epyc.astro.washington.edu:9094',
'default.topic.config': {
'auto.offset.reset': 'smallest'
}
}
if args.group:
conf['group.id'] = args.group
else:
conf['group.id'] = os.environ['HOSTNAME']
# Configure Avro reader schema
schema_files = [
"./ztf-avro-alert/schema/candidate.avsc",
"./ztf-avro-alert/schema/cutout.avsc",
"./ztf-avro-alert/schema/prv_candidate.avsc",
"./ztf-avro-alert/schema/alert.avsc"
]
# Start consumer and collect alerts in a stream
with alertConsumer.AlertConsumer(args.topic, schema_files,
**conf) as streamReader:
with tarfile.open("./" + args.tarName + ".tar", "a") as tar:
while True:
try:
msg = streamReader.poll(decode=args.avroFlag)
if msg is None:
print('currenttime: ', int(strftime('%H')))
if (int(strftime('%H')) >= stopTIME):
print("break \n")
break
else:
print("continue \n")
continue
else:
for record in msg:
#record0 = msg_text(record)
candidate_data = record.get('candidate')
fn = str(candidate_data['candid']) + ".avro"
with io.BytesIO() as avro_file:
record0 = [record]
fastavro.writer(avro_file,
(combineSchemas(schema_files)),
record0)
avro_file.seek(0)
tarinfo = tarfile.TarInfo(name=fn)
tarinfo.size = len(avro_file.getvalue())
tarinfo.mtime = time.time()
tarinfo.mode = 0o744
tarinfo.type = tarfile.REGTYPE
tarfile.uid = tarfile.gid = 0
tarfile.unmae = tarfile.gname = "root"
tar.addfile(tarinfo, avro_file)
#print( "%s \t %8.9f \t %8.5f \t %8.5f \n" % \
# (record.get('objectId'),candidate_data['jd'],candidate_data['ra'],candidate_data['dec']) )
except alertConsumer.EopError as e:
# Write when reaching end of partition
sys.stderr.write(e.message)
#continue
except IndexError:
sys.stderr.write('%% Data cannot be decoded\n')
except UnicodeDecodeError:
sys.stderr.write('%% Unexpected data format received\n')
except KeyboardInterrupt:
sys.stderr.write('%% Aborted by user\n')
break
print('we reached the end of stream at: {}'.format(
strftime("%b %d %Y %H:%M:%S")))
sys.exit()
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('broker', type=str,
help='Hostname or IP and port of Kafka broker.')
parser.add_argument('topic', type=str,
help='Name of Kafka topic to listen to.')
parser.add_argument('filterNum', type=int,
help='Number of the filter in range '
'(1-100) to deploy.')
parser.add_argument('--group', type=str,
help='Globally unique name of the consumer group. '
'Consumers in the same group will share messages '
'(i.e., only one consumer will receive a message, '
'as in a queue). Default is value of $HOSTNAME.')
avrogroup = parser.add_mutually_exclusive_group()
avrogroup.add_argument('--decode', dest='avroFlag', action='store_true',
help='Decode from Avro format. (default)')
avrogroup.add_argument('--decode-off', dest='avroFlag',
action='store_false',
help='Do not decode from Avro format.')
parser.set_defaults(avroFlag=True)
args = parser.parse_args()
fnum = args.filterNum
# Configure Avro reader schema
schema_files = ["../sample-avro-alert/schema/diasource.avsc",
"../sample-avro-alert/schema/diaobject.avsc",
"../sample-avro-alert/schema/ssobject.avsc",
"../sample-avro-alert/schema/cutout.avsc",
"../sample-avro-alert/schema/alert.avsc"]
# Configure consumer connection to Kafka broker
cconf = {'bootstrap.servers': args.broker,
'default.topic.config': {'auto.offset.reset': 'smallest'}}
if args.group:
cconf['group.id'] = args.group
else:
cconf['group.id'] = os.environ['HOSTNAME']
pconf = {'bootstrap.servers': args.broker}
# Choose filter class to deploy from filters module
filter_class = inspect.getmembers(filters, inspect.isclass)[fnum][1]
# Name output stream using filter class name
topic_name = filter_class.__name__
prod = alertProducer.AlertProducer(topic_name, schema_files, **pconf)
exp = filterBase.StreamExporter(prod)
apply_filter = filter_class(exp)
# Start consumer and print alert stream
with alertConsumer.AlertConsumer(args.topic, schema_files,
**cconf) as streamReader:
while True:
try:
msg = streamReader.poll(decode=True)
if msg is None:
continue
else:
# Apply filter to each alert
apply_filter(msg)
except alertConsumer.EopError as e:
# Write when reaching end of partition
sys.stderr.write(e.message)
except IndexError:
sys.stderr.write('%% Data cannot be decoded\n')
except UnicodeDecodeError:
sys.stderr.write('%% Unexpected data format received\n')
except KeyboardInterrupt:
sys.stderr.write('%% Aborted by user\n')
sys.exit()
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('topic',
type=str,
help='Name of Kafka topic to listen to.')
parser.add_argument("--kafka",
type=str,
default="localhost:9092",
help="Address of Kafka server")
parser.add_argument('--group',
type=str,
help='Globally unique name of the consumer group. '
'Consumers in the same group will share messages '
'(i.e., only one consumer will receive a message, '
'as in a queue). Default is value of $HOSTNAME.')
args = parser.parse_args()
# Configure consumer connection to Kafka broker
conf = {
'bootstrap.servers': args.kafka,
'default.topic.config': {
'auto.offset.reset': 'earliest'
}
}
if args.group:
conf['group.id'] = args.group
else:
conf['group.id'] = "cts_test"
redis_conn = redis.StrictRedis("127.0.0.1")
# Configure Avro reader schema
schema_files = [
"../ztf-avro-alert/schema/candidate.avsc",
"../ztf-avro-alert/schema/cutout.avsc",
"../ztf-avro-alert/schema/prv_candidate.avsc",
"../ztf-avro-alert/schema/alert.avsc"
]
# Start consumer and print alert stream
streamReader = alertConsumer.AlertConsumer(args.topic, schema_files,
**conf)
while True:
try:
messages = streamReader.poll(decode=True, timeout=2)
if messages is None:
continue
for msg in messages:
process_alert(msg, redis_conn, date=True)
except alertConsumer.EopError as e:
# Write when reaching end of partition
sys.stderr.write(e.message)
except IndexError:
sys.stderr.write('%% Data cannot be decoded\n')
except UnicodeDecodeError:
sys.stderr.write('%% Unexpected data format received\n')
except KeyboardInterrupt:
sys.stderr.write('%% Aborted by user\n')
sys.exit()
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('topic',
type=str,
help='Name of Kafka topic to listen to.')
parser.add_argument('--group',
type=str,
help='Globally unique name of the consumer group. '
'Consumers in the same group will share messages '
'(i.e., only one consumer will receive a message, '
'as in a queue). Default is value of $HOSTNAME.')
parser.add_argument('--stampDir',
type=str,
help='Output directory for writing postage stamp'
'cutout files.')
avrogroup = parser.add_mutually_exclusive_group()
avrogroup.add_argument('--decode',
dest='avroFlag',
action='store_true',
help='Decode from Avro format. (default)')
avrogroup.add_argument('--decode-off',
dest='avroFlag',
action='store_false',
help='Do not decode from Avro format.')
parser.set_defaults(avroFlag=True)
args = parser.parse_args()
# Configure consumer connection to Kafka broker
conf = {
'bootstrap.servers': 'kafka:9092',
'default.topic.config': {
'auto.offset.reset': 'smallest'
}
}
if args.group:
conf['group.id'] = args.group
else:
conf['group.id'] = os.environ['HOSTNAME']
# Configure Avro reader schema
schema_files = [
"../sample-avro-alert/schema/diasource.avsc",
"../sample-avro-alert/schema/diaobject.avsc",
"../sample-avro-alert/schema/ssobject.avsc",
"../sample-avro-alert/schema/cutout.avsc",
"../sample-avro-alert/schema/alert.avsc"
]
# Start consumer and print alert stream
streamReader = alertConsumer.AlertConsumer(args.topic, schema_files,
**conf)
msg_count = 0
while True:
try:
msg = streamReader.poll(decode=args.avroFlag)
if msg is None:
continue
else:
msg_count += 1
if msg_count % 100 == 0:
print(msg_count, flush=True),
print(msg_text(msg), flush=True)
if args.stampDir: # Collect postage stamps
write_stamp_file(msg.get('cutoutDifference'),
args.stampDir)
write_stamp_file(msg.get('cutoutTemplate'), args.stampDir)
except alertConsumer.EopError as e:
# Write when reaching end of partition
sys.stderr.write(e.message)
except IndexError:
sys.stderr.write('%% Data cannot be decoded\n')
except UnicodeDecodeError:
sys.stderr.write('%% Unexpected data format received\n')
except KeyboardInterrupt:
sys.stderr.write('%% Aborted by user\n')
sys.exit()