def sync(source):
start_time = time.time()
sys.stdout.write("[+] Staring to sync source: {label}!\n".format(
label=source.label))
source_service.sync(source)
sys.stdout.write(
"[+] Finished syncing source: {label}. Run Time: {time}\n".format(
label=source.label, time=(time.time() - start_time)))
def sync(source):
start_time = time.time()
sys.stdout.write("[+] Staring to sync source: {label}!\n".format(label=source.label))
source_service.sync(source)
sys.stdout.write(
"[+] Finished syncing source: {label}. Run Time: {time}\n".format(
label=source.label,
time=(time.time() - start_time)
)
)
def sync_sources(labels):
"""
Attempts to run several methods Certificate discovery. This is
run on a periodic basis and updates the Lemur datastore with the
information it discovers.
"""
if not labels:
sys.stdout.write("Active\tLabel\tDescription\n")
for source in source_service.get_all():
sys.stdout.write(
"{active}\t{label}\t{description}!\n".format(
label=source.label,
description=source.description,
active=source.active
)
)
else:
start_time = time.time()
lock_file = "/tmp/.lemur_lock"
sync_lock = LockFile(lock_file)
while not sync_lock.i_am_locking():
try:
sync_lock.acquire(timeout=10) # wait up to 10 seconds
sys.stdout.write("[+] Staring to sync sources: {labels}!\n".format(labels=labels))
labels = labels.split(",")
if labels[0] == 'all':
sync()
else:
sync(labels=labels)
sys.stdout.write(
"[+] Finished syncing sources. Run Time: {time}\n".format(
time=(time.time() - start_time)
)
)
except LockTimeout:
sys.stderr.write(
"[!] Unable to acquire file lock on {file}, is there another sync running?\n".format(
file=lock_file
)
)
sync_lock.break_lock()
sync_lock.acquire()
sync_lock.release()
sync_lock.release()
def sync_sources(labels):
"""
Attempts to run several methods Certificate discovery. This is
run on a periodic basis and updates the Lemur datastore with the
information it discovers.
"""
if not labels:
sys.stdout.write("Active\tLabel\tDescription\n")
for source in source_service.get_all():
sys.stdout.write(
"{active}\t{label}\t{description}!\n".format(
label=source.label,
description=source.description,
active=source.active
)
)
else:
start_time = time.time()
lock_file = "/tmp/.lemur_lock"
sync_lock = LockFile(lock_file)
while not sync_lock.i_am_locking():
try:
sync_lock.acquire(timeout=10) # wait up to 10 seconds
sys.stdout.write("[+] Staring to sync sources: {labels}!\n".format(labels=labels))
labels = labels.split(",")
if labels[0] == 'all':
sync()
else:
sync(labels=labels)
sys.stdout.write(
"[+] Finished syncing sources. Run Time: {time}\n".format(
time=(time.time() - start_time)
)
)
except LockTimeout:
sys.stderr.write(
"[!] Unable to acquire file lock on {file}, is there another sync running?\n".format(
file=lock_file
)
)
sync_lock.break_lock()
sync_lock.acquire()
sync_lock.release()
sync_lock.release()
def sync(source_strings):
source_objs = validate_sources(source_strings)
for source in source_objs:
start_time = time.time()
print("[+] Staring to sync source: {label}!\n".format(
label=source.label))
user = user_service.get_by_username('lemur')
try:
data = source_service.sync(source, user)
print("[+] Certificates: New: {new} Updated: {updated}".format(
new=data['certificates'][0], updated=data['certificates'][1]))
print("[+] Endpoints: New: {new} Updated: {updated}".format(
new=data['endpoints'][0], updated=data['endpoints'][1]))
print("[+] Finished syncing source: {label}. Run Time: {time}".
format(label=source.label, time=(time.time() - start_time)))
except Exception as e:
current_app.logger.exception(e)
print("[X] Failed syncing source {label}!\n".format(
label=source.label))
metrics.send('sync_failed',
'counter',
1,
metric_tags={'source': source.label})
def sync(source_strings):
sources = validate_sources(source_strings)
for source in sources:
status = FAILURE_METRIC_STATUS
start_time = time.time()
print("[+] Staring to sync source: {label}!\n".format(
label=source.label))
user = user_service.get_by_username("lemur")
try:
data = source_service.sync(source, user)
print("[+] Certificates: New: {new} Updated: {updated}".format(
new=data["certificates"][0], updated=data["certificates"][1]))
print(
"[+] Endpoints: New: {new} Updated: {updated} Removed: {removed}"
.format(
new=data["endpoints"][0],
updated=data["endpoints"][1],
removed=data["endpoints"][2],
))
print("[+] Finished syncing source: {label}. Run Time: {time}".
format(label=source.label, time=(time.time() - start_time)))
status = SUCCESS_METRIC_STATUS
except Exception as e:
current_app.logger.exception(e)
print("[X] Failed syncing source {label}!\n".format(
label=source.label))
sentry.captureException()
metrics.send(
"source_sync_fail",
"counter",
1,
metric_tags={
"source": source.label,
"status": status
},
)
metrics.send(
"source_sync",
"counter",
1,
metric_tags={
"source": source.label,
"status": status
},
)
def sync(source_strings):
source_objs = validate_sources(source_strings)
for source in source_objs:
start_time = time.time()
print("[+] Staring to sync source: {label}!\n".format(
label=source.label))
user = user_service.get_by_username('lemur')
try:
source_service.sync(source, user)
print("[+] Finished syncing source: {label}. Run Time: {time}".
format(label=source.label, time=(time.time() - start_time)))
except Exception as e:
current_app.logger.exception(e)
print("[X] Failed syncing source {label}!\n".format(
label=source.label))
metrics.send('sync_failed',
'counter',
1,
metric_tags={'source': source.label})
def sync(source_strings):
sources = validate_sources(source_strings)
for source in sources:
status = FAILURE_METRIC_STATUS
start_time = time.time()
print("[+] Staring to sync source: {label}!\n".format(label=source.label))
user = user_service.get_by_username('lemur')
try:
data = source_service.sync(source, user)
print(
"[+] Certificates: New: {new} Updated: {updated}".format(
new=data['certificates'][0],
updated=data['certificates'][1]
)
)
print(
"[+] Endpoints: New: {new} Updated: {updated}".format(
new=data['endpoints'][0],
updated=data['endpoints'][1]
)
)
print(
"[+] Finished syncing source: {label}. Run Time: {time}".format(
label=source.label,
time=(time.time() - start_time)
)
)
status = SUCCESS_METRIC_STATUS
except Exception as e:
current_app.logger.exception(e)
print(
"[X] Failed syncing source {label}!\n".format(label=source.label)
)
sentry.captureException()
metrics.send('source_sync_fail', 'counter', 1, metric_tags={'source': source.label, 'status': status})
metrics.send('source_sync', 'counter', 1, metric_tags={'source': source.label, 'status': status})
def sync_source(source_label):
"""
This celery task will sync the specified source.
:param source:
:return:
"""
function = f"{__name__}.{sys._getframe().f_code.co_name}"
logger = logging.getLogger(function)
task_id = None
if celery.current_task:
task_id = celery.current_task.request.id
log_data = {
"source": source_label,
"task_id": task_id,
}
if task_id and is_task_active(function, task_id, (source_label, )):
logger.debug("Skipping task: Task is already active", extra=log_data)
return
logger.info("Starting syncing source", extra=log_data)
user = user_service.get_by_username("lemur")
source = source_service.get_by_label(source_label)
if not source:
raise RuntimeError(f"Source {source_label} not found")
result = source_service.sync(source, user)
log_data["result"] = result
logger.info("Done syncing source", extra=log_data)
return