def lock(self):
"""
Trys to write a lock file. Returns True if we get the lock, else return
False.
"""
try:
if not os.path.exists(os.path.dirname(self.LOCK_FILE)):
os.makedirs(os.path.dirname(self.LOCK_FILE))
self.log("Attempting acquire mutext "
"({0})...".format(self.LOCK_FILE))
self.lock_fd = open(self.LOCK_FILE, 'w+')
fcntl.flock(self.lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
self.log(self.format_title("Mutex Acquired"))
return True
except IOError, exc_value:
self.lock_fd = None
# IOError: [Errno 11] Resource temporarily unavailable
if exc_value[0] == 11:
self.log(
"DNS build script attempted to acquire the "
"build mutux but another process already has it."
)
fail_mail(
"An attempt was made to start the DNS build script "
"while an instance of the script was already running. "
"The attempt was denied.",
subject="Concurrent DNS builds attempted.")
return False
else:
raise
def main():
parser = argparse.ArgumentParser(description='DNS Build scripts')
parser.add_argument('--stage-only', dest='STAGE_ONLY', action='store_true',
default=False, help="Just build staging and don't "
"copy to prod. named-checkzone will still be run.")
parser.add_argument('--clobber-stage', dest='CLOBBER_STAGE',
action='store_true', default=False, help="If stage "
"already exists delete it before running the build "
"script.")
parser.add_argument('--ship-it', dest='PUSH_TO_PROD',
action='store_true', default=False, help="Check files "
"into rcs and push upstream.")
parser.add_argument('--preserve-stage', dest='PRESERVE_STAGE',
action='store_true', default=False, help="Do not "
"remove staging area after build completes.")
parser.add_argument('--no-build', dest='BUILD_ZONES',
action='store_false', default=True, help="Do not "
"build zone files.")
parser.add_argument('--no-syslog', dest='LOG_SYSLOG',
action='store_false', default=True, help="Do not "
"log to syslog.")
parser.add_argument('--debug', dest='DEBUG',
action='store_true', default=False, help="Print "
"copious amounts of text.")
parser.add_argument('--force', dest='FORCE',
action='store_true', default=False, help="Ignore "
"all change delta thresholds, clobber stagig area, "
"and build even when no tasks are scheduled.")
parser.add_argument('--status', dest='STATUS',
action='store_true', default=False, help="Display "
"info about the build configuration and exit.")
nas = parser.parse_args(sys.argv[1:])
b = DNSBuilder(**dict(nas._get_kwargs()))
if nas.STATUS:
b.status()
return
message = "DNS Build Error. Error: '{0}'. The build was unsuccessful."
def write_stop_update(error):
if os.path.exists(STOP_UPDATE_FILE):
return
with open(STOP_UPDATE_FILE, 'w+') as fd:
msg = ("This file was placed here because there was an error:\n"
"=============== ERROR MESSAGE ======================+\n")
fd.write(msg)
fd.write(error)
try:
with open(LAST_RUN_FILE, 'w+') as fd:
fd.write(str(int(time.time())))
b.build_dns()
except BuildError as why:
b.log(why, log_level='LOG_ERR')
write_stop_update(str(why))
fail_mail(message.format(why))
except Exception as err:
fail_mail(message.format(err))
b.log(err, log_level='LOG_CRIT')
error_msg = "{0}\n{1}".format(str(err), traceback.format_exc())
write_stop_update(error_msg)
def lock(self):
"""
Trys to write a lock file. Returns True if we get the lock, else return
False.
"""
try:
if not os.path.exists(os.path.dirname(self.LOCK_FILE)):
os.makedirs(os.path.dirname(self.LOCK_FILE))
self.log("Attempting acquire mutext "
"({0})...".format(self.LOCK_FILE))
self.lock_fd = open(self.LOCK_FILE, 'w+')
fcntl.flock(self.lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
self.log(self.format_title("Mutex Acquired"))
return True
except IOError, exc_value:
self.lock_fd = None
# IOError: [Errno 11] Resource temporarily unavailable
if exc_value[0] == 11:
self.log("DNS build script attempted to acquire the "
"build mutux but another process already has it.")
fail_mail(
"An attempt was made to start the DNS build script "
"while an instance of the script was already running. "
"The attempt was denied.",
subject="Concurrent DNS builds attempted.")
return False
else:
raise
def build_zone_data(view, root_domain, soa, logf=None):
"""
This function does the heavy lifting of building a zone. It coordinates
getting all of the data out of the db into BIND format.
:param soa: The SOA corresponding to the zone being built.
:type soa: SOA
:param root_domain: The root domain of this zone.
:type root_domain: str
:returns public_file_path: The path to the zone file in the STAGEING dir
:type public_file_path: str
:returns public_data: The data that should be written to public_file_path
:type public_data: str
:returns view_zone_file: The path to the zone file in the STAGEING dir
:type view_zone_file: str
:param view_data: The data that should be written to view_zone_file
:type view_data: str
"""
ztype = "reverse" if root_domain.is_reverse else "forward"
if soa.has_record_set(view=view, exclude_ns=True) and not root_domain.nameserver_set.filter(views=view).exists():
msg = (
"The {0} zone has a records in the {1} view, but there are "
"no nameservers in that view. A zone file for {1} won't be "
"built. Use the search string 'zone=:{0} view=:{1}' to find "
"the troublesome records".format(root_domain, view.name)
)
fail_mail(msg, subject="Shitty edge case detected.")
logf("LOG_WARNING", msg)
return ""
domains = soa.domain_set.all().order_by("name")
# Bulid the mega filter!
domain_mega_filter = Q(domain=root_domain)
for domain in domains:
domain_mega_filter = domain_mega_filter | Q(domain=domain)
rdomain_mega_filter = Q(reverse_domain=root_domain)
for reverse_domain in domains:
rdomain_mega_filter = rdomain_mega_filter | Q(reverse_domain=reverse_domain)
soa_data = render_soa_only(soa=soa, root_domain=root_domain)
try:
if ztype == "forward":
view_data = render_forward_zone(view, domain_mega_filter)
else:
view_data = render_reverse_zone(view, domain_mega_filter, rdomain_mega_filter)
except View.DoesNotExist:
view_data = ""
if view_data:
view_data = soa_data + view_data
return view_data
def stop_update_exists(self):
"""
Look for a file referenced by `STOP_UPDATE_FILE` and if it exists,
cancel the build.
"""
if os.path.exists(self.STOP_UPDATE_FILE):
msg = ("The STOP_UPDATE_FILE ({0}) exists. Build canceled. \n"
"Reason for skipped build: \n"
"{1}".format(self.STOP_UPDATE_FILE,
open(self.STOP_UPDATE_FILE).read()))
fail_mail(msg, subject="DNS builds have stoped")
self.log(msg)
return True
else:
return False
def stop_update_exists(self):
"""
Look for a file referenced by `STOP_UPDATE_FILE` and if it exists,
cancel the build.
"""
if os.path.exists(self.STOP_UPDATE_FILE):
msg = ("The STOP_UPDATE_FILE ({0}) exists. Build canceled. \n"
"Reason for skipped build: \n"
"{1}".format(self.STOP_UPDATE_FILE,
open(self.STOP_UPDATE_FILE).read()))
fail_mail(msg, subject="DNS builds have stoped")
self.log(msg)
return True
else:
return False
def build_zone_data(view, root_domain, soa, logf=None):
"""
This function does the heavy lifting of building a zone. It coordinates
getting all of the data out of the db into BIND format.
:param soa: The SOA corresponding to the zone being built.
:type soa: SOA
:param root_domain: The root domain of this zone.
:type root_domain: str
"""
ztype = 'reverse' if root_domain.is_reverse else 'forward'
if (soa.has_record_set(view=view, exclude_ns=True) and
not root_domain.nameserver_set.filter(views=view).exists()):
msg = ("The {0} zone has a records in the {1} view, but there are "
"no nameservers in that view. Use the search string 'zone=:{0} "
"view=:{1}' to find the troublesome records".format(
root_domain, view.name)
)
fail_mail(msg, subject="Shitty edge case detected.")
logf('LOG_WARNING', msg)
domains = soa.domain_set.all().order_by('name')
# Bulid the mega filter!
domain_mega_filter = Q(domain=root_domain)
for domain in domains:
domain_mega_filter = domain_mega_filter | Q(domain=domain)
rdomain_mega_filter = Q(reverse_domain=root_domain)
for reverse_domain in domains:
rdomain_mega_filter = rdomain_mega_filter | Q(
reverse_domain=reverse_domain)
soa_data = render_soa_only(soa=soa, root_domain=root_domain)
try:
if ztype == "forward":
view_data = render_forward_zone(view, domain_mega_filter)
else:
view_data = render_reverse_zone(view, domain_mega_filter,
rdomain_mega_filter)
except View.DoesNotExist:
view_data = ""
if view_data:
view_data = soa_data + view_data
return view_data
def build_zone_data(view, root_domain, soa, logf=None):
"""
This function does the heavy lifting of building a zone. It coordinates
getting all of the data out of the db into BIND format.
:param soa: The SOA corresponding to the zone being built.
:type soa: SOA
:param root_domain: The root domain of this zone.
:type root_domain: str
"""
ztype = 'reverse' if root_domain.is_reverse else 'forward'
if (soa.has_record_set(view=view, exclude_ns=True)
and not root_domain.nameserver_set.filter(views=view).exists()):
msg = ("The {0} zone has a records in the {1} view, but there are "
"no nameservers in that view. Use the search string 'zone=:{0} "
"view=:{1}' to find the troublesome records".format(
root_domain, view.name))
fail_mail(msg, subject="Shitty edge case detected.")
logf('LOG_WARNING', msg)
domains = soa.domain_set.all().order_by('name')
# Bulid the mega filter!
domain_mega_filter = Q(domain=root_domain)
for domain in domains:
domain_mega_filter = domain_mega_filter | Q(domain=domain)
rdomain_mega_filter = Q(reverse_domain=root_domain)
for reverse_domain in domains:
rdomain_mega_filter = rdomain_mega_filter | Q(
reverse_domain=reverse_domain)
soa_data = render_soa_only(soa=soa, root_domain=root_domain)
try:
if ztype == "forward":
view_data = render_forward_zone(view, domain_mega_filter)
else:
view_data = render_reverse_zone(view, domain_mega_filter,
rdomain_mega_filter)
except View.DoesNotExist:
view_data = ""
if view_data:
view_data = soa_data + view_data
return view_data
scope.delete()
except IndexError:
scope.delete()
if len(dhcp_scopes) > 0 or always_push_svn:
os.chdir(output_dir)
os.system('/usr/bin/svn update')
os.system('/usr/bin/svn add * --force')
os.system(
'/usr/bin/svn commit -m "Autogenerated addition from inventory"')
#os.system('/usr/bin/git push origin master')
def write_stop_update(error):
if os.path.exists(STOP_UPDATE_FILE):
return
with open(STOP_UPDATE_FILE, 'w+') as fd:
msg = ("This file was placed here because there was an error:\n"
"=============== ERROR MESSAGE ======================+\n")
fd.write(msg)
fd.write(error)
if __name__ == '__main__':
try:
main()
except Exception as err:
message = "DHCP Build Error. Error: '{0}'. The build was unsuccessful."
fail_mail(message.format(err))
error_msg = "{0}\n{1}".format(str(err), traceback.format_exc())
write_stop_update(error_msg)
d = DHCPFile(dhcp_scope=dhcp_scope,file_text=output_text)
d.save()
scope.delete()
except IndexError:
scope.delete()
if len(dhcp_scopes) > 0 or always_push_svn:
os.chdir(output_dir)
os.system('/usr/bin/svn update')
os.system('/usr/bin/svn add * --force')
os.system('/usr/bin/svn commit -m "Autogenerated addition from inventory"')
#os.system('/usr/bin/git push origin master')
def write_stop_update(error):
if os.path.exists(STOP_UPDATE_FILE):
return
with open(STOP_UPDATE_FILE, 'w+') as fd:
msg = ("This file was placed here because there was an error:\n"
"=============== ERROR MESSAGE ======================+\n")
fd.write(msg)
fd.write(error)
if __name__ == '__main__':
try:
main()
except Exception as err:
message = "DHCP Build Error. Error: '{0}'. The build was unsuccessful."
fail_mail(message.format(err))
error_msg = "{0}\n{1}".format(str(err), traceback.format_exc())
write_stop_update(error_msg)
def main():
parser = argparse.ArgumentParser(description='DNS Build scripts')
parser.add_argument('--stage-only',
dest='STAGE_ONLY',
action='store_true',
default=False,
help="Just build staging and don't "
"copy to prod. named-checkzone will still be run.")
parser.add_argument('--clobber-stage',
dest='CLOBBER_STAGE',
action='store_true',
default=False,
help="If stage "
"already exists delete it before running the build "
"script.")
parser.add_argument('--ship-it',
dest='PUSH_TO_PROD',
action='store_true',
default=False,
help="Check files "
"into rcs and push upstream.")
parser.add_argument('--preserve-stage',
dest='PRESERVE_STAGE',
action='store_true',
default=False,
help="Do not "
"remove staging area after build completes.")
parser.add_argument('--no-build',
dest='BUILD_ZONES',
action='store_false',
default=True,
help="Do not "
"build zone files.")
parser.add_argument('--no-syslog',
dest='LOG_SYSLOG',
action='store_false',
default=True,
help="Do not "
"log to syslog.")
parser.add_argument('--debug',
dest='DEBUG',
action='store_true',
default=False,
help="Print "
"copious amounts of text.")
parser.add_argument('--force-checkin',
dest='FORCE',
action='store_true',
default=False,
help="Ignore "
"all change delta thresholds, clobber stagig area, "
"and build even when no tasks are scheduled.")
parser.add_argument('--force-build',
dest='FORCE_BUILD',
action='store_true',
default=False,
help="Do a buil "
"even if no zones are requesting to be rebuilt.")
parser.add_argument('--status',
dest='STATUS',
action='store_true',
default=False,
help="Display "
"info about the build configuration and exit.")
nas = parser.parse_args(sys.argv[1:])
b = DNSBuilder(**dict(nas._get_kwargs()))
if nas.STATUS:
b.status()
return
message = "DNS Build Error. Error: '{0}'. The build was unsuccessful."
def write_stop_update(error):
if os.path.exists(STOP_UPDATE_FILE):
return
with open(STOP_UPDATE_FILE, 'w+') as fd:
msg = ("This file was placed here because there was an error:\n"
"=============== ERROR MESSAGE ======================+\n")
fd.write(msg)
fd.write(error)
try:
with open(LAST_RUN_FILE, 'w+') as fd:
fd.write(str(int(time.time())))
b.build_dns()
except BuildError as why:
b.log(why, log_level='LOG_ERR')
write_stop_update(str(why))
fail_mail(message.format(why))
except Exception as err:
fail_mail(message.format(err))
b.log(err, log_level='LOG_CRIT')
error_msg = "{0}\n{1}".format(str(err), traceback.format_exc())
write_stop_update(error_msg)
def build_zone_files(self, soas, soa_pks_to_rebuild, gen_config=True):
"""
For every soa in soas, build a views that are valid for its zone. If
gen_config is true, generate a config statement for a view and store it
in zone_stmts (returned by this function). When an soa's pk is in
`soa_pks_to_rebuild` calculate a new serial for the zone.
:param soas: The soa objects that will have config statments made
with gen_config is True
:param soa_pks_to_rebuild: The primary keys of soa objects that have
been flagged as needing to be rebuilt. These soas will have their
serial numbers recalculated.
:param gen_config: Defaults to True, this parameter decides whether
or not zone_statements will be be calculated for the soa objects
found in the `soas` parameter
:returns zone_stmts: A dictionary mapping zone names to config
statements
"""
zone_stmts = {}
for soa in soas:
# If anything happens during this soa's build we need to mark
# it as dirty so it can be rebuild
try:
root_domain = soa.root_domain # This is an expensive lookup
if not root_domain:
# TODO, figure out how to send a nagios alert on this case.
self.log("No root domain found in zone {0}".format(soa))
fail_mail("{0} is an orphan. It's being skipped in the "
"builds".format(soa),
subject="{0} is "
"orphaned".format(soa))
continue
force_rebuild = soa.pk in soa_pks_to_rebuild or soa.dirty
if force_rebuild and self.PUSH_TO_PROD:
soa.dirty = False
soa.save()
self.log("====== Processing {0} {1} ======".format(
root_domain, soa.serial))
view_contents = self.build_views(soa, root_domain, gen_config,
force_rebuild)
next_serial = soa.get_incremented_serial()
if force_rebuild:
# Bypass save so we don't have to save a possible stale
# 'dirty' value to the db.
SOA.objects.filter(pk=soa.pk).update(serial=next_serial)
self.log("Zone will be rebuilt at serial {0}".format(
next_serial),
root_domain=root_domain)
else:
self.log("Zone is stable at serial {0}".format(soa.serial),
root_domain=root_domain)
self.do_build(zone_stmts, soa, view_contents, root_domain,
gen_config, force_rebuild, next_serial)
except Exception:
soa.schedule_full_rebuild()
raise
return zone_stmts
def build_zone_files(self, soa_pks_to_rebuild):
zone_stmts = {}
for soa in SOA.objects.all():
# If anything happens during this soa's build we need to mark
# it as dirty so it can be rebuild
try:
root_domain = soa.root_domain # This is an expensive lookup
if not root_domain:
# TODO, figure out how to send a nagios alert on this case.
self.log("No root domain found in zone {0}".format(soa))
fail_mail(
"{0} is an orphan. It's being skipped in "
"the builds".format(soa), subject="{0} is "
"orphaned".format(soa))
continue
"""
General order of things:
* Find which views should have a zone file built and add them
to a list.
* If any of the view's zone file have been tampered with or
the zone is new, trigger the rebuilding of all the zone's
view files. (rebuil all views in a zone keeps the serial
synced across all views)
* Either rebuild all of a zone's view files because one view
needed to be rebuilt due to tampering or the zone was dirty
(again, this is to keep their serial synced) or just call
named-checkzone on the existing zone files for good measure.
Also generate a zone statement and add it to a dictionary for
later use during BIND configuration generation.
"""
force_rebuild = soa.pk in soa_pks_to_rebuild or soa.dirty
if force_rebuild:
soa.dirty = False
soa.save()
self.log('====== Processing {0} {1} ======'.format(
root_domain, soa.serial)
)
views_to_build = []
self.log(
"SOA was seen with dirty == {0}".format(force_rebuild),
root_domain=root_domain
)
# This for loop decides which views will be canidates for
# rebuilding.
for view in View.objects.all():
self.log("++++++ Looking at < {0} > view ++++++".
format(view.name), root_domain=root_domain)
t_start = time.time() # tic
view_data = build_zone_data(view, root_domain, soa,
logf=self.log)
build_time = time.time() - t_start # toc
self.log('< {0} > Built {1} data in {2} seconds'
.format(view.name, soa, build_time),
root_domain=root_domain, build_time=build_time)
if not view_data:
self.log('< {0} > No data found in this view. '
'No zone file will be made or included in any'
' config for this view.'.format(view.name),
root_domain=root_domain)
continue
self.log('< {0} > Non-empty data set for this '
'view. Its zone file will be included in the '
'config.'.format(view.name),
root_domain=root_domain)
file_meta = self.get_file_meta(view, root_domain, soa)
was_bad_prev, safe_serial = self.verify_previous_build(
file_meta, view, root_domain, soa
)
if was_bad_prev:
soa.serial = safe_serial
force_rebuild = True
views_to_build.append(
(view, file_meta, view_data)
)
self.log(
'----- Building < {0} > ------'.format(
' | '.join([v.name for v, _, _ in views_to_build])
), root_domain=root_domain
)
next_serial = soa.get_incremented_serial()
if force_rebuild:
# Bypass save so we don't have to save a possible stale
# 'dirty' value to the db.
SOA.objects.filter(pk=soa.pk).update(serial=next_serial)
self.log('Zone will be rebuilt at serial {0}'
.format(next_serial), root_domain=root_domain)
else:
self.log('Zone is stable at serial {0}'
.format(soa.serial), root_domain=root_domain)
for view, file_meta, view_data in views_to_build:
if (root_domain.name, view.name) in ZONES_WITH_NO_CONFIG:
self.log(
'!!! Not going to emit zone statements for '
'{0}\n'.format(root_domain.name),
root_domain=root_domain
)
else:
view_zone_stmts = zone_stmts.setdefault(view.name, [])
# If we see a view in this loop it's going to end up in
# the config
view_zone_stmts.append(
self.render_zone_stmt(soa, root_domain, file_meta)
)
# If it's dirty or we are rebuilding another view, rebuild
# the zone
if force_rebuild:
self.log(
'Rebuilding < {0} > view file {1}'
.format(view.name, file_meta['prod_fname']),
root_domain=root_domain)
prod_fname = self.build_zone(
view, file_meta,
# Lazy string evaluation
view_data.format(serial=next_serial),
root_domain
)
assert prod_fname == file_meta['prod_fname']
else:
self.log(
'NO REBUILD needed for < {0} > view file {1}'
.format(view.name, file_meta['prod_fname']),
root_domain=root_domain
)
# run named-checkzone for good measure
if self.STAGE_ONLY:
self.log("Not calling named-checkconf.",
root_domain=root_domain)
else:
self.named_checkzone(
file_meta['prod_fname'], root_domain
)
except Exception:
soa.schedule_rebuild()
raise
return zone_stmts
def build_zone_files(self, soas, soa_pks_to_rebuild, gen_config=True):
"""
For every soa in soas, build a views that are valid for its zone. If
gen_config is true, generate a config statement for a view and store it
in zone_stmts (returned by this function). When an soa's pk is in
`soa_pks_to_rebuild` calculate a new serial for the zone.
:param soas: The soa objects that will have config statments made
with gen_config is True
:param soa_pks_to_rebuild: The primary keys of soa objects that have
been flagged as needing to be rebuilt. These soas will have their
serial numbers recalculated.
:param gen_config: Defaults to True, this parameter decides whether
or not zone_statements will be be calculated for the soa objects
found in the `soas` parameter
:returns zone_stmts: A dictionary mapping zone names to config
statements
"""
zone_stmts = {}
for soa in soas:
# If anything happens during this soa's build we need to mark
# it as dirty so it can be rebuild
try:
root_domain = soa.root_domain # This is an expensive lookup
if not root_domain:
# TODO, figure out how to send a nagios alert on this case.
self.log("No root domain found in zone {0}".format(soa))
fail_mail(
"{0} is an orphan. It's being skipped in the "
"builds".format(soa), subject="{0} is "
"orphaned".format(soa))
continue
force_rebuild = soa.pk in soa_pks_to_rebuild or soa.dirty
if force_rebuild and self.PUSH_TO_PROD:
soa.dirty = False
soa.save()
self.log("====== Processing {0} {1} ======".format(
root_domain, soa.serial)
)
view_contents = self.build_views(
soa, root_domain, gen_config, force_rebuild
)
next_serial = soa.get_incremented_serial()
if force_rebuild:
# Bypass save so we don't have to save a possible stale
# 'dirty' value to the db.
SOA.objects.filter(pk=soa.pk).update(serial=next_serial)
self.log(
"Zone will be rebuilt at serial {0}"
.format(next_serial),
root_domain=root_domain
)
else:
self.log(
"Zone is stable at serial {0}".format(soa.serial),
root_domain=root_domain
)
self.do_build(
zone_stmts, soa, view_contents, root_domain,
gen_config, force_rebuild, next_serial
)
except Exception:
soa.schedule_full_rebuild()
raise
return zone_stmts
