def main():
config = ConfigParser.ConfigParser()
filename = config.read(["shavar_list_creation.ini"])
if not filename:
sys.stderr.write("Error loading shavar_list_creation.ini\n")
sys.exit(-1)
chunknum = int(time.time())
for section in config.sections():
if section == "main":
continue
if (section in PRE_DNT_SECTIONS or section in DNT_SECTIONS):
output_file, log_file = get_tracker_lists(config, section,
chunknum)
if section in PLUGIN_SECTIONS:
# load the plugin blocklist
blocked = set()
blocklist_url = config.get(section, "blocklist")
if blocklist_url:
for line in urllib2.urlopen(blocklist_url).readlines():
line = line.strip()
# don't add blank lines or comments
if not line or line.startswith('#'):
continue
blocked.add(line)
output_file, log_file = get_output_and_log_files(config, section)
process_plugin_blocklist(blocked, chunknum, output_file, log_file,
section)
if section in WHITELIST_SECTIONS:
output_file, log_file = get_entity_lists(config, section, chunknum)
if output_file:
output_file.close()
if log_file:
log_file.close()
publish_to_cloud(config, chunknum)
# create and publish versioned lists
resp = requests.get(GITHUB_API_URL + SHAVAR_PROD_LISTS_BRANCHES_PATH)
if resp:
shavar_prod_lists_branches = resp.json()
start_versioning(config, chunknum, shavar_prod_lists_branches)
else:
print('\n\n*** Unable to get branches from shavar-prod-lists repo ***')
def start_versioning(config, chunknum, shavar_prod_lists_branches):
for branch in shavar_prod_lists_branches:
branch_name = branch.get('name')
ver = p_version.parse(branch_name)
if isinstance(ver, p_version.Version):
print('\n\n*** Start Versioning for {ver} ***'.format(
ver=branch_name))
get_versioned_lists(config, chunknum, version=branch_name)
print('\n*** Publish Versioned Lists ***')
publish_to_cloud(config, chunknum, check_versioning=True)
print('\n*** Revert Configs ***')
revert_config(config, branch_name)
else:
print('\n\n*** {branch} is not a versioning branch ***'.format(
branch=branch_name))