def approove():
from uuid import uuid4
id = uuid4()
settings.LOGGER.info("Cron %s touched. Approoving" % id)
try:
with Lock(name="approove_lock"):
for ad in Advert.objects.filter(interval__gt=0).filter(
status=Advert.SENT):
find = False
for box in Mailbox.objects.filter(active=True):
settings.LOGGER.info(
"Cron %s. Checking links for %s in box %s" %
(id, ad.id, box.login))
links = get_links(box.server, box.login, box.password,
ad.text)
if links is not None:
ad.aproove(links)
find = True
settings.LOGGER.info(
"Cron %s. Checking links for %s complete" % (id, ad.id))
if not find:
settings.LOGGER.warn("Cron %s. Did not find links for %s" %
(id, ad.id))
dt = timezone.now() - ad.status_changed
if dt > timezone.timedelta(minutes=ad.interval):
notify_fail(ad)
ad.remove(id)
except CannotAcquireLock:
settings.LOGGER.info(
"Cron %s. Another approoving cron working. Exiting" % id)
settings.LOGGER.info("Cron %s finished" % id)
def send():
from uuid import uuid4
id = uuid4()
settings.LOGGER.info("Cron %s touched. Sending" % id)
was_action = False
try:
with Lock(name="send_lock"):
for ad in Advert.objects.filter(interval__gt=0):
settings.LOGGER.info("Cron %s. Checking ad %s" % (id, ad.id))
if was_action:
sleep(150)
was_action = False
ad.refresh_from_db()
if ad.status_changed is None:
ad.send(id)
was_action = True
continue
if ad.status == ad.WAITING:
ad.send(id)
was_action = True
continue
if ad.last_post is not None and (ad.status != ad.SENT):
dt = timezone.now() - ad.last_post
if dt > timezone.timedelta(minutes=ad.interval):
ad.send(id)
was_action = True
continue
except CannotAcquireLock:
settings.LOGGER.info("Cron %s. Another sending cron working. Exiting" %
id)
settings.LOGGER.info("Cron %s finished" % id)
def provision_certificates_cmdline():
import sys
from exclusiveprocess import Lock
from utils import load_environment
Lock(die=True).forever()
env = load_environment()
quiet = False
domains = []
for arg in sys.argv[1:]:
if arg == "-q":
quiet = True
else:
domains.append(arg)
# Go.
status = provision_certificates(env, limit_domains=domains)
# Show what happened.
for request in status:
if isinstance(request, str):
print(request)
else:
if quiet and request['result'] == 'skipped':
continue
print(request['result'] + ":", ", ".join(request['domains']) + ":")
for line in request["log"]:
print(line)
print()
def handle(self, *args, **options):
# Ensure this process doesn't run multiple times concurrently.
Lock(die=True).forever()
if options["forever"]:
# Loop forever.
while True:
self.send_new_emails()
time.sleep(20)
else:
# Run on-off job.
self.send_new_emails()
def perform_backup(full_backup):
env = load_environment()
# Create an global exclusive lock so that the backup script
# cannot be run more than one.
Lock(die=True).forever()
config = get_backup_config(env)
backup_root = os.path.join(env["STORAGE_ROOT"], 'backup')
backup_cache_dir = os.path.join(backup_root, 'cache')
backup_dir = os.path.join(backup_root, 'encrypted')
# Are backups disabled?
if config["target"] == "off":
return
# On the first run, always do a full backup. Incremental
# will fail. Otherwise do a full backup when the size of
# the increments since the most recent full backup are
# large.
try:
full_backup = full_backup or should_force_full(config, env)
except Exception as e:
# This was the first call to duplicity, and there might
# be an error already.
print(e)
sys.exit(1)
# Stop services.
def service_command(service, command, quit=None):
# Execute silently, but if there is an error then display the output & exit.
code, ret = shell('check_output',
["/usr/sbin/service", service, command],
capture_stderr=True,
trap=True)
if code != 0:
print(ret)
if quit:
sys.exit(code)
service_command("php7.2-fpm", "stop", quit=True)
service_command("postfix", "stop", quit=True)
service_command("dovecot", "stop", quit=True)
# Execute a pre-backup script that copies files outside the homedir.
# Run as the STORAGE_USER user, not as root. Pass our settings in
# environment variables so the script has access to STORAGE_ROOT.
pre_script = os.path.join(backup_root, 'before-backup')
if os.path.exists(pre_script):
shell('check_call',
['su', env['STORAGE_USER'], '-c', pre_script, config["target"]],
env=env)
# Run a backup of STORAGE_ROOT (but excluding the backups themselves!).
# --allow-source-mismatch is needed in case the box's hostname is changed
# after the first backup. See #396.
try:
shell('check_call', [
"/usr/bin/duplicity", "full" if full_backup else "incr",
"--verbosity", "warning", "--no-print-statistics", "--archive-dir",
backup_cache_dir, "--exclude", backup_root, "--volsize", "250",
"--gpg-options", "--cipher-algo=AES256", env["STORAGE_ROOT"],
config["target"], "--allow-source-mismatch"
] + rsync_ssh_options, get_env(env))
finally:
# Start services again.
service_command("dovecot", "start", quit=False)
service_command("postfix", "start", quit=False)
service_command("php7.2-fpm", "start", quit=False)
# Remove old backups. This deletes all backup data no longer needed
# from more than 3 days ago.
shell('check_call', [
"/usr/bin/duplicity", "remove-older-than",
"%dD" % config["min_age_in_days"], "--verbosity", "error",
"--archive-dir", backup_cache_dir, "--force", config["target"]
] + rsync_ssh_options, get_env(env))
# From duplicity's manual:
# "This should only be necessary after a duplicity session fails or is
# aborted prematurely."
# That may be unlikely here but we may as well ensure we tidy up if
# that does happen - it might just have been a poorly timed reboot.
shell('check_call', [
"/usr/bin/duplicity", "cleanup", "--verbosity", "error",
"--archive-dir", backup_cache_dir, "--force", config["target"]
] + rsync_ssh_options, get_env(env))
# Change ownership of backups to the user-data user, so that the after-bcakup
# script can access them.
if get_target_type(config) == 'file':
shell('check_call',
["/bin/chown", "-R", env["STORAGE_USER"], backup_dir])
# Execute a post-backup script that does the copying to a remote server.
# Run as the STORAGE_USER user, not as root. Pass our settings in
# environment variables so the script has access to STORAGE_ROOT.
post_script = os.path.join(backup_root, 'after-backup')
if os.path.exists(post_script):
shell('check_call',
['su', env['STORAGE_USER'], '-c', post_script, config["target"]],
env=env)
# Our nightly cron job executes system status checks immediately after this
# backup. Since it checks that dovecot and postfix are running, block for a
# bit (maximum of 10 seconds each) to give each a chance to finish restarting
# before the status checks might catch them down. See #381.
wait_for_service(25, True, env, 10)
wait_for_service(993, True, env, 10)
def provision_certificates_cmdline():
import sys
from exclusiveprocess import Lock
from utils import load_environment
Lock(die=True).forever()
env = load_environment()
verbose = False
headless = False
force_domains = None
show_extended_problems = True
args = list(sys.argv)
args.pop(0) # program name
if args and args[0] == "-v":
verbose = True
args.pop(0)
if args and args[0] == "-q":
show_extended_problems = False
args.pop(0)
if args and args[0] == "--headless":
headless = True
args.pop(0)
if args and args[0] == "--force":
force_domains = "ALL"
args.pop(0)
else:
force_domains = args
agree_to_tos_url = None
while True:
# Run the provisioning script. This installs certificates. If there are
# a very large number of domains on this box, it issues separate
# certificates for groups of domains. We have to check the result for
# each group.
def my_logger(message):
if verbose:
print(">", message)
status = provision_certificates(
env,
agree_to_tos_url=agree_to_tos_url,
logger=my_logger,
force_domains=force_domains,
show_extended_problems=show_extended_problems)
agree_to_tos_url = None # reset to prevent infinite looping
if not status["requests"]:
# No domains need certificates.
if not headless or verbose:
if len(status["problems"]) == 0:
print(
"No domains hosted on this box need a new TLS certificate at this time."
)
elif len(status["problems"]) > 0:
print(
"No TLS certificates could be provisoned at this time:"
)
print()
for domain in sort_domains(status["problems"], env):
print("%s: %s" % (domain, status["problems"][domain]))
sys.exit(0)
# What happened?
wait_until = None
wait_domains = []
for request in status["requests"]:
if request["result"] == "agree-to-tos":
# We may have asked already in a previous iteration.
if agree_to_tos_url is not None:
continue
# Can't ask the user a question in this mode. Warn the user that something
# needs to be done.
if headless:
print(", ".join(request["domains"]) +
" need a new or renewed TLS certificate.")
print()
print(
"This box can't do that automatically for you until you agree to Let's Encrypt's"
)
print(
"Terms of Service agreement. Use the Mail-in-a-Box control panel to provision"
)
print("certificates for these domains.")
sys.exit(1)
print("""
I'm going to provision a TLS certificate (formerly called a SSL certificate)
for you from Let's Encrypt (letsencrypt.org).
TLS certificates are cryptographic keys that ensure communication between
you and this box are secure when getting and sending mail and visiting
websites hosted on this box. Let's Encrypt is a free provider of TLS
certificates.
Please open this document in your web browser:
%s
It is Let's Encrypt's terms of service agreement. If you agree, I can
provision that TLS certificate. If you don't agree, you will have an
opportunity to install your own TLS certificate from the Mail-in-a-Box
control panel.
Do you agree to the agreement? Type Y or N and press <ENTER>: """ %
request["url"],
end='',
flush=True)
if sys.stdin.readline().strip().upper() != "Y":
print("\nYou didn't agree. Quitting.")
sys.exit(1)
# Okay, indicate agreement on next iteration.
agree_to_tos_url = request["url"]
if request["result"] == "wait":
# Must wait. We'll record until when. The wait occurs below.
if wait_until is None:
wait_until = request["until"]
else:
wait_until = max(wait_until, request["until"])
wait_domains += request["domains"]
if request["result"] == "error":
print(", ".join(request["domains"]) + ":")
print(request["message"])
if request["result"] == "installed":
print("A TLS certificate was successfully installed for " +
", ".join(request["domains"]) + ".")
if wait_until:
# Wait, then loop.
import time, datetime
print()
print("A TLS certificate was requested for: " +
", ".join(wait_domains) + ".")
first = True
while wait_until > datetime.datetime.now():
if not headless or first:
print(
"We have to wait",
int(
round((wait_until -
datetime.datetime.now()).total_seconds())),
"seconds for the certificate to be issued...")
time.sleep(10)
first = False
continue # Loop!
if agree_to_tos_url:
# The user agrees to the TOS. Loop to try again by agreeing.
continue # Loop!
# Unless we were instructed to wait, or we just agreed to the TOS,
# we're done for now.
break
# And finally show the domains with problems.
if len(status["problems"]) > 0:
print("TLS certificates could not be provisoned for:")
for domain in sort_domains(status["problems"], env):
print("%s: %s" % (domain, status["problems"][domain]))
# echo $congress;
# analysis/text_incorporation.py analyze $congress;
# analysis/text_incorporation.py load $congress;
# done
import sys
import re
import unicodedata
from io import StringIO
import lxml.etree
from numpy import percentile
# Ensure we don't run two times concurrently since this process
# updates a data file.
from exclusiveprocess import Lock
Lock(die=True).forever()
if sys.stdout.isatty():
from tqdm import tqdm
else:
def tqdm(iter, *args, **kwargs):
return iter
def extract_text(fn):
# Given a path to a bill text XML file from Congress,
# serialize the substantive legislative text into a
# flat string that can be passed to text comparison
# tools. Returns the string.
# Log what happens so we can see that the locks are acquired and released.
import logging
logging.getLogger().setLevel('INFO')
# Begin tests...
from exclusiveprocess import Lock, CannotAcquireLock
# Try with a with block.
try:
with Lock(name="test1") as lock:
print("Hello!", lock.lockfile)
except CannotAcquireLock:
raise ValueError("should not get here")
# Try with a with block, but choosing the lock name automatically.
try:
with Lock() as lock:
print("The lock name is based on the file path", lock.lockfile)
except CannotAcquireLock:
raise ValueError("should not get here")
# Try with a decorator.
@Lock
def myfunc():
print("This happened inside a lock with automatic lock name.")