def get_data(url):
"""Get data from URL.
Args:
url: URL.
Returns:
result: URL data
"""
# Initialize key variables
result = None
html = None
count = 0
max_count = 5
# Log
log_message = 'Processing {}'.format(url)
log.log2debug(2003, log_message)
# Retry up to 5 times
while count <= max_count:
# Loop counter
count += 1
# Conscientious sleep
time.sleep(2 + (random.random() * count))
# Read data
try:
with urllib.request.urlopen(url) as response:
html = response.read()
except:
pass
# Stop if we get data
if bool(html) is True:
break
# Process data
if bool(html) is True:
# Convert data to dict
data = json.loads(html)
org = Org(data)
result = org.everything()
# Return
return result
def main():
"""Main Function."""
# Initialize key variables
# Get the CLI arguments
args = cli()
human_file = os.path.abspath(os.path.expanduser(args.human_file))
output_file = os.path.abspath(os.path.expanduser(args.output_file))
history_file = os.path.abspath(os.path.expanduser(args.history_file))
# Log start
log_message = 'Starting Mailto job'
log.log2debug(3000, log_message)
# Get human records
humans_ = humans.Humans(human_file)
everyone = humans_.complete()
# Filter persons
strainer_ = humans.Strainer(everyone)
caribbean = strainer_.caribbean()
# Create object for generating emails
mailto = lib_email.Mailto(history_file, output_file, args.subject)
# Process GOATs
label(args.output_file, 'Goats')
goats = humans.goats(humans_)
generator(mailto, goats)
# Process Caribbean
label(args.output_file, 'Caribbean')
generator(mailto, caribbean)
# Process state
if bool(args.states) is True:
for state in args.states.split(','):
# Update the stuff
label(args.output_file, state.upper())
citizens = strainer_.state(state.upper(), individuals_only=True)
generator(mailto, citizens)
# Log stop
log_message = 'Mailto estimate job complete'
log.log2debug(3001, log_message)
def main():
"""Main Function."""
# Initialize key variables
# Get the CLI arguments
args = cli()
human_file = os.path.abspath(os.path.expanduser(args.human_file))
body_file = os.path.abspath(os.path.expanduser(args.body_file))
cache_directory = os.path.abspath(os.path.expanduser(args.cache_directory))
attachment = os.path.abspath(os.path.expanduser(args.attachment)) if bool(
args.attachment) else None
# Determine the output filename
_campaign = lib_email.campaign_files(args.campaign,
cache_directory=cache_directory)
output_file = _campaign.thunderbird_file
# Log start
log_message = 'Starting Thunderbird file creation job'
log.log2debug(6000, log_message)
# Get human records
humans_ = humans.SimpleHumans(human_file)
everyone = humans_.complete()
# Create object for generating emails
thunderbird = lib_email.Thunderbird(args.campaign,
body_file,
args.subject,
args.sender,
cache_directory=cache_directory,
attachment=attachment)
# Process Everyone
label(output_file, 'Everyone Email')
generator(thunderbird, everyone)
# Log stop
log_message = 'Thunderbird file creation job complete'
log.log2debug(6001, log_message)
def main():
"""Main Function."""
# Initialize key variables
count = 0
interval = 5
# Get the CLI arguments
args = cli()
_campaign = lib_email.campaign_files(
args.campaign, cache_directory=args.cache_directory)
input_file = _campaign.thunderbird_file
# Log start
log_message = 'Starting Thunderbird sending job'
log.log2debug(5000, log_message)
# Read body_file into a string
with open(input_file, 'r') as fh_:
lines = fh_.readlines()
records = [_.strip() for _ in lines if '#' not in _]
for record in records:
count += 1
print(count)
# Process command
with subprocess.Popen(
shlex.split(record),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE) as process:
stdoutdata, stderrdata = process.communicate()
returncode = process.returncode
if count % interval == 0:
input('Press any key to continue...')
# Log stop
log_message = 'Thunderbird sending job complete'
log.log2debug(5001, log_message)
def main():
"""Main Function."""
# Get the CLI arguments
args = cli()
human_file = os.path.abspath(os.path.expanduser(args.human_file))
# Log start
log_message = 'Starting analysis job'
log.log2debug(9000, log_message)
# Get human records
humans_ = humans.Humans(human_file)
# Process
everyone = humans_.complete()
lookup = humans.histogram(everyone, individuals_only=not args.teams)
# Print result
for key, value in sorted(lookup.items(), key=lambda item: item[1]):
print(key, '\t', value)
# Log stop
log_message = 'Analysis job complete'
log.log2debug(9001, log_message)
def main():
"""Main Function."""
# Get the CLI arguments
args = cli()
# Log start
log_message = 'Starting mailer job'
log.log2debug(1000, log_message)
# Get configuration
config_filepath = os.path.abspath(os.path.expanduser(args.config))
with open(config_filepath) as fh_:
config = yaml.safe_load(fh_)
# Get email account information
sender = Person(firstname=config['firstname'],
lastname=config['lastname'],
email=config['username'],
individual=True,
validated=True)
# Get humans
persons = humans.Humans(
os.path.abspath(os.path.expanduser(args.human_file)))
recipients = persons.uniques()
sys.exit(0)
# See output
print(recipients)
sys.exit(0)
# Read the email body
textfile = os.path.abspath(os.path.expanduser(args.html_file))
with open(textfile) as fh_:
body = fh_.read()
# Read the email attachment
imagefile = os.path.abspath(os.path.expanduser(args.image_file))
# Get authentication information
auth_ = MailAuth(username=config['username'], password=config['password'])
# Send mail to each recipient
for recipient in recipients:
mail_ = Mail(sender=sender,
receiver=recipient,
body=body,
subject=args.subject,
image=imagefile)
# Send the email
result = lib_email.send(auth_, mail_)
if bool(result) is True:
log_message = 'Successfully sent to {}'.format(recipient.email)
log.log2debug(1002, log_message)
else:
log_message = 'Failed to send to {}'.format(recipient.email)
log.log2debug(1002, log_message)
# Log stop
log_message = 'Mailer job complete'
log.log2debug(1001, log_message)
def main():
"""Main Function."""
# Initialize key variables
# Get the CLI arguments
args = cli()
# Log start
log_message = 'Starting Job'
log.log2debug(2000, log_message)
# Get URLs from file
filepath = os.path.abspath(os.path.expanduser(args.html_file))
urls = get_urls(filepath)
# Process Contact
with open(args.output_file, 'w', newline='') as fh_:
linewriter = csv.writer(
fh_, delimiter='\t', quotechar='|', quoting=csv.QUOTE_MINIMAL)
# Create header row
linewriter.writerow(
['business_org', 'business_address', 'business_handle',
'business_kind', 'business_registration', 'business_updated',
'contact_name', 'contact_email',
'contact_kind', 'contact_status']
)
for url in urls:
business = get_data(url)
# Skip if business was invalid
if bool(business) is False:
continue
# Process business
for contact in business.contacts:
# Log message
log_message = 'Creating entry for {}'.format(contact.email)
log.log2debug(2001, log_message)
# Update file
linewriter.writerow([
business.org,
business.address,
business.handle,
business.kind,
business.registration,
business.updated,
contact.name,
contact.email,
contact.kind,
contact.status,
])
# Flush to disk immediately
fh_.flush()
# Log stop
log_message = 'Job complete'
log.log2debug(2002, log_message)
def main():
"""Main Function."""
# Initialize key variables
timestamp = None
# Get the CLI arguments
args = cli()
human_file = os.path.abspath(os.path.expanduser(args.human_file))
body_file = os.path.abspath(os.path.expanduser(args.body_file))
if bool(args.cache_directory):
cache_directory = os.path.abspath(
os.path.expanduser(args.cache_directory))
# Get timestamp
if bool(args.date) is True:
timestamp = misc.timestamp(args.date)
# Determine the output filename
_campaign = lib_email.campaign_files(args.campaign,
cache_directory=cache_directory)
output_file = _campaign.thunderbird_file
# Log start
log_message = 'Starting Thunderbird file creation job'
log.log2debug(4000, log_message)
# Get human records
humans_ = humans.Humans(human_file)
everyone = humans_.complete()
# Filter persons
strainer_ = humans.Strainer(everyone)
# for i in strainer_.smallfry(individuals_only=True):
# print(i)
# sys.exit()
# Create object for generating emails
thunderbird = lib_email.Thunderbird(args.campaign,
body_file,
args.subject,
args.sender,
cache_directory=cache_directory)
# process Anglophone organizations
if not args.spanish:
# Process GOATs
# label(output_file, 'Goats')
# goats = humans.goats(humans_)
# generator(thunderbird, goats)
#
# # Process Caribbean
# label(output_file, 'Caribbean')
# generator(thunderbird, strainer_.caribbean())
# Process Educational
label(output_file, 'Educational')
generator(thunderbird, strainer_.edu(individuals_only=True))
# Process states/provinces with few organizations
label(output_file, 'SmallFry')
generator(thunderbird,
strainer_.smallfry(individuals_only=True, strict=True))
# Process state
if bool(args.states) is True:
for state in args.states.split(','):
# Update the stuff
label(output_file, state.upper())
citizens = strainer_.state(state.upper(),
individuals_only=not bool(args.teams),
timestamp=timestamp,
strict=True)
generator(thunderbird, citizens, spanish=args.spanish)
# Log stop
log_message = 'Thunderbird file creation job complete'
log.log2debug(4001, log_message)
