def do_something():
headlines = build_headlines()
maybe_better_headlines = {}
r_url = env.require('REDISCLOUD_URL')
rdb = redis.StrictRedis.from_url(r_url)
for text, url in headlines.items():
new_text, count = DJ_SEARCH.subn('DJ\\2', text)
if count:
maybe_better_headlines[new_text] = url
# see which of these headlines are new
possible_tweets = []
for text, url in maybe_better_headlines.items():
key = 'headline:{}'.format(md5(text.encode('utf8')).hexdigest())
if not rdb.get(key):
possible_tweets.append('{} {}'.format(text, url))
rdb.set(key, url)
rdb.expire(key, EXPIRES)
# have a human pick a tweet
if possible_tweets:
print 'Which headlines are worth tweeting?'
print '-' * 80
for idx, text in enumerate(possible_tweets, start=1):
print '\t', idx, text
print '-' * 80
while True:
out = raw_input('> (return to exit) ')
if out:
try:
queue(rdb, possible_tweets[int(out) - 1])
except (IndexError, ValueError):
out = 'foo'
else:
break
if 'send' in sys.argv[1:]:
try:
send(rdb)
except tweepy.TweepError as e:
# TODO figure out error code, need json?
import ipdb; ipdb.set_trace()
# DELETEME below, just for debuggin
print 'queue:'
from pprint import pprint
pprint(list(rdb.lrange(QUEUE_KEY, 0, -1))) # DELETEME
def test_require_acts_like_get(self):
os.environ['FOO'] = 'BAR'
self.assertEqual(env.require('FOO'), 'BAR')
del os.environ['FOO'] # teardown
def test_require_raises_exception_with_stupid_default(self):
with self.assertRaises(ImproperlyConfigured):
env.require('FOO', default='')
with self.assertRaises(ImproperlyConfigured):
env.require('FOO', default=u'')
def test_require_raises_exception(self):
with self.assertRaises(ImproperlyConfigured):
env.require('FOO')
},
},
'handlers': {
'console': {
'level': 'DEBUG',
'class': 'project_runpy.ColorizingStreamHandler',
'formatter': 'verbose',
},
},
'loggers': {
'py.warnings': {
# how do i get colored warnings without duplicates?
'propagate': False,
},
'django.db.backends': {
'level': 'DEBUG' if env.get('SQL') else 'INFO',
'filters': ['require_debug_true', 'readable_sql'],
},
'factory': {
'level': 'ERROR',
'propagate': False,
},
},
}
# App settings
AWS_ACCESS_KEY_ID = env.require('AWS_ACCESS_KEY_ID')
AWS_SECRET_ACCESS_KEY = env.require('AWS_SECRET_ACCESS_KEY')
{
'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
},
]
AUTHENTICATION_BACKENDS = (
# https://sellercentral.amazon.com/home?cor=login_NA&
'social_core.backends.amazon.AmazonOAuth2',
'django.contrib.auth.backends.ModelBackend',
)
SOCIAL_AUTH_AMAZON_KEY = env.require('SOCIAL_AUTH_AMAZON_KEY')
SOCIAL_AUTH_AMAZON_SECRET = env.require('SOCIAL_AUTH_AMAZON_SECRET')
SOCIAL_AUTH_POSTGRES_JSONFIELD = True
# https://python-social-auth.readthedocs.io/en/latest/pipeline.html#authentication-pipeline
# https://python-social-auth.readthedocs.io/en/latest/configuration/django.html?highlight=social_core.pipeline.user.user_details#personalized-configuration
SOCIAL_AUTH_PIPELINE = (
'social_core.pipeline.social_auth.social_details',
'social_core.pipeline.social_auth.social_uid',
'social_core.pipeline.social_auth.social_user',
'social_core.pipeline.user.get_username',
'social_core.pipeline.user.create_user',
'social_core.pipeline.social_auth.associate_user',
'social_core.pipeline.social_auth.load_extra_data',
'social_core.pipeline.user.user_details',
)
if not DEBUG:
def test_require_acts_like_get(self):
os.environ['FOO'] = 'BAR'
self.assertEqual(env.require('FOO'), 'BAR')
del os.environ['FOO'] # teardown
def test_require_raises_exception_with_stupid_default(self):
with self.assertRaises(ImproperlyConfigured):
env.require('FOO', default='')
with self.assertRaises(ImproperlyConfigured):
env.require('FOO', default='')
def test_require_raises_exception(self):
with self.assertRaises(ImproperlyConfigured):
env.require('FOO')
def get_csvs(uid_path, mvl_path):
ipeds_email = env.require('IPEDS_EMAIL')
ipeds_password = env.require('IPEDS_PASSWORD')
# Validate inputs
if not os.path.isfile(uid_path):
exit('Exiting: .uid file must exist')
if not os.path.exists(mvl_path):
exit('Exiting: .mvl must exist')
if os.path.isfile(mvl_path):
mvl_files = [mvl_path]
else:
mvl_files = list(iglob(os.path.join(mvl_path, '*.mvl')))
if not mvl_files:
exit('Exiting no .mvl files found')
# sanitize: the browser expects files to be absolute paths
mvl_files = map(os.path.abspath, mvl_files)
driver = webdriver.Chrome() # DELETEME Chrome can go to Hell
# driver = webdriver.Firefox() # XXX Firefox, why do you keep breaking?
# Firefox driver needs waits to work.
# start session
driver.get('http://nces.ed.gov/ipeds/datacenter/')
# "Compare Institutions"
driver.find_element_by_id('tdInstData').click()
# > sent to http://nces.ed.gov/ipeds/datacenter/login.aspx
# TODO explicitly click "use final release data"
driver.find_element_by_id('ibtnLoginLevelOne').click()
# > sent to http://nces.ed.gov/ipeds/datacenter/InstitutionByName.aspx
driver.execute_script('$("#tbInstitutionSearch").val("{}")'
.format(u','.join(get_uids(uid_path))))
driver.find_element_by_id('ctl00_contentPlaceHolder_ibtnSelectInstitutions').click()
# > sent to "1. Select Institutions" part two - checkboxes
driver.execute_script('CheckGVInstitutions()')
driver.find_element_by_id('ctl00_contentPlaceHolder_ibtnContinue').click()
# > sent to "1. Select Institutions" part three - final list
driver.get('http://nces.ed.gov/ipeds/datacenter/mastervariablelist.aspx?stepId=2')
# > sent to "2. Select Variables"
driver.get('http://nces.ed.gov/ipeds/datacenter/UploadMasterList.aspx?stepId=2')
# login to enable upload by variable
driver.get('http://nces.ed.gov/ipeds/datacenter/PowerUserLogin.aspx')
driver.find_element_by_id('tbPowerUserEmail').send_keys(ipeds_email)
driver.find_element_by_id('tbPowerUserPassword').send_keys(ipeds_password)
driver.find_element_by_id('ibtnLogin').click() # submit
for i, mvl_file_path in enumerate(mvl_files, start=1):
# Go back to this screen
driver.get('http://nces.ed.gov/ipeds/datacenter/UploadMasterList.aspx?stepId=2')
################ upload mvl
field = driver.find_element_by_id('ctl00_contentPlaceHolder_fulFile')
field.send_keys(mvl_file_path)
driver.find_element_by_id('ctl00_contentPlaceHolder_ibtnSubmit').click() # submit
driver.find_element_by_xpath('//a[text()="Select all"]').click()
driver.find_element_by_id('ctl00_contentMainBody_iActionButton').click() # submit
# select "Short variable name"
driver.find_element_by_id('ctl00_contentPlaceHolder_rbShortVariableName').click()
# Download Report
# download_data(driver)
driver.find_element_by_id('ctl00_contentPlaceHolder_imgbtnGetCustomDataSet').click()
# Clear Variables
if i < len(mvl_files):
driver.get('http://nces.ed.gov/ipeds/datacenter/mastervariablelist.aspx?delete=true')
# Wait. Otherwise Selenium closes the browser before download finishes
raw_input('Press Enter to finish.')
driver.close()
from delorean import parse
# Refresh a database from another one
# Does the following:
# - deletes the target database
# - restores the most recent snapshot of the source db as the target db
# - removes the snapshot
# Requires:
# - boto
# - AWS credentials
# - project_runpy
# - Delorean
target_db = env.require('TARGET_DB')
source_db = env.require('SOURCE_DB')
db_subnet_group = env.get('DB_SUBNET_GROUP')
security_group = env.require('SECURITY_GROUP')
target_pass = env.require('TARGET_PASS')
region = env.get('AWS_REGION', 'us-east-1')
instance_class = env.get('INSTANCE_CLASS', 'db.t1.micro')
if 'prod' in target_db:
exit("that looks like production; do it yourself")
rds_connection = boto.rds.connect_to_region(region)
# print "deleting {} snapshot...".format(snapshot)
# try:
def only_send():
"""Hack to give a way to only send from the queue"""
r_url = env.require('REDISCLOUD_URL')
rdb = redis.StrictRedis.from_url(r_url)
send(rdb)