def process_alexa_top_site_updater(top_site_url='http://s3.amazonaws.com/alexa-static/top-1m.csv.zip'):
logging.info("Starting Alexa Top Site job")
#-- Get data
zipped_alexa_data = urllib2.urlopen(top_site_url).read() # open('/var/tmp/top-1m.csv.zip').read()
alexa_data = zipfile.ZipFile(StringIO.StringIO(zipped_alexa_data))
alexa_data_file = alexa_data.open(alexa_data.infolist()[0])
rank_data = map(lambda (rank, domain): [int(rank.strip()), domain.strip()], [line.split(',') for line in alexa_data_file])
alexa_data_file.close()
#max_batch_update_size = 100000
#rank_update_chunks = list([rank_data[i:i + self.max_batch_update_size] for i in xrange(0, len(rank_data), self.max_batch_update_size)])
#-- Save data
domain_metadata_db = utils.get_mongodb_connection().domain_metadata
default_ranking = {'alexa_metadata': { 'popularity_rank' : -1, 'reach_rank' : -1 }, 'alexa_timestamp': datetime.now()}
domain_metadata_db.update({}, {'$set': default_ranking}, multi=True)
for rank, domain_name in rank_data:
domain = models.Domain.get(domain_name)
domain.metadata.alexa_metadata = { 'popularity_rank' : rank, 'reach_rank' : rank }
domain.metadata.alexa_timestamp = datetime.now()
domain.metadata.save()
logging.info("Completed Alexa Top Site job")
return True
jinja2.Environment(variable_start_string='[[', variable_end_string=']]')
class CustomFlask(Flask):
jinja_options = Flask.jinja_options.copy()
jinja_options.update(dict(
variable_start_string='[[',
variable_end_string=']]',
))
secret_key = config.secret_key
app = CustomFlask(__name__)
#----- Mongo session setting
app.config['MONGODB_DB'] = config.mongo['db']
db = utils.get_mongodb_connection()
with app.app_context(): app.session_interface = MongoDBSessionInterface(app, db, 'sessions')
#----- Mail setting
app.config.update(
MAIL_SERVER = config.mail["host"],
MAIL_PORT = config.mail.get("port", 587),
MAIL_USE_TLS = config.mail.get("tls", False),
MAIL_USE_SSL = config.mail.get("ssl", False),
MAIL_USERNAME = config.mail.get("user", ""),
MAIL_PASSWORD = config.mail.get("password", ""),
DEFAULT_MAIL_SENDER = config.mail["from_address"]
)
mailer = Mail(app)
