def process_alexa_top_site_updater(top_site_url='http://s3.amazonaws.com/alexa-static/top-1m.csv.zip'): logging.info("Starting Alexa Top Site job") #-- Get data zipped_alexa_data = urllib2.urlopen(top_site_url).read() # open('/var/tmp/top-1m.csv.zip').read() alexa_data = zipfile.ZipFile(StringIO.StringIO(zipped_alexa_data)) alexa_data_file = alexa_data.open(alexa_data.infolist()[0]) rank_data = map(lambda (rank, domain): [int(rank.strip()), domain.strip()], [line.split(',') for line in alexa_data_file]) alexa_data_file.close() #max_batch_update_size = 100000 #rank_update_chunks = list([rank_data[i:i + self.max_batch_update_size] for i in xrange(0, len(rank_data), self.max_batch_update_size)]) #-- Save data domain_metadata_db = utils.get_mongodb_connection().domain_metadata default_ranking = {'alexa_metadata': { 'popularity_rank' : -1, 'reach_rank' : -1 }, 'alexa_timestamp': datetime.now()} domain_metadata_db.update({}, {'$set': default_ranking}, multi=True) for rank, domain_name in rank_data: domain = models.Domain.get(domain_name) domain.metadata.alexa_metadata = { 'popularity_rank' : rank, 'reach_rank' : rank } domain.metadata.alexa_timestamp = datetime.now() domain.metadata.save() logging.info("Completed Alexa Top Site job") return True
jinja2.Environment(variable_start_string='[[', variable_end_string=']]') class CustomFlask(Flask): jinja_options = Flask.jinja_options.copy() jinja_options.update(dict( variable_start_string='[[', variable_end_string=']]', )) secret_key = config.secret_key app = CustomFlask(__name__) #----- Mongo session setting app.config['MONGODB_DB'] = config.mongo['db'] db = utils.get_mongodb_connection() with app.app_context(): app.session_interface = MongoDBSessionInterface(app, db, 'sessions') #----- Mail setting app.config.update( MAIL_SERVER = config.mail["host"], MAIL_PORT = config.mail.get("port", 587), MAIL_USE_TLS = config.mail.get("tls", False), MAIL_USE_SSL = config.mail.get("ssl", False), MAIL_USERNAME = config.mail.get("user", ""), MAIL_PASSWORD = config.mail.get("password", ""), DEFAULT_MAIL_SENDER = config.mail["from_address"] ) mailer = Mail(app)