def __init__(self, logger=None):
"""
:type logger: logging.Logger
"""
self.not_implemented_api_method = cherrypy.HTTPError(
405, 'Method Not Allowed')
self.logger = logger or get_logger()
def build_instagram_client_using_env_variables(
client_id_variable_name=INSTAGRAM_CLIENT_ID_ENV_NAME,
access_token_variable_name=INSTAGRAM_ACCESS_TOKEN_ENV_NAME):
client_id = get_env_variable(client_id_variable_name, "")
access_token = get_env_variable(access_token_variable_name, "")
return InstagramClient(client_id=client_id,
access_token=access_token,
logger=get_logger())
def create_app():
""" Create the istance for Flask application """
###############################
# Create a FLASK application
app = Flask(__name__)
# Note: since the app is defined inside this file,
# the static dir will be searched inside this subdirectory
###############################
# Apply configuration
app.config.from_object(CONFIG_MODULE + '.MyConfig')
logger = get_logger(__name__, False) # app.config['DEBUG'])
###############################
# # Cache
# # http://flask.pocoo.org/docs/0.10/patterns/caching/#setting-up-a-cache
# from werkzeug.contrib.cache import SimpleCache
# cache = SimpleCache()
# ###############################
# # Database
# db.init_app(app)
# ###############################
# # Application context
# with app.app_context():
# db.create_all()
# logger.info("Initialized Database")
# ###############################
# Add basic things to this app
app.register_blueprint(cms)
###############################
# Flask LOGIN
lm.init_app(app)
lm.login_view = '.login'
# Logging
@app.after_request
def log_response(resp):
log = logger.debug
if resp.status_code == hcodes.HTTP_NOT_MODIFIED:
log = logger.debug
if 'static/' not in req.url and '/js/' not in req.url:
log = logger.info
from commons.logs import obscure_passwords
log("{} {} {} {}".format(
req.method, req.url,
obscure_passwords(req.data), resp))
return resp
return app
def _wait_for_task_and_get_result(async_result,):
"""
:type async_result: celery.result.AsyncResult
:rtype: dict | list | object
"""
logger = get_logger()
logger.info("Waiting synchronously for task {} and retuning results...".format(async_result.task_id))
try:
return async_result.get()
except Exception as e:
logger.error("Error while waiting for result: {}. Details: {}".format(async_result.task_id, e))
raise e
def __init__(self, api_key, market, locale, currency):
"""
:type api_key: str
:param market: The users market country, Skyscanner country code
:type market: str
:param locale: The users selected language, ISO locale code
:type locale: str
:param currency: The users selected currency, ISO currency code
:type currency: str
"""
self.api_key = api_key
self.locale = locale
self.market = market
self.currency = currency
self.logger = get_logger()
import os
import json
import unittest
import commons.htmlcodes as hcodes
from commons.logs import get_logger
from restapi.server import create_app
from restapi.confs.config import USER, PWD, \
TEST_HOST, SERVER_PORT, API_URL, AUTH_URL
from commons.tests.utilities import TestUtilities
# , API_URI, OK, NO_CONTENT, BAD_REQUEST, FORBIDDEN, NOTFOUND, NOT_ALLOWED
from commons import myself
__author__ = myself
logger = get_logger(__name__, True)
API_URI = 'http://%s:%s%s' % (TEST_HOST, SERVER_PORT, API_URL)
AUTH_URI = 'http://%s:%s%s' % (TEST_HOST, SERVER_PORT, AUTH_URL)
class TestDataObjects(TestUtilities):
@classmethod
def setUpClass(cls):
logger.info('### Setting up flask server ###')
app = create_app(testing=True)
cls.app = app.test_client()
loginURI = os.path.join(AUTH_URI, 'login')
r = cls.app.post(loginURI,
"""
The most basic (and standard) Rest Resource
we could provide back then
"""
import pytz
import dateutil.parser
from datetime import datetime
from flask import g
from flask_restful import request, Resource, reqparse
from ..confs.config import API_URL # , STACKTRACE
from ..response import ResponseElements
from commons import htmlcodes as hcodes
from commons.logs import get_logger
logger = get_logger(__name__)
###################
# Paging costants
CURRENTPAGE_KEY = 'currentpage'
DEFAULT_CURRENTPAGE = 1
PERPAGE_KEY = 'perpage'
DEFAULT_PERPAGE = 10
###################
# Extending the concept of rest generic resource
class ExtendedApiResource(Resource):
"""
Implements a generic Resource for our Restful APIs model
"""
# import io
# import os
# import json
import unittest
import logging
from commons.logs import get_logger
from restapi.server import create_app
__author__ = "Paolo D'Onorio De Meo (GitHub@pdonorio)"
# API_URI = 'http://%s:%s%s' % (TEST_HOST, SERVER_PORT, API_URL)
# AUTH_URI = 'http://%s:%s%s' % (TEST_HOST, SERVER_PORT, AUTH_URL)
logger = get_logger(__name__)
logger.setLevel(logging.DEBUG)
class TestDataObjects(unittest.TestCase):
@classmethod
def setUpClass(cls):
logger.info('### Setting up flask server ###')
app = create_app(testing=True)
cls.app = app.test_client()
# r = cls.app.post(
# AUTH_URI + '/login',
# data=json.dumps({'username': USER, 'password': PWD}))
# content = json.loads(r.data.decode('utf-8'))
# token = content['Response']['data']['token']
from __future__ import absolute_import, unicode_literals
from clients.instagram import build_instagram_client_using_env_variables
from clients.open_weather import build_open_weather_client_using_env_variables
from clients.sky_scanner import build_sky_scanner_client_from_env_vars
from clients.uber import build_uber_client_using_env_variables
from commons.conversion import parse_date
from commons.logs import get_logger
from commons.model import Location, City
from .celery import app
logger = get_logger()
@app.task
def retrieve_weather(location_dict):
"""
:type location_dict: dict
:rtype: list[dict]
"""
location = Location.from_serializable(location_dict)
client = build_open_weather_client_using_env_variables()
forecasts = client.get_5_days_forecast(location)
return [f.to_serializable() for f in forecasts]
@app.task
def retrieve_instagram_media(city_dict):
"""
:type city_dict: dict
:rtype: list[dict]
def create_app(name=__name__, debug=False,
worker_mode=False, testing_mode=False,
avoid_context=False, enable_security=True,
skip_endpoint_mapping=False,
**kwargs):
""" Create the server istance for Flask application """
#################################################
# Flask app instance
#################################################
from .confs import config
microservice = Flask(name, **kwargs)
##############################
# @microservice.before_first_request
# def first():
# print("BEFORE THE VERY FIRST REQUEST", g)
# @microservice.before_request
# def before():
# print("BEFORE EVERY REQUEST...")
# @microservice.after_request
# def after(response):
# print("AFTER EVERY REQUEST...")
# return response
##############################
# Disable security if launching celery workers
if worker_mode:
enable_security = False
# Set app internal testing mode if create_app received the parameter
if testing_mode:
microservice.config['TESTING'] = testing_mode
##############################
# Flask configuration from config file
microservice.config.from_object(config)
if ENVVAR_DEBUG is not None:
try:
tmp = int(ENVVAR_DEBUG) == 1
except:
tmp = str(ENVVAR_DEBUG).lower() == 'true'
debug = tmp # bool(tmp)
microservice.config['DEBUG'] = debug
# Set the new level of debugging
logger = get_logger(__name__, debug)
logger.info("FLASKING! Created application")
##############################
if PRODUCTION:
logger.info("Production server ON")
## // TO FIX or CHECK
# # Check and use a random file a secret key.
# install_secret_key(microservice)
# probably useless
# # http://stackoverflow.com/a/26636880/2114395
# microservice.config.update(
# dict(PREFERRED_URL_SCHEME = 'https')
# )
# # To enable exceptions printing inside uWSGI
# # http://stackoverflow.com/a/17839750/2114395
# from werkzeug.debug import DebuggedApplication
# app.wsgi_app = DebuggedApplication(app.wsgi_app, True)
#################################################
# Other components
#################################################
##############################
# Cors
from .cors import cors
cors.init_app(microservice)
logger.info("FLASKING! Injected CORS")
##############################
# DATABASE/SERVICEs CHECKS
from .resources.services.detect import services as internal_services
for service, myclass in internal_services.items():
logger.info("Available service %s" % service)
myclass(check_connection=True, app=microservice)
##############################
# Enabling our internal Flask customized response
from .response import InternalResponse
microservice.response_class = InternalResponse
##############################
# Flask security
if enable_security:
# Dynamically load the authentication service
meta = Meta()
module_base = __package__ + ".resources.services.authentication"
auth_service = os.environ.get('BACKEND_AUTH_SERVICE', '')
module_name = module_base + '.' + auth_service
logger.debug("Trying to load the module %s" % module_name)
module = meta.get_module_from_string(module_name)
init_auth = create_auth_instance(
module, internal_services, microservice)
# Global namespace inside the Flask server
@microservice.before_request
def enable_authentication_per_request():
""" Save auth object """
# Authentication the right (per-instance) way
custom_auth = create_auth_instance(
module, internal_services, microservice)
# Save globally across the code
g._custom_auth = custom_auth
# OLD BAD
# def enable_global_authentication():
# """ Save auth object """
# g._custom_auth = custom_auth
# Enabling also OAUTH library
from .oauth import oauth
oauth.init_app(microservice)
logger.info("FLASKING! Injected security internal module")
if not worker_mode:
# Global namespace inside the Flask server
@microservice.before_request
def enable_global_services():
""" Save all databases/services """
g._services = internal_services
##############################
# Restful plugin
if not skip_endpoint_mapping:
from .rest import Api, EndpointsFarmer, create_endpoints
# Defining AUTOMATIC Resources
current_endpoints = \
create_endpoints(EndpointsFarmer(Api), enable_security, debug)
# Restful init of the app
current_endpoints.rest_api.init_app(microservice)
##############################
# Init objects inside the app context
if not avoid_context:
with microservice.app_context():
# Set global objects for celery workers
if worker_mode:
from commons.globals import mem
mem.services = internal_services
# Note:
# Databases are already initialized inside the instances farm
# Outside of the context
# p.s. search inside this file for 'myclass('
# Init users/roles for Security
if enable_security:
# custom_auth.setup_secret(microservice.config['SECRET_KEY'])
# custom_auth.init_users_and_roles()
init_auth.init_users_and_roles()
# Allow a custom method for mixed services init
try:
from .resources.custom import services as custom_services
custom_services.init(internal_services, enable_security)
except:
logger.debug("No custom init available for mixed services")
##############################
# Logging responses
@microservice.after_request
def log_response(response):
data = handle_log_output(request.data)
# Limit the parameters string size, sometimes it's too big
for k in data:
# print("K", k, "DATA", data)
try:
if not isinstance(data[k], str):
continue
if len(data[k]) > MAX_CHAR_LEN:
data[k] = data[k][:MAX_CHAR_LEN] + "..."
except IndexError:
pass
logger.info("{} {} {} {}".format(
request.method, request.url, data, response))
return response
##############################
# Enabling user callbacks after a request
@microservice.after_request
def call_after_request_callbacks(response):
for callback in getattr(g, 'after_request_callbacks', ()):
callback(response)
return response
##############################
# App is ready
return microservice
def __init__(self, logger=None):
"""
:type logger: logging.Logger
"""
self.logger = logger or get_logger()
import csv
from commons.logs import get_logger
from commons.model import Location, City
CITIES_CSV = '../ServerlessPwrInz-Cities.csv'
COUNTRY_COLUMN_INDEX = 0
CITY_COLUMN_INDEX = 1
POPULATION_COLUMN_INDEX = 2
LATITUDE_COLUMN_INDEX = 3
LONGITUDE_COLUMN_INDEX = 4
SKY_SCANNER_CITY_CODE_COLUMN_INDEX = 5
logger = get_logger(calling_module="csv_improrter")
def import_from_file(csv_file_name=CITIES_CSV):
logger.info("Opening file: {} for reading cities...".format(csv_file_name))
cities = []
with open(csv_file_name, 'rb') as csv_file:
places_csv_reader = csv.reader(csv_file, delimiter=',', quotechar='|')
for row in places_csv_reader:
if row:
try:
lat = float(row[LATITUDE_COLUMN_INDEX])
lon = float(row[LONGITUDE_COLUMN_INDEX])
location = Location(latitude=lat, longitude=lon)
country = unicode(row[COUNTRY_COLUMN_INDEX])
city_name = unicode(row[CITY_COLUMN_INDEX])
population = int(row[POPULATION_COLUMN_INDEX])
