def get_tweets(): """ returns twitter feed with settings as described below, contains all related twitter settings """ api = twitter.Api(consumer_key=get_env_variable('TWITTER_CONSUMER_KEY'), consumer_secret=get_env_variable('TWITTER_CONSUMER_SERCRET'), access_token_key=get_env_variable('TWITTER_ACCESS_TOKEN_KEY'), access_token_secret=get_env_variable('TWITTER_ACCESS_TOKEN_SECRET')) return api.GetUserTimeline(screen_name='onekiloparsec', exclude_replies=True, include_rts=False)
def search_tweets(term=None, geocode=None, since_id=None, max_id=None, until=None, count=15, lang=None, locale=None, result_type='popular', include_entities=None): """ returns twitter feed with settings as described below, contains all related twitter settings """ api = twitter.Api(consumer_key=get_env_variable('TWITTER_CONSUMER_KEY'), consumer_secret=get_env_variable('TWITTER_CONSUMER_SERCRET'), access_token_key=get_env_variable('TWITTER_ACCESS_TOKEN_KEY'), access_token_secret=get_env_variable('TWITTER_ACCESS_TOKEN_SECRET')) return api.GetSearch(term=term, geocode=geocode, since_id=since_id, max_id=max_id, until=until, count=count, lang=lang, locale=locale, result_type=result_type, include_entities=include_entities)
def test_get_env_variable_success(): os.environ["MY_ENV_VAR"] = "my-env-var" try: env_var = get_env_variable("MY_ENV_VAR") assert env_var == "my-env-var" assert not env_var == "wrong-var" except Exception: assert False
def test_get_env_variable_success(): os.environ['MY_ENV_VAR'] = 'my-env-var' try: env_var = get_env_variable('MY_ENV_VAR') assert env_var == 'my-env-var' assert not env_var == 'wrong-var' except Exception: assert False
def get_config(env=None): if env is None: try: env = get_env_variable('ENV') except Exception: env = 'development' print('env is not set, using env:', env) if env == 'production': return ProductionConfig() elif env == 'test': return TestConfig() return DevelopmentConfig()
def get_config(env=None): if env is None: try: env = get_env_variable("ENV") except Exception: env = "development" print("env is not set, using env:", env) if env == "production": return ProductionConfig() elif env == "test": return TestConfig() return DevelopmentConfig()
from utils import get_env_variable POSTGRES_URL = get_env_variable('POSTGRES_URL') POSTGRES_USER = get_env_variable('POSTGRES_USER') POSTGRES_PASSWORD = get_env_variable('POSTGRES_PASSWORD') POSTGRES_DB = get_env_variable('POSTGRES_DB') class Config(object): DEBUG = False TESTING = False # SQLAlchemy uri_template = 'postgresql+psycopg2://{user}:{pw}@{url}/{db}' SQLALCHEMY_DATABASE_URI = uri_template.format(user=POSTGRES_USER, pw=POSTGRES_PASSWORD, url=POSTGRES_URL, db=POSTGRES_DB) # Silence the deprecation warning SQLALCHEMY_TRACK_MODIFICATIONS = False # API settings API_PAGINATION_PER_PAGE = 10 class DevelopmentConfig(Config): DEBUG = True class TestConfig(Config): TESTING = True
import utils import users, dispatch from tornado.platform.asyncio import to_asyncio_future, AsyncIOMainLoop import telepot.async import asyncio TELEGRAM_API_TOKEN = utils.get_env_variable('TELEGRAM_API_TOKEN') bot = telepot.async.Bot(TELEGRAM_API_TOKEN) @asyncio.coroutine def telegram_send_message(msg): keyboard = None if msg.get('keyboard'): keyboard = {'keyboard': [msg['keyboard']], 'one_time_keyboard': True, 'resize_keyboard': True} if msg.get('text'): yield bot.sendMessage(msg['chat_id'], msg['text'], reply_markup=keyboard) elif msg.get('location'): yield bot.sendLocation(msg['chat_id'], longitude=msg['location']['longitude'], latitude=msg['location']['latitude'], reply_markup=keyboard) if __name__ == '__main__': @asyncio.coroutine
False, 'Prints the outgoing HTTP request along with headers and body.') flags.DEFINE_string( 'credentials_file', 'taskqueue.dat', 'File where you want to store the auth credentails for later user') # Set up a Flow object to be used if we need to authenticate. This # sample uses OAuth 2.0, and we set up the OAuth2WebServerFlow with # the information it needs to authenticate. Note that it is called # the Web Server Flow, but it can also handle the flow for native # applications <http://code.google.com/apis/accounts/docs/OAuth2.html#IA> # The client_id client_secret are copied from the Identity tab on # the Google APIs Console <http://code.google.com/apis/console> FLOW = OAuth2WebServerFlow( client_id=get_env_variable('GOOGLE_CLIENT_ID'), client_secret=get_env_variable('GOOGLE_CLIENT_SECRET'), scope='https://www.googleapis.com/auth/taskqueue', user_agent='taskqueue-cmdline-sample/1.0') class TaskQueueClient: """Class to setup connection with taskqueue API.""" def __init__(self): if not FLAGS.project_name: raise app.UsageError('You must specify a project name' ' using the "--project_name" flag.') discovery_uri = ( FLAGS.api_host + 'discovery/v1/apis/{api}/{apiVersion}/rest') logger.info(discovery_uri)
from utils import get_env_variable DB_USER = get_env_variable("DATABASE_USER") DB_PASSWORD = get_env_variable("DATABASE_PASSWORD") DB_NAME = get_env_variable("DATABASE_NAME") DB_HOST = get_env_variable("DATABASE_HOST") DB_PORT = get_env_variable("DATABASE_PORT") HTTP_SCORING_AUTH = get_env_variable("HEADERS")
################### # HEROKU SETTINGS # ################### # Parse database configuration from $DATABASE_URL DATABASES['default'] = dj_database_url.config() # Honor the 'X-Forwarded-Proto' header for request.is_secure() SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') ################## # DJANGO # ################## # TODO: check SCERETE_KEY here SECRET_KEY = get_env_variable('SITE', 'SECRET_KEY', False) NEVERCACHE_KEY = get_env_variable('SITE', 'NEVERCACHE_KEY', False) REST_FRAMEWORK = { 'DEFAULT_AUTHENTICATION_CLASSES': ( 'rest_framework.authentication.SessionAuthentication', ) } AUTH_USER_MODEL = 'authentication.Account' ################### # S3 STATIC FILES # ################### # We are not using SSL. You can change this to "True" and to "https:" # if you are using a SSL Certificate.
# configurate the logger logging.basicConfig( filename='etl_app.log', filemode='a', format='%(asctime)s,%(msecs)d | %(name)s | %(levelname)s | %(message)s', datefmt='%Y-%m-%d %H:%M:%S', level=logging.INFO) logger = logging.getLogger('loading_csv_to_mongo') logger.info("Running Loadind from CSV to Mongo") load_env_variables_from_file(env_file='.env') # the path to zip file DATA_ZIP_PATH = get_env_variable('DATA_ZIP_PATH') # the env variable for mongo MONGO_DATABASE = get_env_variable('MONGO_DATABASE') MONGO_HOST = get_env_variable('MONGO_HOST') MONGO_PORT = int(get_env_variable('MONGO_PORT')) DATE_COLS = { 'order': ['created_at', 'date_tz', 'updated_at', 'fulfillment_date_tz'], 'user': ['created_at', 'updated_at'] } # if the maximum rows per batch is set in env, use it, otherwise set a default value try: MAX_RECORD_NUMBER = get_env_variable('MAX_RECORD_NUMBER') except KeyError:
from base import * from utils import get_env_variable DEBUG=False DATABASES = { 'default': { 'ENGINE': 'django.db.backends.postgresql_psycopg2', 'NAME': 'django', 'USER': '******', 'PASSWORD': get_env_variable("SBOMB_PROD_PASSWORD"), 'HOST': 'localhost', 'PORT': '', } }
import utils import users, dispatch from tornado.platform.asyncio import to_asyncio_future, AsyncIOMainLoop import telepot. async import asyncio TELEGRAM_API_TOKEN = utils.get_env_variable('TELEGRAM_API_TOKEN') bot = telepot. async .Bot(TELEGRAM_API_TOKEN) @asyncio.coroutine def telegram_send_message(msg): keyboard = None if msg.get('keyboard'): keyboard = { 'keyboard': [msg['keyboard']], 'one_time_keyboard': True, 'resize_keyboard': True } if msg.get('text'): yield bot.sendMessage(msg['chat_id'], msg['text'], reply_markup=keyboard) elif msg.get('location'): yield bot.sendLocation(msg['chat_id'], longitude=msg['location']['longitude'], latitude=msg['location']['latitude'], reply_markup=keyboard)
return api.GetUserTimeline(screen_name='onekiloparsec', exclude_replies=True, include_rts=False) def search_tweets(term=None, geocode=None, since_id=None, max_id=None, until=None, count=15, lang=None, locale=None, result_type='popular', include_entities=None): """ returns twitter feed with settings as described below, contains all related twitter settings """ api = twitter.Api(consumer_key=get_env_variable('TWITTER_CONSUMER_KEY'), consumer_secret=get_env_variable('TWITTER_CONSUMER_SERCRET'), access_token_key=get_env_variable('TWITTER_ACCESS_TOKEN_KEY'), access_token_secret=get_env_variable('TWITTER_ACCESS_TOKEN_SECRET')) return api.GetSearch(term=term, geocode=geocode, since_id=since_id, max_id=max_id, until=until, count=count, lang=lang, locale=locale, result_type=result_type, include_entities=include_entities) if __name__ == "__main__": # for tweet in search_tweets(term="#Tunisia", include_entities=True): # print json.dumps(tweet, cls=TweetEncoder, sort_keys=True, indent=4) # print '\n\n\n' payload = {"grant_type":"client_credentials"} secret = get_env_variable('TWITTER_CONSUMER_KEY')+":"+get_env_variable('TWITTER_CONSUMER_SERCRET') post_headers = {'Authorization': "Basic "+base64.b64encode(secret)} post_resp = requests.post("https://api.twitter.com/oauth2/token", headers=post_headers, data=payload) resp_data = json.loads(post_resp.text) get_headers = {'Authorization': "Bearer "+resp_data["access_token"]} get_resp = requests.get("https://api.twitter.com/1.1/search/tweets.json?q=%23Tunisia&result_type=popular", headers=get_headers) print get_resp.text
from utils import get_env_variable POSTGRES_URL = get_env_variable("POSTGRES_URL") POSTGRES_USER = get_env_variable("POSTGRES_USER") POSTGRES_PASSWORD = get_env_variable("POSTGRES_PASSWORD") POSTGRES_DB = get_env_variable("POSTGRES_DB") class Config(object): DEBUG = False TESTING = False # SQLAlchemy uri_template = "postgresql+psycopg2://{user}:{pw}@{url}/{db}" SQLALCHEMY_DATABASE_URI = uri_template.format( user=POSTGRES_USER, pw=POSTGRES_PASSWORD, url=POSTGRES_URL, db=POSTGRES_DB ) # Silence the deprecation warning SQLALCHEMY_TRACK_MODIFICATIONS = False # API settings API_PAGINATION_PER_PAGE = 10 class DevelopmentConfig(Config): DEBUG = True class TestConfig(Config): TESTING = True
import os from utils import get_env_variable basedir = os.path.abspath(os.path.dirname(__file__)) POSTGRES_URL = get_env_variable('POSTGRES_URL') POSTGRES_USER = get_env_variable('POSTGRES_USER') POSTGRES_PASSWORD = get_env_variable('POSTGRES_PASSWORD') POSTGRES_DB = get_env_variable('POSTGRES_DB') SITE_DOMAIN = get_env_variable('SITE_DOMAIN') SITE_PROTOCOL = get_env_variable('SITE_PROTOCOL') SITE_PORT = get_env_variable('SITE_PORT') class Config(object): SECRET_KEY = 'super secret key' CELERY_BROKER_URL = 'redis://*****:*****@{url}/{db}' SQLALCHEMY_DATABASE_URI = uri_template.format( user=POSTGRES_USER,
For the full list of settings and their values, see https://docs.djangoproject.com/en/1.8/ref/settings/ """ # Build paths inside the project like this: os.path.join(BASE_DIR, ...) import os from utils import get_env_variable BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/1.8/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! SECRET_KEY = get_env_variable("SBOMB_SECRET_KEY") # SECURITY WARNING: don't run with debug turned on in production! DEBUG = False ALLOWED_HOSTS = ["*"] # Application definition INSTALLED_APPS = ( #'registration', #'suit', 'django.contrib.admin', 'django.contrib.auth', 'django.contrib.contenttypes', 'django.contrib.sessions',
logger = logging.getLogger('etl_service') logger.info("Running ETL Service") DEFAULT_MAX_DATE = datetime(2020, 1, 1, 0, 0, 0) DEFAULT_MAX_ROWS = 1000 BLANK_USER_DICT = { 'user_first_name': None, 'user_last_name': None, 'user_merchant_id': None, 'user_phone_number': None, 'user_created_at': None, 'user_updated_at': None } MONGO_DATABASE = get_env_variable('MONGO_DATABASE') MONGO_HOST = get_env_variable('MONGO_HOST') MONGO_PORT = int(get_env_variable('MONGO_PORT')) # Config the postgres connection # the values of those depend on your setup POSTGRES_URL = get_env_variable("POSTGRES_URL") POSTGRES_USER = get_env_variable("POSTGRES_USER") POSTGRES_PW = get_env_variable("POSTGRES_PW") POSTGRES_DB = get_env_variable("POSTGRES_DB") app = Flask(__name__) crontab = Crontab(app) # connect to mongodb, read the data # establish the connection
""" import os from .celery import app as celery_app from utils import get_env_variable # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! SECRET_KEY = get_env_variable("DJ_SECRET_KEY") # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True ALLOWED_HOSTS = [] # Application definition INSTALLED_APPS = [ 'django.contrib.admin', 'django.contrib.auth', 'django.contrib.contenttypes', 'django.contrib.sessions', 'django.contrib.messages', 'django.contrib.staticfiles',
def test_get_env_variable_fail(): try: get_env_variable("not-existing-env") assert False, "Expected KeyError exception was skipped" except Exception: assert True, "Expected KeyError exception was raised"
# https://docs.djangoproject.com/en/1.8/howto/static-files/ ################### # HEROKU SETTINGS # ################### # Parse database configuration from $DATABASE_URL DATABASES['default'] = dj_database_url.config() # Honor the 'X-Forwarded-Proto' header for request.is_secure() SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') ################## # DJANGO # ################## # TODO: check SCERETE_KEY here SECRET_KEY = get_env_variable('SITE', 'SECRET_KEY', False) NEVERCACHE_KEY = get_env_variable('SITE', 'NEVERCACHE_KEY', False) REST_FRAMEWORK = { 'DEFAULT_AUTHENTICATION_CLASSES': ('rest_framework.authentication.SessionAuthentication', ) } AUTH_USER_MODEL = 'authentication.Account' ################### # S3 STATIC FILES # ################### # We are not using SSL. You can change this to "True" and to "https:" # if you are using a SSL Certificate. AWS_S3_SECURE_URLS = False AWS_S3_ENCRYPTION = False
from .base import * import os from utils import get_env_variable, nth_parent_directory # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = nth_parent_directory(3, __file__) # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! SECRET_KEY = get_env_variable('LEARNEASE_SECRET_KEY') # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True INTERNAL_IPS = ['127.0.0.1'] ALLOWED_HOSTS = [] # Application definition INSTALLED_APPS = [ 'words.apps.WordsConfig', 'django.contrib.admin', 'django.contrib.auth',