def get_tweets():
"""
returns twitter feed with settings as described below, contains all related twitter settings
"""
api = twitter.Api(consumer_key=get_env_variable('TWITTER_CONSUMER_KEY'),
consumer_secret=get_env_variable('TWITTER_CONSUMER_SERCRET'),
access_token_key=get_env_variable('TWITTER_ACCESS_TOKEN_KEY'),
access_token_secret=get_env_variable('TWITTER_ACCESS_TOKEN_SECRET'))
return api.GetUserTimeline(screen_name='onekiloparsec', exclude_replies=True, include_rts=False)
def search_tweets(term=None, geocode=None, since_id=None, max_id=None, until=None, count=15, lang=None, locale=None, result_type='popular', include_entities=None):
"""
returns twitter feed with settings as described below, contains all related twitter settings
"""
api = twitter.Api(consumer_key=get_env_variable('TWITTER_CONSUMER_KEY'),
consumer_secret=get_env_variable('TWITTER_CONSUMER_SERCRET'),
access_token_key=get_env_variable('TWITTER_ACCESS_TOKEN_KEY'),
access_token_secret=get_env_variable('TWITTER_ACCESS_TOKEN_SECRET'))
return api.GetSearch(term=term, geocode=geocode, since_id=since_id, max_id=max_id, until=until,
count=count, lang=lang, locale=locale, result_type=result_type, include_entities=include_entities)
def test_get_env_variable_success():
os.environ["MY_ENV_VAR"] = "my-env-var"
try:
env_var = get_env_variable("MY_ENV_VAR")
assert env_var == "my-env-var"
assert not env_var == "wrong-var"
except Exception:
assert False
def test_get_env_variable_success():
os.environ['MY_ENV_VAR'] = 'my-env-var'
try:
env_var = get_env_variable('MY_ENV_VAR')
assert env_var == 'my-env-var'
assert not env_var == 'wrong-var'
except Exception:
assert False
def get_config(env=None):
if env is None:
try:
env = get_env_variable('ENV')
except Exception:
env = 'development'
print('env is not set, using env:', env)
if env == 'production':
return ProductionConfig()
elif env == 'test':
return TestConfig()
return DevelopmentConfig()
def get_config(env=None):
if env is None:
try:
env = get_env_variable("ENV")
except Exception:
env = "development"
print("env is not set, using env:", env)
if env == "production":
return ProductionConfig()
elif env == "test":
return TestConfig()
return DevelopmentConfig()
from utils import get_env_variable
POSTGRES_URL = get_env_variable('POSTGRES_URL')
POSTGRES_USER = get_env_variable('POSTGRES_USER')
POSTGRES_PASSWORD = get_env_variable('POSTGRES_PASSWORD')
POSTGRES_DB = get_env_variable('POSTGRES_DB')
class Config(object):
DEBUG = False
TESTING = False
# SQLAlchemy
uri_template = 'postgresql+psycopg2://{user}:{pw}@{url}/{db}'
SQLALCHEMY_DATABASE_URI = uri_template.format(user=POSTGRES_USER,
pw=POSTGRES_PASSWORD,
url=POSTGRES_URL,
db=POSTGRES_DB)
# Silence the deprecation warning
SQLALCHEMY_TRACK_MODIFICATIONS = False
# API settings
API_PAGINATION_PER_PAGE = 10
class DevelopmentConfig(Config):
DEBUG = True
class TestConfig(Config):
TESTING = True
import utils
import users, dispatch
from tornado.platform.asyncio import to_asyncio_future, AsyncIOMainLoop
import telepot.async
import asyncio
TELEGRAM_API_TOKEN = utils.get_env_variable('TELEGRAM_API_TOKEN')
bot = telepot.async.Bot(TELEGRAM_API_TOKEN)
@asyncio.coroutine
def telegram_send_message(msg):
keyboard = None
if msg.get('keyboard'):
keyboard = {'keyboard': [msg['keyboard']],
'one_time_keyboard': True,
'resize_keyboard': True}
if msg.get('text'):
yield bot.sendMessage(msg['chat_id'], msg['text'],
reply_markup=keyboard)
elif msg.get('location'):
yield bot.sendLocation(msg['chat_id'],
longitude=msg['location']['longitude'],
latitude=msg['location']['latitude'],
reply_markup=keyboard)
if __name__ == '__main__':
@asyncio.coroutine
False,
'Prints the outgoing HTTP request along with headers and body.')
flags.DEFINE_string(
'credentials_file',
'taskqueue.dat',
'File where you want to store the auth credentails for later user')
# Set up a Flow object to be used if we need to authenticate. This
# sample uses OAuth 2.0, and we set up the OAuth2WebServerFlow with
# the information it needs to authenticate. Note that it is called
# the Web Server Flow, but it can also handle the flow for native
# applications <http://code.google.com/apis/accounts/docs/OAuth2.html#IA>
# The client_id client_secret are copied from the Identity tab on
# the Google APIs Console <http://code.google.com/apis/console>
FLOW = OAuth2WebServerFlow(
client_id=get_env_variable('GOOGLE_CLIENT_ID'),
client_secret=get_env_variable('GOOGLE_CLIENT_SECRET'),
scope='https://www.googleapis.com/auth/taskqueue',
user_agent='taskqueue-cmdline-sample/1.0')
class TaskQueueClient:
"""Class to setup connection with taskqueue API."""
def __init__(self):
if not FLAGS.project_name:
raise app.UsageError('You must specify a project name'
' using the "--project_name" flag.')
discovery_uri = (
FLAGS.api_host + 'discovery/v1/apis/{api}/{apiVersion}/rest')
logger.info(discovery_uri)
from utils import get_env_variable
DB_USER = get_env_variable("DATABASE_USER")
DB_PASSWORD = get_env_variable("DATABASE_PASSWORD")
DB_NAME = get_env_variable("DATABASE_NAME")
DB_HOST = get_env_variable("DATABASE_HOST")
DB_PORT = get_env_variable("DATABASE_PORT")
HTTP_SCORING_AUTH = get_env_variable("HEADERS")
###################
# HEROKU SETTINGS #
###################
# Parse database configuration from $DATABASE_URL
DATABASES['default'] = dj_database_url.config()
# Honor the 'X-Forwarded-Proto' header for request.is_secure()
SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')
##################
# DJANGO #
##################
# TODO: check SCERETE_KEY here
SECRET_KEY = get_env_variable('SITE', 'SECRET_KEY', False)
NEVERCACHE_KEY = get_env_variable('SITE', 'NEVERCACHE_KEY', False)
REST_FRAMEWORK = {
'DEFAULT_AUTHENTICATION_CLASSES': (
'rest_framework.authentication.SessionAuthentication',
)
}
AUTH_USER_MODEL = 'authentication.Account'
###################
# S3 STATIC FILES #
###################
# We are not using SSL. You can change this to "True" and to "https:"
# if you are using a SSL Certificate.
# configurate the logger
logging.basicConfig(
filename='etl_app.log',
filemode='a',
format='%(asctime)s,%(msecs)d | %(name)s | %(levelname)s | %(message)s',
datefmt='%Y-%m-%d %H:%M:%S',
level=logging.INFO)
logger = logging.getLogger('loading_csv_to_mongo')
logger.info("Running Loadind from CSV to Mongo")
load_env_variables_from_file(env_file='.env')
# the path to zip file
DATA_ZIP_PATH = get_env_variable('DATA_ZIP_PATH')
# the env variable for mongo
MONGO_DATABASE = get_env_variable('MONGO_DATABASE')
MONGO_HOST = get_env_variable('MONGO_HOST')
MONGO_PORT = int(get_env_variable('MONGO_PORT'))
DATE_COLS = {
'order': ['created_at', 'date_tz', 'updated_at', 'fulfillment_date_tz'],
'user': ['created_at', 'updated_at']
}
# if the maximum rows per batch is set in env, use it, otherwise set a default value
try:
MAX_RECORD_NUMBER = get_env_variable('MAX_RECORD_NUMBER')
except KeyError:
from base import *
from utils import get_env_variable
DEBUG=False
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.postgresql_psycopg2',
'NAME': 'django',
'USER': '******',
'PASSWORD': get_env_variable("SBOMB_PROD_PASSWORD"),
'HOST': 'localhost',
'PORT': '',
}
}
import utils
import users, dispatch
from tornado.platform.asyncio import to_asyncio_future, AsyncIOMainLoop
import telepot. async
import asyncio
TELEGRAM_API_TOKEN = utils.get_env_variable('TELEGRAM_API_TOKEN')
bot = telepot. async .Bot(TELEGRAM_API_TOKEN)
@asyncio.coroutine
def telegram_send_message(msg):
keyboard = None
if msg.get('keyboard'):
keyboard = {
'keyboard': [msg['keyboard']],
'one_time_keyboard': True,
'resize_keyboard': True
}
if msg.get('text'):
yield bot.sendMessage(msg['chat_id'],
msg['text'],
reply_markup=keyboard)
elif msg.get('location'):
yield bot.sendLocation(msg['chat_id'],
longitude=msg['location']['longitude'],
latitude=msg['location']['latitude'],
reply_markup=keyboard)
return api.GetUserTimeline(screen_name='onekiloparsec', exclude_replies=True, include_rts=False)
def search_tweets(term=None, geocode=None, since_id=None, max_id=None, until=None, count=15, lang=None, locale=None, result_type='popular', include_entities=None):
"""
returns twitter feed with settings as described below, contains all related twitter settings
"""
api = twitter.Api(consumer_key=get_env_variable('TWITTER_CONSUMER_KEY'),
consumer_secret=get_env_variable('TWITTER_CONSUMER_SERCRET'),
access_token_key=get_env_variable('TWITTER_ACCESS_TOKEN_KEY'),
access_token_secret=get_env_variable('TWITTER_ACCESS_TOKEN_SECRET'))
return api.GetSearch(term=term, geocode=geocode, since_id=since_id, max_id=max_id, until=until,
count=count, lang=lang, locale=locale, result_type=result_type, include_entities=include_entities)
if __name__ == "__main__":
# for tweet in search_tweets(term="#Tunisia", include_entities=True):
# print json.dumps(tweet, cls=TweetEncoder, sort_keys=True, indent=4)
# print '\n\n\n'
payload = {"grant_type":"client_credentials"}
secret = get_env_variable('TWITTER_CONSUMER_KEY')+":"+get_env_variable('TWITTER_CONSUMER_SERCRET')
post_headers = {'Authorization': "Basic "+base64.b64encode(secret)}
post_resp = requests.post("https://api.twitter.com/oauth2/token", headers=post_headers, data=payload)
resp_data = json.loads(post_resp.text)
get_headers = {'Authorization': "Bearer "+resp_data["access_token"]}
get_resp = requests.get("https://api.twitter.com/1.1/search/tweets.json?q=%23Tunisia&result_type=popular", headers=get_headers)
print get_resp.text
from utils import get_env_variable
POSTGRES_URL = get_env_variable("POSTGRES_URL")
POSTGRES_USER = get_env_variable("POSTGRES_USER")
POSTGRES_PASSWORD = get_env_variable("POSTGRES_PASSWORD")
POSTGRES_DB = get_env_variable("POSTGRES_DB")
class Config(object):
DEBUG = False
TESTING = False
# SQLAlchemy
uri_template = "postgresql+psycopg2://{user}:{pw}@{url}/{db}"
SQLALCHEMY_DATABASE_URI = uri_template.format(
user=POSTGRES_USER, pw=POSTGRES_PASSWORD, url=POSTGRES_URL, db=POSTGRES_DB
)
# Silence the deprecation warning
SQLALCHEMY_TRACK_MODIFICATIONS = False
# API settings
API_PAGINATION_PER_PAGE = 10
class DevelopmentConfig(Config):
DEBUG = True
class TestConfig(Config):
TESTING = True
import os
from utils import get_env_variable
basedir = os.path.abspath(os.path.dirname(__file__))
POSTGRES_URL = get_env_variable('POSTGRES_URL')
POSTGRES_USER = get_env_variable('POSTGRES_USER')
POSTGRES_PASSWORD = get_env_variable('POSTGRES_PASSWORD')
POSTGRES_DB = get_env_variable('POSTGRES_DB')
SITE_DOMAIN = get_env_variable('SITE_DOMAIN')
SITE_PROTOCOL = get_env_variable('SITE_PROTOCOL')
SITE_PORT = get_env_variable('SITE_PORT')
class Config(object):
SECRET_KEY = 'super secret key'
CELERY_BROKER_URL = 'redis://*****:*****@{url}/{db}'
SQLALCHEMY_DATABASE_URI = uri_template.format(
user=POSTGRES_USER,
For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.8/ref/settings/
"""
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os
from utils import get_env_variable
BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.8/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = get_env_variable("SBOMB_SECRET_KEY")
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = False
ALLOWED_HOSTS = ["*"]
# Application definition
INSTALLED_APPS = (
#'registration',
#'suit',
'django.contrib.admin',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
logger = logging.getLogger('etl_service')
logger.info("Running ETL Service")
DEFAULT_MAX_DATE = datetime(2020, 1, 1, 0, 0, 0)
DEFAULT_MAX_ROWS = 1000
BLANK_USER_DICT = {
'user_first_name': None,
'user_last_name': None,
'user_merchant_id': None,
'user_phone_number': None,
'user_created_at': None,
'user_updated_at': None
}
MONGO_DATABASE = get_env_variable('MONGO_DATABASE')
MONGO_HOST = get_env_variable('MONGO_HOST')
MONGO_PORT = int(get_env_variable('MONGO_PORT'))
# Config the postgres connection
# the values of those depend on your setup
POSTGRES_URL = get_env_variable("POSTGRES_URL")
POSTGRES_USER = get_env_variable("POSTGRES_USER")
POSTGRES_PW = get_env_variable("POSTGRES_PW")
POSTGRES_DB = get_env_variable("POSTGRES_DB")
app = Flask(__name__)
crontab = Crontab(app)
# connect to mongodb, read the data
# establish the connection
"""
import os
from .celery import app as celery_app
from utils import get_env_variable
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = get_env_variable("DJ_SECRET_KEY")
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
ALLOWED_HOSTS = []
# Application definition
INSTALLED_APPS = [
'django.contrib.admin',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles',
def test_get_env_variable_fail():
try:
get_env_variable("not-existing-env")
assert False, "Expected KeyError exception was skipped"
except Exception:
assert True, "Expected KeyError exception was raised"
# https://docs.djangoproject.com/en/1.8/howto/static-files/
###################
# HEROKU SETTINGS #
###################
# Parse database configuration from $DATABASE_URL
DATABASES['default'] = dj_database_url.config()
# Honor the 'X-Forwarded-Proto' header for request.is_secure()
SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')
##################
# DJANGO #
##################
# TODO: check SCERETE_KEY here
SECRET_KEY = get_env_variable('SITE', 'SECRET_KEY', False)
NEVERCACHE_KEY = get_env_variable('SITE', 'NEVERCACHE_KEY', False)
REST_FRAMEWORK = {
'DEFAULT_AUTHENTICATION_CLASSES':
('rest_framework.authentication.SessionAuthentication', )
}
AUTH_USER_MODEL = 'authentication.Account'
###################
# S3 STATIC FILES #
###################
# We are not using SSL. You can change this to "True" and to "https:"
# if you are using a SSL Certificate.
AWS_S3_SECURE_URLS = False
AWS_S3_ENCRYPTION = False
from .base import *
import os
from utils import get_env_variable, nth_parent_directory
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = nth_parent_directory(3, __file__)
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = get_env_variable('LEARNEASE_SECRET_KEY')
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
INTERNAL_IPS = ['127.0.0.1']
ALLOWED_HOSTS = []
# Application definition
INSTALLED_APPS = [
'words.apps.WordsConfig',
'django.contrib.admin',
'django.contrib.auth',