def tweepy_auth(): at = get_key( find_dotenv(), 'TWITTER-ACCESS-TOKEN' ) ats = get_key( find_dotenv(), 'TWITTER-ACCESS-TOKEN-SECRET' ) ck = get_key( find_dotenv(), 'TWITTER-CONSUMER-KEY' ) cs = get_key( find_dotenv(), 'TWITTER-CONSUMER-SECRET' ) auth = tweepy.OAuthHandler(ck, cs) auth.set_access_token(at, ats) api = tweepy.API(auth) return api
def from_cmd_line():
from s3_wrapper.envdefault import EnvDefault, truthy
from dotenv import load_dotenv, find_dotenv
load_dotenv(find_dotenv(usecwd=True))
parser = ArgumentParser()
parser.add_argument('-d', '--db-url', required=True, action=EnvDefault, envvar='MONGO_URI',
help='Mongodb url')
parser.add_argument('-c', '--collection', required=True, action=EnvDefault, envvar='MONGO_COLLECTION',
help='Mongodb collection')
parser.add_argument('-b', '--bucket', action=EnvDefault, envvar='S3_BUCKET',
help='S3 bucket')
parser.add_argument('-a', '--access-key', action=EnvDefault, envvar='S3_ACCESS_KEY',
help='S3 access key')
parser.add_argument('-s', '--secret-key', action=EnvDefault, envvar='S3_SECRET_KEY',
help='S3 secret key')
parser.add_argument('--is-secure', action=EnvDefault, required=False, envvar='S3_SSL', type=truthy, default=False,
help='S3 use ssl')
parser.add_argument('-H', '--host', action=EnvDefault, required=False, envvar='S3_HOST',
help='S3 host')
parser.add_argument('--calling-format', action=EnvDefault, required=False, envvar='S3_CALLING_FORMAT',
help='S3 calling format')
parser.add_argument('files', nargs='*',
help='Files to process')
args = parser.parse_args()
if args.files:
files = args.files
if files[0] == '-':
files = sys.stdin
else:
files = sys.stdin
ingest(files, args)
def __init__(self, *args, **kwargs):
"""
We take the kwargs and apply all of the configuration values
that are provided. They will have precedence over environment
variables.
:param *args: The arguments (that we ignore)
:param **kwargs: The settings collected from kwargs are directly applied
to the configuration.
"""
# database_url: '...' => DATABASE_URL: '...'
kwargs = {k.upper(): v for k, v in kwargs.items()}
# Create our configuration starting with the
self.configuration = kwargs.copy()
self.apps = []
env_file = self.detect('ENV_FILE', default='.env')
if os.path.exists(env_file):
load_dotenv(env_file)
else:
self.set('ENV_FILE', find_dotenv(env_file))
load_dotenv(self.get('ENV_FILE'))
def test_env(self):
# Check that a local .env has been set or that their is a production variable.
env = find_dotenv()
print(env)
load_dotenv(env)
try:
# Python 2
connection_string = os.environ['PYSAGE_CNXN'].decode('utf8')
except AttributeError:
# Python 3
connection_string = os.environ['PYSAGE_CNXN']
print(connection_string)
assert(len(connection_string+' ') > 1)
def test_find_dotenv():
"""
Create a temporary folder structure like the following:
tmpXiWxa5/
└── child1
├── child2
│ └── child3
│ └── child4
└── .env
Then try to automatically `find_dotenv` starting in `child4`
"""
tmpdir = tempfile.mkdtemp()
curr_dir = tmpdir
dirs = []
for f in ['child1', 'child2', 'child3', 'child4']:
curr_dir = os.path.join(curr_dir, f)
dirs.append(curr_dir)
os.mkdir(curr_dir)
child1, child4 = dirs[0], dirs[-1]
# change the working directory for testing
os.chdir(child4)
# try without a .env file and force error
with pytest.raises(IOError):
find_dotenv(raise_error_if_not_found=True, usecwd=True)
# try without a .env file and fail silently
assert find_dotenv(usecwd=True) == ''
# now place a .env file a few levels up and make sure it's found
filename = os.path.join(child1, '.env')
with open(filename, 'w') as f:
f.write("TEST=test\n")
assert find_dotenv(usecwd=True) == filename
def test_find_dotenv(tmp_path):
"""
Create a temporary folder structure like the following:
test_find_dotenv0/
└── child1
├── child2
│ └── child3
│ └── child4
└── .env
Then try to automatically `find_dotenv` starting in `child4`
"""
curr_dir = tmp_path
dirs = []
for f in ['child1', 'child2', 'child3', 'child4']:
curr_dir /= f
dirs.append(curr_dir)
curr_dir.mkdir()
child1, child4 = dirs[0], dirs[-1]
# change the working directory for testing
os.chdir(str(child4))
# try without a .env file and force error
with pytest.raises(IOError):
find_dotenv(raise_error_if_not_found=True, usecwd=True)
# try without a .env file and fail silently
assert find_dotenv(usecwd=True) == ''
# now place a .env file a few levels up and make sure it's found
dotenv_file = child1 / '.env'
dotenv_file.write_bytes(b"TEST=test\n")
assert find_dotenv(usecwd=True) == str(dotenv_file)
def main():
#load_dotenv(os.path.join(os.path.dirname(__file__), ".env"))
load_dotenv(find_dotenv())
api_key=os.environ.get("API_KEY")
#visual_recognition = WatsonVisualRecognition(api_key)
sdk_vr = VisualRecognitionV3('2016-05-20', api_key=api_key)
my_vr = WatsonVisualRecognition(api_key)
'''
image_path = '/Users/joshuazheng/Downloads/Beagle_hero.jpg'
image_file = open('/Users/joshuazheng/Downloads/Beagle_hero.jpg').read()
image_url = 'http://www.sbarro.com/wp-content/uploads/2015/04/12-sbarro-spaghetti-meatballs.jpg'
classifier_id = 'dogs_2117373684'
pprint(my_vr.classify_image(classifier_id, image_file=None, image_url=image_url, threshold=0))
pos_file_paths = ['bundles/moleskine/journaling.zip', 'bundles/moleskine/journaling.zip']
neg_file_path = 'bundles/moleskine/negative.zip'
pos_file_list = [journal, land]
pos_files = {'journ': journal, 'land': land}
pos_names = ['journal', 'land']
files = {
'negative_examples': open('../bundles/moleskine/negative.zip').read(),
'journal_positive_examples': open('../bundles/moleskine/journaling.zip').read(),
'landscape_positive_examples': open('../bundles/moleskine/journaling.zip').read()
}
response = my_vr.create_classifier("mol_script_zipnew", files)
#response = my_vr.create_classifier_from_file("mol_script_file1", pos_names, pos_file_list, neg_file_path)
#import pdb; pdb.set_trace()
print response
print(sdk_vr.get_classifier('beagle_81816899'))
print
print "SDK"
classes = my_vr.list_classifiers()
pprint(classes)
print
print
for c in classes:
info = sdk_vr.get_classifier(c['classifier_id'])
pprint(info)
print
'''
print my_vr.get_classifier('piz_525944347')
def load_dotenv(path=None):
"""Load "dotenv" files in order of precedence to set environment variables.
If an env var is already set it is not overwritten, so earlier files in the
list are preferred over later files.
Changes the current working directory to the location of the first file
found, with the assumption that it is in the top level project directory
and will be where the Python path should import local packages from.
This is a no-op if `python-dotenv`_ is not installed.
.. _python-dotenv: https://github.com/theskumar/python-dotenv#readme
:param path: Load the file at this location instead of searching.
:return: ``True`` if a file was loaded.
.. versionadded:: 1.0
"""
if dotenv is None:
if path or os.path.exists('.env') or os.path.exists('.flaskenv'):
click.secho(
' * Tip: There are .env files present.'
' Do "pip install python-dotenv" to use them',
fg='yellow')
return
if path is not None:
return dotenv.load_dotenv(path)
new_dir = None
for name in ('.env', '.flaskenv'):
path = dotenv.find_dotenv(name, usecwd=True)
if not path:
continue
if new_dir is None:
new_dir = os.path.dirname(path)
dotenv.load_dotenv(path)
if new_dir and os.getcwd() != new_dir:
os.chdir(new_dir)
return new_dir is not None # at least one file was located and loaded
group.add_argument(
'--silent',
dest='silent',
action='store_true',
help="disable logging to stdout.")
args = parser.parse_args()
if args.silent:
log_handler = logbook.NullHandler()
else:
log_handler = logbook.StreamHandler(sys.stdout, level=logbook.lookup_level(args.level.upper()))
with log_handler:
if args.env:
env = find_dotenv(args.env)
log.info('loading environment from {}', env)
load_dotenv(env)
docker_client = docker.from_env(version='auto')
repo = args.repository or os.environ.get('BUILD_REPOSITORY', '')
tags = set(args.tags or {os.environ.get('BUILD_TAG', 'latest')})
if args.will_add_latest:
tags.add('latest')
build_args = {k[10:]: v for k, v in os.environ.items() if k.startswith('BUILD_ARG_')}
path = args.path
if args.is_amazon_mode:
log.info('running in AMAZON mode')
log.info('attempting to get ECR registry token')
def parse_args(description, arguments=[], colwidth_max=125, logging_kwargs={}):
parser = argparse.ArgumentParser(description=description)
arguments += [
dict(name_or_flags="--debug",
dest="level",
action="store_const",
const=logging.DEBUG,
default=logging.INFO,
help="enable debugging"),
dict(name_or_flags="--seed",
default=hash(TODAY.date),
type=int,
help="setup initial seed"),
dict(name_or_flags="--output",
default=os.path.join("output", TODAY.strftime("%Y-%m-%d"),
parser.prog),
help="where to write to")
]
seen = {}
for arg in arguments:
name = arg.pop("name_or_flags")
if name in seen:
logging.debug("Skipping duplicate argument: %s: %s", name, arg)
continue
parser.add_argument(name, **arg)
seen[name] = arg
args = parser.parse_args()
args.prog = parser.prog
args.started_at = TODAY
logging_kwargs = dict(logging_kwargs)
logging_kwargs["level"] = args.level
initialize_logging(**logging_kwargs)
if isinstance(args.output, str) and not os.path.exists(args.output):
os.makedirs(args.output)
# NOTE: Logs every unhandled exception, even without an explicit try-catch.
# Stop silent failures!
if os.environ.get("DJANGO_ENV", "development") != "production":
sys.excepthook = handle_unhandled_exception
# NOTE: For some reason, there appears to be significant variability in
# __score between runs separated by an extended period of time. My best
# guess is that this is due to seeding of the psuedo-RNG. We can either
# explicitly (a) set the same seed and/or (b) re-compute baseline score
# before we get started.
args.seed %= 2**32 - 1
try:
import torch # naive torch install via pip fails on Heroku, so only set this if torch is explicitly installed.
torch.manual_seed(args.seed)
except:
pass
np.random.seed(args.seed)
random.seed(args.seed)
# other setup
pd.options.display.max_colwidth = colwidth_max
dotenv = find_dotenv(usecwd=True)
logging.debug("Loading discovered .env file: %s", dotenv)
load_dotenv(dotenv)
logging.info("Beginning %s CLI: %s", args.prog, args)
return args
import os
import sys
from dotenv import find_dotenv, load_dotenv
# Reference: /scripts/envs.sh.template
# The .env file is optional. Environment variables can be sourced from
# the bash script shown above.
dot_env = find_dotenv()
if dot_env:
load_dotenv(dot_env)
try:
FACEBOOK = os.environ["FACEBOOK"]
FACEBOOK_TV = os.environ["FACEBOOK_TV"]
FILM_COLLECTION = os.environ["FILM_COLLECTION"]
EPISODE_COLLECTION = os.environ["EPISODE_COLLECTION"]
FRAMES_DIR = os.environ["FRAMES_DIR"]
NSFW_MODEL = os.environ["NSFW_MODEL"]
FONTS = os.environ["FONTS"]
TMDB = os.environ["TMDB"]
RANDOMORG = os.environ["RANDOMORG"]
RADARR = os.environ["RADARR"]
RADARR_URL = os.environ["RADARR_URL"]
REQUESTS_JSON = os.environ["REQUESTS_JSON"]
OFFENSIVE_JSON = os.environ["OFFENSIVE_JSON"]
KINOBASE = os.environ["KINOBASE"]
REQUESTS_DB = os.environ["REQUESTS_DB"]
DISCORD_WEBHOOK = os.environ["DISCORD_WEBHOOK"]
DISCORD_WEBHOOK_TEST = os.environ["DISCORD_WEBHOOK_TEST"]
import os
import sys
from dotenv import load_dotenv, find_dotenv
if find_dotenv():
load_dotenv(find_dotenv())
_PATH_ = os.path.dirname(os.path.dirname(__file__))
if _PATH_ not in sys.path:
sys.path.append(_PATH_)
if __name__ == "__main__":
from keras_glow import manager
manager.start()
VERBOSE = True
DEBUG = True
import os
from dotenv import find_dotenv, load_dotenv
load_dotenv(find_dotenv(), verbose=VERBOSE)
SPOTIFY_CLIENT_ID = os.getenv("SPOTIFY_CLIENT_ID")
SPOTIFY_CLIENT_SECRET = os.getenv("SPOTIFY_CLIENT_SECRET")
LASTFM_API_KEY = os.getenv("LASTFM_API_KEY")
LASTFM_SHARED_SECRET = os.getenv("LASTFM_SHARED_SECRET")
#!/usr/bin/env python
# coding: utf-8
# In[1]:
# In[2]:
import psycopg2
import os
# In[3]:
from dotenv import load_dotenv, find_dotenv
load_dotenv(find_dotenv())
dbname = os.environ.get('dbname')
user = os.environ.get('user')
password = os.environ.get('password')
host = os.environ.get('host')
# In[61]:
pg_conn = psycopg2.connect(dbname=dbname,
user=user,
password=password,
host=host)
# In[17]:
pg_curs = pg_conn.cursor()
def main():
import argparse # noqa
common_parser = argparse.ArgumentParser(add_help=False)
common_parser.add_argument("--token", "-t", help="인증 토큰을 설정합니다.")
common_parser.add_argument("--client-id",
"-u",
help="Open API의 client id값을 설정합니다.")
common_parser.add_argument("--client-secret",
"-p",
help="Open API의 client secret값을 설정합니다.")
common_parser.add_argument(
"--blog",
"-b",
help="블로그 이름을 설정합니다. 예) `xvezda.tistory.com` 의 경우 `xvezda`",
)
common_parser.add_argument(
"--verbose",
"-v",
action="count",
default=0,
help="로그의 정보량을 설정합니다. `v`의 갯수에 따라 정보량이 달라집니다.",
)
common_parser.add_argument(
"--version",
"-V",
action="version",
version=__version__,
help="버전 정보를 출력하고 종료합니다.",
)
parser = argparse.ArgumentParser(parents=[common_parser])
subparsers = parser.add_subparsers(dest="command")
info_parser = subparsers.add_parser("info",
parents=[common_parser],
help="자신의 블로그 정보를 가져오는 API 입니다.")
info_parser.add_argument("--post-id",
"-i",
action="append",
help="정보를 가져올 포스트 아이디를 설정합니다.")
info_parser.set_defaults(func=_info_command)
post_parser = subparsers.add_parser("post",
parents=[common_parser],
help="블로그 글을 관리하는 API 입니다.")
# NOTE: Tistory API v1 does not support deleting post.. WHAT?
# post_parser.add_argument("--delete", "-d", action="store_true")
post_parser.add_argument(
"--file",
"-f",
action="append",
help="마크다운 또는 JSON 파일의 경로를 설정합니다. "
"`-` 으로 설정하여 stdin으로 부터 읽어올 수 있습니다.",
)
post_parser.add_argument(
"--demo",
"-D",
action="store_true",
help="블로그에 데모 포스팅을 작성합니다.",
)
post_parser.add_argument("files", nargs="+")
post_parser.set_defaults(func=_post_command)
posts_parser = subparsers.add_parser("posts",
parents=[common_parser],
help="포스트 목록을 가져옵니다.")
posts_parser.add_argument(
"--encode-url",
"-e",
action="store_true",
help="포스트 주소를 URL 인코딩 형태로 보여줍니다.",
)
posts_parser.set_defaults(func=_posts_command)
category_parser = subparsers.add_parser("category",
parents=[common_parser],
help="블로그 카테고리를 정보를 가져오는 API 입니다.")
category_parser.add_argument("--name",
"-n",
action="append",
default=[],
help="카테고리 이름")
category_parser.add_argument("--label",
"-l",
action="append",
default=[],
help="카테고리 라벨")
category_parser.add_argument("--id",
"-i",
action="append",
default=[],
help="카테고리 아이디")
category_parser.add_argument("--parent",
"-m",
action="append",
default=[],
help="부모 카테고리 아이디")
category_parser.set_defaults(func=_category_command)
comment_parser = subparsers.add_parser("comment",
parents=[common_parser],
help="블로그 댓글을 관리하는 API 입니다.")
comment_parser.add_argument("--list",
"-l",
action="store_true",
help="댓글 목록을 가져옵니다.")
comment_parser.add_argument("--new",
"-n",
action="store_true",
help="최근 댓글 목록을 가져옵니다.")
comment_parser.add_argument("--delete",
"-d",
action="store_true",
help="댓글을 삭제합니다.")
comment_parser.add_argument("--parent-id",
"-m",
type=str,
help="대댓글을 작성할 댓글의 아이디.")
comment_parser.add_argument("--comment-id",
"-i",
type=str,
help="댓글의 아이디.")
comment_parser.add_argument("--post-id",
"-A",
required=True,
type=str,
help="댓글을 작성할 포스트의 아이디.")
comment_parser.add_argument(
"content",
nargs="?",
type=str,
help="댓글의 내용. 설정하지 않으면 stdin으로 부터 읽어옵니다.",
)
comment_parser.set_defaults(func=_comment_command)
import_parser = subparsers.add_parser("import",
parents=[common_parser],
help="게시된 게시글을 불러오는 명령어입니다.")
import_parser.add_argument(
"--output-dir",
"-O",
help="게시글을 저장할 디렉토리를 설정합니다. "
"기본값은 블로그의 이름을 사용합니다.",
)
import_parser.set_defaults(func=_import_command)
args = parser.parse_args()
try:
# dotenv 모듈이 설치된 경우 `.env` 파일로 작성된 환경변수를 자동으로 읽어옵니다.
from dotenv import load_dotenv, find_dotenv # noqa
load_dotenv(find_dotenv(usecwd=True), verbose=(args.verbose > 0))
except ImportError:
pass
# 로그가 작성되는 단계를 설정합니다.
if args.verbose == 1:
logger.setLevel(logging.INFO)
elif args.verbose == 2:
logger.setLevel(logging.DEBUG)
if not args.command:
parser.error("too few arguments")
try:
args.func(args)
except Exception as err:
if args.verbose > 0:
# 오류 발생시 로그의 정보량 단계가 지정된 경우 콜스택 정보를 보여줍니다.
print(traceback.format_exc(), file=sys.stderr)
parser.error(u(err))
parser.print_help()
def make_submission_CV(**kwargs):
"""
StateFarm competition:
Training a simple model on the whole training set. Save it for future use
args: model (keras model)
**kwargs (dict) keyword arguments that specify the model hyperparameters
"""
# Roll out the parameters
data_file = kwargs["data_file"]
tr_weights_path = kwargs["tr_weights_path"]
list_folds = kwargs["list_folds"]
# Load env variables in (in .env file at the root of the project)
load_dotenv(find_dotenv())
# Compile model.
for fold in list_folds:
pretr_weights_file = os.path.join(tr_weights_path, 'resnet_weights_fold%s.pickle' % fold)
assert os.path.isfile(pretr_weights_file)
# Get mean values from the pretr_weights_file
with open(pretr_weights_file, "rb") as f:
net_train = pickle.load(f)
mean_values = net_train["mean_image"]
# Define model
input_var = T.tensor4('inputs')
network = build_model(input_var, usage="inference", pretr_weights_file=pretr_weights_file)
prediction = lasagne.layers.get_output(network, deterministic=True)
predict_function = theano.function([input_var], prediction)
with h5py.File(data_file, "r") as hf:
X_test = hf["test_data"]
id_test = hf["test_id"][:]
y_test_pred = np.zeros((X_test.shape[0], 10))
# Split image list into num_chunks chunks
chunk_size = 32
num_imgs = X_test.shape[0]
num_chunks = num_imgs / chunk_size
list_chunks = np.array_split(np.arange(num_imgs), num_chunks)
for index, chunk_idx in enumerate(list_chunks):
sys.stdout.write("\rFold %s Processing img %s/%s" %
(fold, index + 1, X_test.shape[0]))
sys.stdout.flush()
X_test_batch = X_test[chunk_idx.tolist()]
# To GBR and normalize
X_test_batch = X_test_batch[:, ::-1, :, :]
X_test_batch = (X_test_batch - mean_values)
y_test_pred[chunk_idx] = predict_function(X_test_batch.astype(np.float32))
print("")
# Create a list of columns
list_col = ["img", "c0", "c1", "c2", "c3", "c4", "c5", "c6", "c7", "c8", "c9"]
# Get id_test
id_test_fold = id_test.copy().reshape(-1, 1)
# Save predictions of the fold
subm_data = np.hstack([id_test_fold, y_test_pred])
df_subm = pd.DataFrame(subm_data, columns=list_col)
# Get directory corresponding to the weights
subm_dir = os.path.basename(tr_weights_path)
subm_name = 'resnet_fold%s.csv' % fold
subm_file = os.path.join(subm_dir, subm_name)
df_subm.to_csv(subm_file, index=False)
import http.client
import json
import requests
from os import environ as env
from dotenv import load_dotenv, find_dotenv
ENV_FILE = find_dotenv()
if ENV_FILE:
load_dotenv(ENV_FILE)
AUTH0_DOMAIN = env.get('AUTH0_DOMAIN')
PAYLOAD = env.get('PAYLOAD')
def get_access_token():
conn = http.client.HTTPSConnection(AUTH0_DOMAIN)
payload = PAYLOAD
headers = {'content-type': "application/json"}
conn.request("POST", "/oauth/token", payload, headers)
res = conn.getresponse()
data = res.read()
return data.decode("utf-8")
def access_api_with_auth():
token_dict = json.loads(get_access_token())
import os
from django.core.exceptions import ImproperlyConfigured
from dotenv import load_dotenv, find_dotenv
ROOT_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
BASE_DIR = os.path.dirname(ROOT_DIR)
TEMPLATES_DIR = os.path.join(BASE_DIR, 'templates')
load_dotenv(find_dotenv(), encoding='utf-8')
def get_env_variable(var_name, default=None):
try:
return os.environ[var_name]
except KeyError:
if default is None:
error_msg= '필수 환경 변수 {}가 설정되지 않았습니다.'.format(var_name)
raise ImproperlyConfigured(error_msg)
return default
SECRET_KEY = get_env_variable('SECRET_KEY')
STATIC_URL = '/static/'
STATIC_ROOT = os.path.join(ROOT_DIR, '.static')
MEDIA_URL = '/media/'
MEDIA_ROOT = os.path.join(ROOT_DIR, '.media')
STATIC_DIR = os.path.join(BASE_DIR, 'static')
STATICFILES_DIRS = [
def __init__(self):
load_dotenv(find_dotenv())
self.account_url = os.environ["AZURE_CONTAINERREGISTRY_URL"]
VARIABLE=value
```
Environment variables take precendence, meaning the values set in the file will be overriden
by any values previously set in your environment.
.. WARNING::
Do not upload `.env` files containing private tokens to version control! If you use this package
as a dependency of your project, please ensure to include the `.env` in your `.gitignore`.
"""
import os
import dotenv
import pdoc
dotenv.load_dotenv(dotenv.find_dotenv(usecwd=True))
class Env:
"""Provides access to environment variables.
Ensures variables are reloaded when environment changes during runtime.
Additionally allows to expose documented instance variables in pdoc
generated output.
"""
@property
def MBED_API_AUTH_TOKEN(self) -> str:
"""Token to use when accessing online API.
Mbed Targets uses the online mbed board database at os.mbed.com as its data source.
from dotenv import dotenv_values, find_dotenv import lldb env_vars = dotenv_values(find_dotenv()) env_vars = ["%s=%s" % (k, v) for k, v in env_vars.items()] launch_info = lldb.target.GetLaunchInfo() launch_info.SetEnvironmentEntries(env_vars, True) lldb.target.SetLaunchInfo(launch_info)
def setup_test():
"""Load dotenv for comparison as part of test setup."""
load_dotenv(find_dotenv())
GatewayTemplateAuthenticationKey, GatewayPatchTemplateAuthenticationKey,
GatewayTemplateGatewayTypeConnectTemplate, GatewayPortIdentity, Gateway,
GatewayBfdConfigTemplate, GatewayBfdPatchTemplate)
# from ibm_cloud_networking_services.direct_link_v1 import (
# GatewayMacsecConfigTemplate)
# from ibm_cloud_networking_services.direct_link_v1 import (
# GatewayMacsecConfigTemplatePrimaryCak)
# from ibm_cloud_networking_services.direct_link_v1 import (
# GatewayMacsecConfigPatchTemplate)
# from ibm_cloud_networking_services.direct_link_v1 import (
# GatewayMacsecConfigPatchTemplateFallbackCak)
from dotenv import load_dotenv, find_dotenv
# load the .env file containing your environment variables
try:
load_dotenv(find_dotenv(filename="dl.env"))
except:
raise unittest.SkipTest('no dl.env file loaded, skipping...')
class TestDirectLinkV1(unittest.TestCase):
""" Test class for DirectLink sdk functions """
@unittest.skip("skipping due to tavis timeout of 10m")
def setUp(self):
""" test case setup """
self.endpoint = os.getenv("DL_SERVICES_SERVICE_URL")
if self.endpoint is None:
self.skipTest("configuration file unavailable")
self.version = datetime.date.today()
env_file = None
# Set development environment variables from env.py file
try:
from pinry.settings.env import vars
for k, v in vars.items():
os.environ[k] = v
env_file = 'pinry/settings/env.py'
except ImportError:
pass
# Set development environment variables from .env file
try:
from dotenv import load_dotenv, find_dotenv
env_file = find_dotenv('.env')
load_dotenv(env_file)
except ImportError:
pass
# Confirm environment variable file
if not env_file:
raise Exception ("Missing and environment file: \n"
"option1 : pinry/settings/env.py with vars = {VARIABLE:'value'}\n"
"option2 : pinry/settings/.env VARIABLE=value")
else:
print 'Loaded %s environment variables from : %s' % (DATA_SOURCE, env_file)
# Load DATABASE_URL
try:
var_prefix = "%s_" % DATA_SOURCE[:1] if DATA_SOURCE in ['PRODUCTION', 'STAGING'] else ''
from typing import Pattern
from typing import Type
from typing import TypeVar
from typing import Union
from typing import cast
import appdirs
from box import Box
from dotenv import dotenv_values
from dotenv import find_dotenv
from ..common import config as fixit_config
appdirs.system = "linux2"
ENV_FILE: str = find_dotenv(".env", usecwd=True)
_env = dotenv_values(ENV_FILE, verbose=True)
class DotDict(MutableMapping):
"""
A `dict` that also supports attribute ("dot") access. Think of this as an extension
to the standard python `dict` object. **Note**: while any hashable object can be added to
a `DotDict`, _only_ valid Python identifiers can be accessed with the dot syntax; this excludes
strings which begin in numbers, special characters, or double underscores.
:param dict init_dict: dictionary to initialize the `DotDict` with
:param kwargs: key, value pairs with which to initialize the DotDict
.. :code-block: python
>>> dotdict = DotDict({'a': 34}, b=56, c=set())
import os
import dotenv
try:
dotenv_file = dotenv.find_dotenv('.env', raise_error_if_not_found=True)
except IOError:
print("Kunde inte hitta .env-fil.")
dotenv.load_dotenv(dotenv_file)
def get(key: str):
return os.getenv(key)
#!/usr/bin/env python3
# The "dotenv" module does not work in Python 2
import sys
if sys.version_info < (3, 0):
print("This script requires Python 3.x.")
sys.exit(1)
# Imports
import os
import dotenv
import psycopg2
# Import environment variables
dotenv.load_dotenv(dotenv.find_dotenv())
# Variables
user = os.getenv("DB_USER")
password = os.getenv("DB_PASS")
host = os.getenv("DB_HOST")
if host == "":
host = "localhost"
port = os.getenv("DB_PORT")
if port == "":
port = "5432"
database = os.getenv("DB_NAME")
# Connect to the PostgreSQL database
conn = psycopg2.connect(
host=host,
# update token regularly to avoid session expire
def update_token():
global spotify, token
try:
token = util.prompt_for_user_token(spotify_username, scope, client_id,
client_secret, redirect_uri)
except (AttributeError, JSONDecodeError):
os.remove(f".cache-{spotify_username}")
token = util.prompt_for_user_token(spotify_username, scope, client_id,
client_secret, redirect_uri)
spotify = spotipy.Spotify(auth=token)
# load environment variables from .env file
dotenv_path = find_dotenv()
load_dotenv(dotenv_path)
# set Spotify API
client_id = os.environ.get("client_id")
client_secret = os.environ.get("client_secret")
redirect_uri = os.environ.get("redirect_uri")
spotify_username = os.environ.get("spotify_username")
scope = os.environ.get("scope")
# create database connection
connection = sqlite3.connect('spotify.sqlite3')
c = connection.cursor()
liked_playlist = ''
disliked_playlist = 'disliked'
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
from __future__ import print_function
import os
import platform
import posixpath
import re
import serial.serialutil
import click
import dotenv
# Load AMPY_PORT et al from .ampy file
# Performed here because we need to beat click's decorators.
config = dotenv.find_dotenv(filename='.ampy', usecwd=True)
if config:
dotenv.load_dotenv(dotenv_path=config)
import ampy.files as files
import ampy.pyboard as pyboard
_board = None
def windows_full_port_name(portname):
# Helper function to generate proper Windows COM port paths. Apparently
# Windows requires COM ports above 9 to have a special path, where ports below
# 9 are just referred to by COM1, COM2, etc. (wacky!) See this post for
# more info and where this code came from:
# http://eli.thegreenplace.net/2009/07/31/listing-all-serial-ports-on-windows-with-python/
"""
Server side of the app
"""
import os
from flask import Flask, send_from_directory, json
from flask_socketio import SocketIO
from flask_cors import CORS
from flask_sqlalchemy import SQLAlchemy
from dotenv import load_dotenv, find_dotenv
import sqlalchemy
load_dotenv(find_dotenv()) # This is to load your env variables from .env
APP = Flask(__name__, static_folder="./build/static")
# Point SQLAlchemy to your Heroku database
APP.config["SQLALCHEMY_DATABASE_URI"] = os.getenv("DATABASE_URL")
# Gets rid of a warning
APP.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False
DB = SQLAlchemy(APP)
# IMPORTANT: This must be AFTER creating db variable to prevent
# circular import issues
import models # pylint: disable=C0413
DB.create_all()
CORS = CORS(APP, resources={r"/*": {"origins": "*"}})
NAMES = []
SOCKETIO = SocketIO(APP,
cors_allowed_origins="*",
json=json,
def getenv_variables():
# find .env automagically by walking up directories until it's found, then
# load up the .env entries as environment variables
load_dotenv(find_dotenv())
import datetime
from pathlib import Path
from typing import Dict
import pandas as pd
from dotenv import load_dotenv, find_dotenv
from pymongo.database import Database
from .utils import extract_region_rows, extract_columns
load_dotenv(find_dotenv(".env.local"))
def get_region_summary_df(xls: pd.ExcelFile) -> pd.DataFrame:
region_summary_original = pd.read_excel(xls, 'Wzrost w województwach')
return pd.DataFrame({
"cases":
extract_region_rows(region_summary_original, from_row=6, to_row=23),
"deaths":
extract_region_rows(region_summary_original, from_row=49, to_row=66),
"recovers":
extract_region_rows(region_summary_original, from_row=89, to_row=106),
}).reset_index().rename(columns={
6: "date",
"level_1": "region"
})
def get_tests_df(xls: pd.ExcelFile) -> pd.DataFrame:
tests_original = pd.read_excel(xls, "Testy")
return extract_columns(
"""
This is the main Python file that sets up rendering and templating
for Techtonica.org
"""
import os
from dateutil.parser import parse
from dotenv import find_dotenv, load_dotenv
from eventbrite import Eventbrite
from flask import Flask, redirect, render_template, url_for
from flask_sslify import SSLify
load_dotenv(find_dotenv(usecwd=True))
# We fetch our constants by taking them from environment variables
# defined in the .env file.
EVENTBRITE_OAUTH_TOKEN = os.environ["EVENTBRITE_OAUTH_TOKEN"]
# Instantiate the Eventbrite API client.
eb = Eventbrite(EVENTBRITE_OAUTH_TOKEN)
app = Flask(__name__)
sslify = SSLify(app)
class Event(object):
def __init__(self, event_dict):
self.title = event_dict["name"]["text"]
self.url = event_dict["url"]
self.location_title = event_dict["venue"]["name"]
self.address = event_dict["venue"]["address"][
def __init__(self, connection_string=''):
""" If update_cache then make sure you keep updating from the database"""
load_dotenv(find_dotenv())
if connection_string == '':
connection_string = get_default_connection_string()
self.update_cache()
from markups import *
import sys
import os
from dotenv import load_dotenv, find_dotenv
from telebot import TeleBot
from plotters import GraphPlotter, FigurePlotter
from database import db, User, State, GraphPlotterModel, FigurePlotterModel
from utils import is_number, parse_shape
if not load_dotenv(find_dotenv()):
sys.exit(1)
db.create_tables([User, GraphPlotterModel, FigurePlotterModel])
bot = TeleBot(os.getenv('BOT_TOKEN'))
@bot.message_handler(commands=['start'])
def start(msg):
user = User.get_or_none(User.uid == msg.from_user.id)
if not user:
User.create(uid=msg.from_user.id)
resp = "Welcome! I will help you with plotting graphs and geometric figures. ✍️"
else:
resp = "Welcome back! Try plotting something! 🎓"
bot.send_message(
msg.from_user.id,
resp,
import time
import gzip
import StringIO
import re
import random
import types
from dotenv import load_dotenv, find_dotenv
from bs4 import BeautifulSoup
reload(sys)
sys.setdefaultencoding('utf-8')
# Load config from .env file
# TODO: Error handling
try:
load_dotenv(find_dotenv(usecwd=True))
base_url = os.environ.get('BASE_URL')
results_per_page = int(os.environ.get('RESULTS_PER_PAGE'))
except:
print "ERROR: Make sure you have .env file with proper config"
sys.exit(1)
user_agents = list()
# results from the search engine
# basically include url, title,content
class SearchResult:
def __init__(self):
self.url = ''
def make_submission_CV(model_name, **kwargs):
"""
StateFarm competition:
Training a simple model on the whole training set. Save it for future use
args: model (keras model)
**kwargs (dict) keyword arguments that specify the model hyperparameters
"""
# Roll out the parameters
data_file = kwargs["data_file"]
tr_weights_path = kwargs["tr_weights_path"]
model_archi_file = kwargs["model_archi_file"]
normalisation_style = kwargs["normalisation_style"]
start_fold = kwargs["start_fold"]
# Load env variables in (in .env file at the root of the project)
load_dotenv(find_dotenv())
# Compile model.
model = model_from_json(open(model_archi_file).read())
model.name = model_name
list_y_test_pred = []
for fold in range(start_fold, 6):
weights_file = os.path.join(tr_weights_path, "%s_weights_fold%s.h5" % (model.name, fold))
model.load_weights(weights_file)
model.compile(optimizer="sgd", loss='categorical_crossentropy')
with h5py.File(data_file, "r") as hf:
X_test = hf["test_data"]
id_test = hf["test_id"][:]
list_pred = []
chunk_size = 10000
num_imgs = X_test.shape[0]
# Split image list into num_chunks chunks
num_chunks = num_imgs / chunk_size
list_chunks = np.array_split(np.arange(num_imgs), num_chunks)
# Loop over chunks
for index, chunk_idx in enumerate(list_chunks):
sys.stdout.write("\rFold %s Processing chunk %s/%s" %
(fold, index + 1, len(list_chunks)))
sys.stdout.flush()
X_test_batch = X_test[chunk_idx, :, :, :]
X_test_batch = normalisation(X_test_batch, normalisation_style)
y_test_pred = model.predict(X_test_batch, batch_size=16, verbose=0)
list_pred.append(y_test_pred)
print("")
# Combine all chunks
y_test_pred = np.vstack(list_pred)
# Get id_test
id_test_fold = id_test.copy().reshape(-1, 1)
# Save predictions of the fold
subm_data = np.hstack([id_test_fold, y_test_pred])
df_subm = pd.DataFrame(subm_data, columns=["img", "c0", "c1", "c2", "c3",
"c4", "c5", "c6", "c7", "c8", "c9"])
# Get directory corresponding to the weights
subm_dir = os.path.dirname(tr_weights_path)
subm_name = model.name + '_fold%s.csv' % fold
subm_file = os.path.join(subm_dir, subm_name)
df_subm.to_csv(subm_file, index=False)
list_y_test_pred.append(y_test_pred)
y_test_pred = np.mean(list_y_test_pred, 0)
id_test = id_test.reshape(-1, 1)
subm_data = np.hstack([id_test, y_test_pred])
df_subm = pd.DataFrame(subm_data, columns=["img", "c0", "c1", "c2", "c3",
"c4", "c5", "c6", "c7", "c8", "c9"])
now = datetime.datetime.now()
# Get directory corresponding to the weights
subm_dir = os.path.dirname(tr_weights_path)
subm_name = model.name + '_' + str(now.strftime("%Y-%m-%d-%H-%M")) + ".csv"
subm_file = os.path.join(subm_dir, subm_name)
df_subm.to_csv(subm_file, index=False)
def load_dotenv():
env_path = dotenv.find_dotenv(usecwd=True)
if len(env_path) > 0:
os.environ['DOT_ENV_DIRECTORY'] = str(pathlib.Path(env_path).parent)
dotenv.load_dotenv(dotenv_path=env_path, override=True)
import os
import pickle
import numpy as np
import pandas as pd
from os import path
import seaborn as sns
from operator import add
from scipy import sparse, io
import matplotlib.pyplot as plt
from dotenv import load_dotenv, find_dotenv
from mpl_toolkits.basemap import Basemap
%matplotlib inline
dotenv_path = find_dotenv()
load_dotenv(dotenv_path)
RAW_DATA_DIR = os.environ.get("RAW_DATA_DIR")
train = pd.read_csv(path.join(RAW_DATA_DIR, 'gender_age_train.csv'))
events = pd.read_csv(path.join(RAW_DATA_DIR, 'events.csv'), parse_dates=['timestamp'])
app_events = pd.read_csv(path.join(RAW_DATA_DIR, 'app_events.csv'))
app_labels = pd.read_csv(path.join(RAW_DATA_DIR, 'app_labels.csv'))
phone = pd.read_csv(path.join(RAW_DATA_DIR, 'phone_brand_device_model.csv'))
train['device_id'].value_counts()
events['device_id'].value_counts()
phone['device_id'].value_counts()
events['event_id'].value_counts()
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import sys
import imaplib
import getpass
import email
import datetime
from email.header import decode_header
import requests
import json
import time
from dotenv import load_dotenv, find_dotenv
import os
load_dotenv(find_dotenv(), override=True)
class Vendor(object):
def __init__(self, name, keywords, say):
self.name = name
self.keywords = keywords
self.say = say
def toString(self):
return 'name: "{:s}", keywords: {:s}, say: "{:s}"'.format(self.name, self.keywords, self.say)
class FoodVendors(object):
def __init__(self):
self.items = []
def searchKeywords(self, subject, keywords):
import sys
import glob
import time
import dweepy
import tweepy
import datetime
import sqlite3
import RPi.GPIO as GPIO
from dotenv import load_dotenv, find_dotenv, get_key
os.system('modprobe w1-gpio')
os.system('modprobe w1-therm')
#base_dir = '/sys/bus/w1/devices/'
base_dir = get_key( find_dotenv(), 'PROBE-BASEDIR' ).lower()
GPIO.setmode(GPIO.BCM)
GPIO.setup(7,GPIO.IN, pull_up_down=GPIO.PUD_UP)
last_temp = {}
conn = None
curs = None
try:
conn=sqlite3.connect('/home/pi/TempPi/temppi.db')
curs = conn.cursor()
except:
print ("No Database")
def read_temp_raw(device_file):
def cross_validate_inmemory(**kwargs):
"""
StateFarm competition:
Training set has 26 unique drivers. We do 26 fold CV where
a driver is alternatively singled out to be the validation set
Load the whole train data in memory for faster operations
args: model (keras model)
**kwargs (dict) keyword arguments that specify the model hyperparameters
"""
# Roll out the parameters
nb_classes = kwargs["nb_classes"]
batch_size = kwargs["batch_size"]
n_batch_per_epoch = kwargs["n_batch_per_epoch"]
nb_epoch = kwargs["nb_epoch"]
prob = kwargs["prob"]
data_file = kwargs["data_file"]
semi_super_file = kwargs["semi_super_file"]
list_folds = kwargs["list_folds"]
weak_labels = kwargs["weak_labels"]
experiment = kwargs["experiment"]
# Load env variables in (in .env file at the root of the project)
load_dotenv(find_dotenv())
# Load env variables
model_dir = os.path.expanduser(os.environ.get("MODEL_DIR"))
data_dir = os.path.expanduser(os.environ.get("DATA_DIR"))
mean_values = np.load("../../data/external/resnet_mean_values.npy")
# Output path where we store experiment log and weights
model_dir = os.path.join(model_dir, "ResNet")
# Create if it does not exist
general_utils.create_dir(model_dir)
# Automatically determine experiment name
list_exp = glob.glob(model_dir + "/*")
# Create the experiment dir and weights dir
if experiment:
exp_dir = exp_dir = os.path.join(model_dir, experiment)
else:
exp_dir = os.path.join(model_dir, "Experiment_%s" % len(list_exp))
general_utils.create_dir(exp_dir)
# Batch generator
DataAug = batch_utils.AugDataGenerator(data_file,
batch_size=batch_size,
prob=prob,
dset="train",
maxproc=4,
num_cached=60,
random_augm=False,
hdf5_file_semi=semi_super_file)
DataAug.add_transform("h_flip")
DataAug.add_transform("random_rot", angle=40)
DataAug.add_transform("random_tr", tr_x=40, tr_y=40)
DataAug.add_transform("random_blur", kernel_size=5)
DataAug.add_transform("random_crop", min_crop_size=140, max_crop_size=160)
epoch_size = n_batch_per_epoch * batch_size
general_utils.pretty_print("Load all data...")
with h5py.File(data_file, "r") as hf:
X = hf["train_data"][:, :, :, :]
y = hf["train_label"][:].astype(np.int32)
try:
for fold in list_folds:
min_valid_loss = 100
# Save losses
list_train_loss = []
list_valid_loss = []
# Load valid data in memory for fast error evaluation
idx_valid = hf["valid_fold%s" % fold][:]
idx_train = hf["train_fold%s" % fold][:]
X_valid = X[idx_valid]
y_valid = y[idx_valid]
# Normalise
X_valid = X_valid[:, ::-1, :, :]
X_valid = (X_valid - mean_values).astype(np.float32)
# Define model
input_var = T.tensor4('inputs')
target_var = T.matrix('targets')
network = build_model(input_var, usage="train")
prediction = lasagne.layers.get_output(network)
loss = lasagne.objectives.categorical_crossentropy(prediction, target_var)
loss = loss.mean()
params = lasagne.layers.get_all_params(network, trainable=True)
updates = lasagne.updates.nesterov_momentum(loss, params, learning_rate=5E-4, momentum=0.9)
train_fn = theano.function([input_var, target_var], loss, updates=updates)
test_prediction = lasagne.layers.get_output(network, deterministic=True)
test_loss = lasagne.objectives.categorical_crossentropy(test_prediction, target_var)
test_loss = test_loss.mean()
val_fn = theano.function([input_var, target_var], test_loss)
# Loop over epochs
for e in range(nb_epoch):
# Initialize progbar and batch counter
progbar = generic_utils.Progbar(epoch_size)
batch_counter = 1
l_train_loss = []
l_valid_loss = []
start = time.time()
for X_train, y_train in DataAug.gen_batch_inmemory(X, y, idx_train=idx_train):
if True:
general_utils.plot_batch(X_train, np.argmax(y_train, 1), batch_size)
# Normalise
X_train - X_train[:, ::-1, :, :]
X_train = (X_train - mean_values).astype(np.float32)
# Train
train_loss = train_fn(X_train, y_train.astype(np.float32))
l_train_loss.append(train_loss)
batch_counter += 1
progbar.add(batch_size, values=[("train loss", train_loss)])
if batch_counter >= n_batch_per_epoch:
break
print("")
print('Epoch %s/%s, Time: %s' % (e + 1, nb_epoch, time.time() - start))
# Split image list into num_chunks chunks
chunk_size = batch_size
num_imgs = X_valid.shape[0]
num_chunks = num_imgs / chunk_size
list_chunks = np.array_split(np.arange(num_imgs), num_chunks)
# Loop over chunks
for chunk_idx in list_chunks:
X_b, y_b = X_valid[chunk_idx].astype(np.float32), y_valid[chunk_idx]
y_b = np_utils.to_categorical(y_b, nb_classes=nb_classes).astype(np.float32)
valid_loss = val_fn(X_b, y_b)
l_valid_loss.append(valid_loss)
train_loss = float(np.mean(l_train_loss)) # use float to make it json saveable
valid_loss = float(np.mean(l_valid_loss)) # use float to make it json saveable
print("Train loss:", train_loss, "valid loss:", valid_loss)
list_train_loss.append(train_loss)
list_valid_loss.append(valid_loss)
# Record experimental data in a dict
d_log = {}
d_log["fold"] = fold
d_log["nb_classes"] = nb_classes
d_log["batch_size"] = batch_size
d_log["n_batch_per_epoch"] = n_batch_per_epoch
d_log["nb_epoch"] = nb_epoch
d_log["epoch_size"] = epoch_size
d_log["prob"] = prob
d_log["augmentator_config"] = DataAug.get_config()
d_log["train_loss"] = list_train_loss
d_log["valid_loss"] = list_valid_loss
json_file = os.path.join(exp_dir, 'experiment_log_fold%s.json' % fold)
general_utils.save_exp_log(json_file, d_log)
# Only save the best epoch
if valid_loss < min_valid_loss:
min_valid_loss = valid_loss
trained_weights_path = os.path.join(exp_dir, 'resnet_weights_fold%s.pickle' % fold)
model = {
'values': lasagne.layers.get_all_param_values(network),
'mean_image': mean_values
}
pickle.dump(model, open(trained_weights_path, 'wb'), protocol=-1)
except KeyboardInterrupt:
pass
#!/usr/bin/env python
import os
import sys
import dotenv
if __name__ == '__main__':
# Load a .env file
dotenv.load_dotenv(dotenv.find_dotenv())
# Set up configuration modules
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'settings')
configuration_name = os.getenv('ENVIRONMENT', 'Development')
os.environ.setdefault('DJANGO_CONFIGURATION', configuration_name)
# Call the Django command
from configurations.management import execute_from_command_line
execute_from_command_line(sys.argv)
import pymysql
import os
import sys
from dotenv import load_dotenv, find_dotenv
load_dotenv(find_dotenv(), override=True)
sql = "select stop_lat, zone_id, stop_lon, stop_id, stop_name, location_type, stopID_short from stops"
db = pymysql.connect(host=os.getenv("HOST"), port=3306 , user=os.getenv("USER"), passwd=os.getenv("PASSWORD"), db="website")
cursor = db.cursor()
cursor.execute(sql)
db.commit()
Data = cursor.fetchall()
for i in range(0, len(Data)):
try:
index=Data[i][3].find('DB')+2
stop=Data[i][3][index:]
stop=int(stop)
sql = 'UPDATE stops SET stopID_short ='+str(stop)+' WHERE stop_id ="'+Data[i][3]+'"'
db = pymysql.connect(host="csi420-01-vm9.ucd.ie", port=3306 , user="******", passwd="comp47360jnnd", db="website")
cursor = db.cursor()
cursor.execute(sql)
db.commit()
except Exception as e:
print(i)
print(e)
cursor.close()
def __init__(self):
load_dotenv(find_dotenv('config.env'))
self.bucket_name = self.get_bucket_name()
self.finder = PathFinder()
def main():
load_dotenv(find_dotenv())
ansible_password = os.environ.get("ANSIBLE_VAULT_PASSWORD", None)
if ansible_password is None:
raise AnsiblePasswordNotSetException()
return ansible_password
def initialize_environment_variables():
dotenv.load_dotenv(dotenv.find_dotenv(usecwd=True))
def cross_validate_inmemory(model_name, **kwargs):
"""
StateFarm competition:
Training set has 26 unique drivers. We do 26 fold CV where
a driver is alternatively singled out to be the validation set
Load the whole train data in memory for faster operations
args: model (keras model)
**kwargs (dict) keyword arguments that specify the model hyperparameters
"""
# Roll out the parameters
nb_classes = kwargs["nb_classes"]
batch_size = kwargs["batch_size"]
n_batch_per_epoch = kwargs["n_batch_per_epoch"]
nb_epoch = kwargs["nb_epoch"]
prob = kwargs["prob"]
do_plot = kwargs["do_plot"]
data_file = kwargs["data_file"]
semi_super_file = kwargs["semi_super_file"]
pretr_weights_file = kwargs["pretr_weights_file"]
normalisation_style = kwargs["normalisation_style"]
weak_labels = kwargs["weak_labels"]
objective = kwargs["objective"]
experiment = kwargs["experiment"]
start_fold = kwargs["start_fold"]
# Load env variables in (in .env file at the root of the project)
load_dotenv(find_dotenv())
# Load env variables
model_dir = os.path.expanduser(os.environ.get("MODEL_DIR"))
data_dir = os.path.expanduser(os.environ.get("DATA_DIR"))
# Output path where we store experiment log and weights
model_dir = os.path.join(model_dir, model_name)
# Create if it does not exist
general_utils.create_dir(model_dir)
# Automatically determine experiment name
list_exp = glob.glob(model_dir + "/*")
# Create the experiment dir and weights dir
if experiment:
exp_dir = os.path.join(model_dir, experiment)
else:
exp_dir = os.path.join(model_dir, "Experiment_%s" % len(list_exp))
general_utils.create_dir(exp_dir)
# Compile model.
# opt = RMSprop(lr=5E-6, rho=0.9, epsilon=1e-06)
opt = SGD(lr=5e-4, decay=1e-6, momentum=0.9, nesterov=True)
# opt = Adam(lr=1E-5, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
# Batch generator
DataAug = batch_utils.AugDataGenerator(data_file,
batch_size=batch_size,
prob=prob,
dset="train",
maxproc=4,
num_cached=60,
random_augm=False,
hdf5_file_semi=semi_super_file)
DataAug.add_transform("h_flip")
# DataAug.add_transform("v_flip")
# DataAug.add_transform("fixed_rot", angle=40)
DataAug.add_transform("random_rot", angle=40)
# DataAug.add_transform("fixed_tr", tr_x=40, tr_y=40)
DataAug.add_transform("random_tr", tr_x=40, tr_y=40)
# DataAug.add_transform("fixed_blur", kernel_size=5)
DataAug.add_transform("random_blur", kernel_size=5)
# DataAug.add_transform("fixed_erode", kernel_size=4)
DataAug.add_transform("random_erode", kernel_size=3)
# DataAug.add_transform("fixed_dilate", kernel_size=4)
DataAug.add_transform("random_dilate", kernel_size=3)
# DataAug.add_transform("fixed_crop", pos_x=10, pos_y=10, crop_size_x=200, crop_size_y=200)
DataAug.add_transform("random_crop", min_crop_size=140, max_crop_size=160)
# DataAug.add_transform("hist_equal")
# DataAug.add_transform("random_occlusion", occ_size_x=100, occ_size_y=100)
epoch_size = n_batch_per_epoch * batch_size
general_utils.pretty_print("Load all data...")
with h5py.File(data_file, "r") as hf:
X = hf["train_data"][:, :, :, :]
y = hf["train_label"][:].astype(np.uint8)
y = np_utils.to_categorical(y, nb_classes=nb_classes) # Format for keras
try:
for fold in range(start_fold, 8):
# for fold in np.random.permutation(26):
min_valid_loss = 100
# Save losses
list_train_loss = []
list_valid_loss = []
# Load valid data in memory for fast error evaluation
idx_valid = hf["valid_fold%s" % fold][:]
idx_train = hf["train_fold%s" % fold][:]
X_valid = X[idx_valid]
y_valid = y[idx_valid]
# Normalise
X_valid = normalisation(X_valid, normalisation_style)
# Compile model
general_utils.pretty_print("Compiling...")
model = models.load(model_name,
nb_classes,
X_valid.shape[-3:],
pretr_weights_file=pretr_weights_file)
model.compile(optimizer=opt, loss=objective)
# Save architecture
json_string = model.to_json()
with open(os.path.join(data_dir, '%s_archi.json' % model.name), 'w') as f:
f.write(json_string)
for e in range(nb_epoch):
# Initialize progbar and batch counter
progbar = generic_utils.Progbar(epoch_size)
batch_counter = 1
l_train_loss = []
start = time.time()
for X_train, y_train in DataAug.gen_batch_inmemory(X, y, idx_train=idx_train):
if do_plot:
general_utils.plot_batch(X_train, np.argmax(y_train, 1), batch_size)
# Normalise
X_train = normalisation(X_train, normalisation_style)
train_loss = model.train_on_batch(X_train, y_train)
l_train_loss.append(train_loss)
batch_counter += 1
progbar.add(batch_size, values=[("train loss", train_loss)])
if batch_counter >= n_batch_per_epoch:
break
print("")
print('Epoch %s/%s, Time: %s' % (e + 1, nb_epoch, time.time() - start))
y_valid_pred = model.predict(X_valid, verbose=0, batch_size=16)
train_loss = float(np.mean(l_train_loss)) # use float to make it json saveable
valid_loss = log_loss(y_valid, y_valid_pred)
print("Train loss:", train_loss, "valid loss:", valid_loss)
list_train_loss.append(train_loss)
list_valid_loss.append(valid_loss)
# Record experimental data in a dict
d_log = {}
d_log["fold"] = fold
d_log["nb_classes"] = nb_classes
d_log["batch_size"] = batch_size
d_log["n_batch_per_epoch"] = n_batch_per_epoch
d_log["nb_epoch"] = nb_epoch
d_log["epoch_size"] = epoch_size
d_log["prob"] = prob
d_log["optimizer"] = opt.get_config()
d_log["augmentator_config"] = DataAug.get_config()
d_log["train_loss"] = list_train_loss
d_log["valid_loss"] = list_valid_loss
json_file = os.path.join(exp_dir, 'experiment_log_fold%s.json' % fold)
general_utils.save_exp_log(json_file, d_log)
# Only save the best epoch
if valid_loss < min_valid_loss:
min_valid_loss = valid_loss
trained_weights_path = os.path.join(exp_dir, '%s_weights_fold%s.h5' % (model.name, fold))
model.save_weights(trained_weights_path, overwrite=True)
except KeyboardInterrupt:
pass
import hashlib
import json
import os
import time
from collections import OrderedDict
from decimal import *
import boto3
import requests
from dotenv import find_dotenv, load_dotenv
# Process environment variables
de = find_dotenv()
if de is not None:
load_dotenv(de)
name = os.environ['AUTHORIZE_NAME']
transactionKey = os.environ['AUTHORIZE_TRANSACTION_KEY']
url = os.environ['AUTHORIZE_URL']
kms_client = boto3.client('kms')
try:
name = kms_client.decrypt(
CiphertextBlob=name.decode('base64')
)['Plaintext']
transactionKey = kms_client.decrypt(
CiphertextBlob=transactionKey.decode('base64')
)['Plaintext']
url = kms_client.decrypt(
CiphertextBlob=url.decode('base64')
def setUp(self):
load_dotenv(find_dotenv())
def __init__(self):
load_dotenv(find_dotenv())
self.host = os.environ.get("host")
self.dname = os.environ.get("dname")
self.user = os.environ.get("user")
self.password = os.environ.get("password")
# -*- coding: utf-8 -*-
import os
from dotenv import load_dotenv, find_dotenv
ENV = os.environ.get('ENV') or 'development'
if ENV == 'development':
load_dotenv(find_dotenv('./.env'))
API_TOKEN = os.environ.get('API_TOKEN') or ''
def main(config):
"""
指定時間ごとにBotを実行してトレードを行う
:param object config: 設定情報
"""
# ログ出力
logger.info("start")
# オプション取得
logger.info(options.debug)
# ライブラリ呼び出し
bot = Core(config)
slack = SlackApi()
# 環境変数設定
load_dotenv(find_dotenv())
while True:
try:
# 取引所情報を取得する
# bitmex = bot.get_exchange()
current_ticker = bot.get_ticker()
close_price = current_ticker["last"]
# 予測情報を取得する
forecast_data = bot.get_forecast()
recommend_pos = bot.forecast_position(forecast_data)
message = "現在価格: {0}".format(close_price)
slack.notify(message)
logger.info(message)
message = "forecast.high: {0}".format(
forecast_data["high"].high_price)
slack.notify(message)
logger.info(message)
message = "forecast.low: {0}".format(
forecast_data["low"].low_price)
slack.notify(message)
logger.info(message)
message = "recommend_pos: {0}".format(recommend_pos)
slack.notify(message)
logger.info(message)
# ポジションを持っているかチェックする
positions = bot.get_positions()
logger.info("positions: {0}".format(positions))
if len(positions) <= 0:
result = bot.do_order_check(recommend_pos, forecast_data)
if result:
message = "{0}から{1}で注文する！".format(
close_price, recommend_pos)
slack.notify(message)
logger.info(message)
order = bot.order(recommend_pos, 1000)
logger.info("order: {0}".format(order))
else:
message = "ポジションを取らない！"
slack.notify(message)
else:
message = "現在損益: {0:.8f} XBT".format(bot.current_profit)
slack.notify(message)
result = bot.release_check(recommend_pos)
if result:
order = bot.close_order(positions[-1])
logger.info("order: {0}".format(order))
message = """
ポジションを解消！
損益: {0:.8f} XBT
手数料: {1:.8f} XBT
トレード回数: {2} 回
最大利益: {3:.8f} XBT
最大損失: {4:.8f} XBT
""".format(
bot.total_profit,
bot.total_tax,
len(bot.profits),
max(bot.profits),
min(bot.profits))
slack.notify(message)
else:
message = "ポジションを維持！"
slack.notify(message)
del forecast_data
gc.collect()
except Exception as e:
logger.exception(e)
time.sleep(sleep_second)
# -*- coding: utf-8 -*-
__version__ = '0.1'
from flask import Flask
from flask_sqlalchemy import SQLAlchemy
from dotenv import load_dotenv, find_dotenv
load_dotenv(find_dotenv())
from datetime import timedelta
import logging
app = Flask('app')
app.config.from_pyfile('settings.py', silent=True)
logging.basicConfig(filename='moringa.log', level=logging.INFO)
logging.basicConfig(filename='moringa.bug', level=logging.DEBUG)
logging.basicConfig(filename='moringa.err', level=logging.ERROR)
app.config['SQLALCHEMY_DATABASE_URI'] = 'mysql://*****:*****@localhost/moringa'
db = SQLAlchemy(app)
# Create all tables in the engine. This is equivalent to "Create Table"
# statements in raw SQL.
from api import routes
from api import voice
"""
The flask env configurations for the entire flask project.
The different configurations are combined into dic with development, production, and testing objects.
"""
# coding=utf-8
import datetime
import os
from dotenv import load_dotenv, find_dotenv
basedir = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
ENV_FILE = find_dotenv()
if ENV_FILE:
load_dotenv(ENV_FILE)
LOCAL_ENV_FILE = find_dotenv('.env.local')
if LOCAL_ENV_FILE:
load_dotenv(LOCAL_ENV_FILE)
class Config(object):
"""
Initial Configurations for the Flask App
"""
CSRF_ENABLED = True
DEBUG = False
SECRET_KEY = os.environ.get(
"SECRET_KEY",
"seLe3MKZ3FeKytUP6nvbjVuuHSwcd4UPcyd5v8jZjTtwjxNY7n5LwdjXAnPqHkxXh2gv3WQNWK34CQgByeGxJtfSXyMY8gtQm7KcENN2bZkfxDKbWasTG43sDeGYnHwx"
)
expires = datetime.timedelta(days=30)