def create_vocab(url, filename, player):
vocab_path = get_dir_cfg()['vocab_path']
filename = local_dir + vocab_path + filename + ".txt"
if not is_on_file(filename):
response = requests.get(url,
headers={
'groups': 'ROLE_AUTOMATION,',
'username': '******'
})
values = response.json()
logger.info('vocab is not on file')
make_dir(filename)
with open(filename, 'w') as f:
for value in values:
label = value['id']
if label is not None:
f.write(label)
f.write('\n')
# now put file away.
head, tail = os.path.split(filename)
put_aws_file_with_path(vocab_path, tail)
write_filenames_index_from_filename(filename)
else:
head, tail = os.path.split(filename)
logger.info('get from aws ' + tail)
#need to load the file from aws potentially
get_aws_file(vocab_path, tail)
return filename
import util.model_utils as model_utils
import util.cache_utils as cache_utils
import dataset.match_dataset as match_dataset
import util.receipt_utils as receipt_utils
import util.training_utils as training_utils
import util.train_history_utils as train_history_utils
from util.config_utils import get_dir_cfg
from util.config_utils import get_learning_cfg
import logging
logger = logging.getLogger(__name__)
local_dir = get_dir_cfg()['local']
history_file = get_dir_cfg()['player_saves_train_history_file']
def train(player, receipt):
learning_cfg = get_learning_cfg("saves")
history = train_history_utils.init_history('in progress', learning_cfg)
training_utils.train(data_range=training_utils.create_data_range(
learning_cfg=learning_cfg, history_file=history_file),
label='saves',
label_values=match_dataset.SAVES,
model_dir="saves",
train_path=training_utils.create_train_path(),
receipt=receipt,
history=history,
history_file=history_file)
import util.model_utils as model_utils
import util.cache_utils as cache_utils
import dataset.match_dataset as match_dataset
import util.receipt_utils as receipt_utils
import util.training_utils as training_utils
import util.train_history_utils as train_history_utils
from util.config_utils import get_dir_cfg
from util.config_utils import get_learning_cfg
import logging
logger = logging.getLogger(__name__)
local_dir = get_dir_cfg()['local']
history_file = get_dir_cfg()['player_yellow_card_train_history_file']
def train(player, receipt):
learning_cfg = get_learning_cfg("yellow")
history = train_history_utils.init_history('in progress', learning_cfg)
training_utils.train(data_range=training_utils.create_data_range(
learning_cfg=learning_cfg, history_file=history_file),
label='yellow',
label_values=match_dataset.CARDS,
model_dir="yellow",
train_path=training_utils.create_train_path(),
receipt=receipt,
history=history,
history_file=history_file)
import train.player_goals_train as player_goals_train
import train.player_assists_train as player_assists_train
import train.player_minutes_train as player_minutes_train
import train.player_conceded_train as player_conceded_train
import train.player_red_card_train as player_red_card_train
import train.player_yellow_card_train as player_yellow_card_train
from util.config_utils import get_dir_cfg
import json
import logging
import threading
import traceback
app = Flask(__name__)
logging.basicConfig(filename=get_dir_cfg()['local'] + 'predictor.log',
level=logging.NOTSET)
logger = logging.getLogger(__name__)
local_dir = get_dir_cfg()['local']
if __name__ == "__main__":
app.run(host='0.0.0.0')
def set_init(init):
if init == 'true':
return True
else:
return False
import tensorflow as tf
from util.file_utils import get_indexes
from util.file_utils import get_aws_file
from util.config_utils import get_dir_cfg
import logging
logger = logging.getLogger(__name__)
local_dir = get_dir_cfg()['local']
def init_models(model_dir):
logger.info('calling init')
indexes = get_indexes(local_dir + model_dir)
for attribute, value in indexes.items():
if (value['active'] == True):
get_aws_file(model_dir + '/', attribute)
indexes = get_indexes(local_dir + model_dir + '/eval')
for attribute, value in indexes.items():
if (value['active'] == True):
get_aws_file(model_dir + '/eval/', attribute)
def create(feature_columns, classes, model_dir, learning_cfg, init):
logger.info('model dir for classifier ' + local_dir + model_dir)
logger.info('tensorflow version ' + tf.__version__)
if init:
from util.index_utils import process_index, read_index
import os.path
import os
import requests
import logging
import csv
import time
import boto3
from botocore.exceptions import ClientError
logger = logging.getLogger(__name__)
s3_client = boto3.client('s3')
aws = get_dir_cfg()['aws']
aws_url = get_dir_cfg()['aws_url']
aws_bucket = get_dir_cfg()['aws_bucket']
local_dir = get_dir_cfg()['local']
def on_finish(tf_models_dir, aws_model_dir):
logger.info(' write index '+tf_models_dir)
write_filenames_index(tf_models_dir)
try:
write_filenames_index(tf_models_dir+'/eval')
except Exception as e:
logger.info('eval dir not created')
logger.info(' put aws files '+aws_model_dir)
import json
import os.path
import logging
import datetime
from util.config_utils import get_dir_cfg
logger = logging.getLogger(__name__)
local_dir = get_dir_cfg()['local']
vocab_file = get_dir_cfg()['vocab_history_file']
def write_history(filename, history):
logger.info('opening ' + filename)
with open(local_dir + filename, 'w') as outfile:
json.dump(history, outfile)
def read_history(filename):
if os.path.isfile(local_dir + filename):
with open(local_dir + filename) as f:
return json.load(f)
else:
return {}
def get_history(filename, key):
history = read_history(filename)
if key in history:
return history[key]
import util.model_utils as model_utils
import util.cache_utils as cache_utils
import dataset.match_dataset as match_dataset
import util.receipt_utils as receipt_utils
import util.training_utils as training_utils
import util.train_history_utils as train_history_utils
from util.config_utils import get_dir_cfg
from util.config_utils import get_learning_cfg
import logging
logger = logging.getLogger(__name__)
local_dir = get_dir_cfg()['local']
history_file = get_dir_cfg()['player_conceded_train_history_file']
def train(player, receipt):
learning_cfg = get_learning_cfg("conceded")
history = train_history_utils.init_history('in progress', learning_cfg)
training_utils.train(data_range=training_utils.create_data_range(
learning_cfg=learning_cfg, history_file=history_file),
label='conceded',
label_values=match_dataset.CONCEDED,
model_dir="conceded",
train_path=training_utils.create_train_path(),
receipt=receipt,
history=history,
history_file=history_file)
def create_train_path():
train_path = get_dir_cfg()['train_path']
return train_path
