def create_vocab(url, filename, player):
vocab_path = get_dir_cfg()['vocab_path']
filename = local_dir + vocab_path + filename + ".txt"
if not is_on_file(filename):
response = requests.get(url,
headers={
'groups': 'ROLE_AUTOMATION,',
'username': '******'
})
values = response.json()
logger.info('vocab is not on file')
make_dir(filename)
with open(filename, 'w') as f:
for value in values:
label = value['id']
if label is not None:
f.write(label)
f.write('\n')
# now put file away.
head, tail = os.path.split(filename)
put_aws_file_with_path(vocab_path, tail)
write_filenames_index_from_filename(filename)
else:
head, tail = os.path.split(filename)
logger.info('get from aws ' + tail)
#need to load the file from aws potentially
get_aws_file(vocab_path, tail)
return filename
def init_models(model_dir):
logger.info('calling init')
indexes = get_indexes(local_dir + model_dir)
for attribute, value in indexes.items():
if (value['active'] == True):
get_aws_file(model_dir + '/', attribute)
indexes = get_indexes(local_dir + model_dir + '/eval')
for attribute, value in indexes.items():
if (value['active'] == True):
get_aws_file(model_dir + '/eval/', attribute)
def create_vocab(filename, country):
vocab_path = get_dir_cfg()['vocab_path']
vocab_path = vocab_path.replace('<key>', country)
filename = local_dir + vocab_path + filename + ".txt"
head, tail = os.path.split(filename)
logger.info('get from aws ' + tail)
# need to load the file from aws potentially
get_aws_file(vocab_path, tail)
return filename
def create_csv(url, filename, start_date, end_date, aws_path):
logger.info('getting csv data...' + filename)
if is_on_file(filename):
logger.info("csv file already created " + filename)
head, tail = os.path.split(filename)
return get_aws_file(head.replace(local_dir, '') + '/', tail)
else:
data = requests.get(url + '/' + start_date.strftime("%d-%m-%Y") + '/' + end_date.strftime("%d-%m-%Y"),
headers={'groups': 'ROLE_AUTOMATION,', 'username': '******'})
has_data = write_csv(filename, data.json())
logger.info('created csv')
head, tail = os.path.split(filename)
put_aws_file_with_path(aws_path, tail)
write_filenames_index_from_filename(filename)
return has_data
from util.file_utils import put_aws_file
from util.file_utils import get_aws_file
put_aws_file('/home/timmytime/results2.txt')
get_aws_file('', 'results2.txt')