def create_vocab(url, filename, player): vocab_path = get_dir_cfg()['vocab_path'] filename = local_dir + vocab_path + filename + ".txt" if not is_on_file(filename): response = requests.get(url, headers={ 'groups': 'ROLE_AUTOMATION,', 'username': '******' }) values = response.json() logger.info('vocab is not on file') make_dir(filename) with open(filename, 'w') as f: for value in values: label = value['id'] if label is not None: f.write(label) f.write('\n') # now put file away. head, tail = os.path.split(filename) put_aws_file_with_path(vocab_path, tail) write_filenames_index_from_filename(filename) else: head, tail = os.path.split(filename) logger.info('get from aws ' + tail) #need to load the file from aws potentially get_aws_file(vocab_path, tail) return filename
def init_models(model_dir): logger.info('calling init') indexes = get_indexes(local_dir + model_dir) for attribute, value in indexes.items(): if (value['active'] == True): get_aws_file(model_dir + '/', attribute) indexes = get_indexes(local_dir + model_dir + '/eval') for attribute, value in indexes.items(): if (value['active'] == True): get_aws_file(model_dir + '/eval/', attribute)
def create_vocab(filename, country): vocab_path = get_dir_cfg()['vocab_path'] vocab_path = vocab_path.replace('<key>', country) filename = local_dir + vocab_path + filename + ".txt" head, tail = os.path.split(filename) logger.info('get from aws ' + tail) # need to load the file from aws potentially get_aws_file(vocab_path, tail) return filename
def create_csv(url, filename, start_date, end_date, aws_path): logger.info('getting csv data...' + filename) if is_on_file(filename): logger.info("csv file already created " + filename) head, tail = os.path.split(filename) return get_aws_file(head.replace(local_dir, '') + '/', tail) else: data = requests.get(url + '/' + start_date.strftime("%d-%m-%Y") + '/' + end_date.strftime("%d-%m-%Y"), headers={'groups': 'ROLE_AUTOMATION,', 'username': '******'}) has_data = write_csv(filename, data.json()) logger.info('created csv') head, tail = os.path.split(filename) put_aws_file_with_path(aws_path, tail) write_filenames_index_from_filename(filename) return has_data
from util.file_utils import put_aws_file from util.file_utils import get_aws_file put_aws_file('/home/timmytime/results2.txt') get_aws_file('', 'results2.txt')