def __init__(self,
data_name,
dest_folder,
split,
remove_local_copy=True,
upload_to_s3=True,
**kwargs):
self.data_name = data_name
self.dest_folder = dest_folder
self.dataset = Box(data=[], info={})
self.dataset.info.data_created = datetime.datetime.now().strftime(
"%Y-%m-%d %H:%M")
self.dataset.info.update(kwargs)
self.image_id = 0
self.split = split
self.remove_local_copy = remove_local_copy
self.upload_to_s3 = upload_to_s3
self.bucket = s3.S3Bucket(
bucket_name='drivendata-competition-clog-loss')
def save_and_push_to_s3(payload, payload_type, default_uri):
"""
Small util to push and get readable link for trains. For some reason,
their stuff is not using the correct link
"""
bucket = s3.S3Bucket(bucket_name='vegai-trains', region_name='eu-west-2')
if not os.path.isdir('/tmp'):
os.makedirs('/tmp')
fpath = '/tmp/figure.png'
if payload_type == 'fig':
payload.savefig(fpath)
elif payload_type == 'image':
payload.save(fpath)
else:
raise NotImplementedError
key = os.path.join(default_uri.replace('s3://vegai-trains/', ''),
'{}.png'.format(uuid.uuid4().hex))
bucket.upload_from_file(fpath, key, overwrite=True)
url = bucket.create_predesigned_url(key, expiration=3600 * 6)
os.remove(fpath)
return url
def __init__(self,
model_name: str,
model_task: str,
model_description: str = "",
expname: str = None,
model_type: str = 'models'):
"""
Args:
model_type (str): Type of the model, 'models'
model_name (str): Name of your model
model_task (str): Task of your model
stage (str): training or prediction
expname (str): Name of the experiment
"""
self.model_type = model_type
self.model_name = model_name
self.model_task = model_task
self.model_description = model_description
self.model_folder = os.path.join(ROOT_DIR, self.model_type,
self.model_task, self.model_name)
self.bucket = s3.S3Bucket(bucket_name='omatai-project')
if not os.path.isdir(self.model_folder):
os.makedirs(self.model_folder)
self.expname = expname
import json
import os
import pickle
import shutil
from box import Box
from vegai.utils import s3
bucket = s3.S3Bucket('omatai-project')
ROOT_DIR = os.environ['ROOT_DIR']
DATA_PATH = os.path.join(ROOT_DIR, 'data', 'interim')
def init_folder(data_name, overwrite=False):
'''
Init the binary directory
'''
dest_folder = os.path.join(DATA_PATH, data_name)
if os.path.isdir(dest_folder):
if overwrite:
shutil.rmtree(dest_folder)
else:
raise ValueError(
'Dataset {} already exist but overwrite set to False'.format(
data_name))
if not os.path.isdir(dest_folder):
os.makedirs(dest_folder)
if not os.path.isdir(os.path.join(dest_folder, 'data')):
os.makedirs(os.path.join(dest_folder, 'data'))
return dest_folder
def __init__(self, model_task: str, model_name: str, config_name: str):
self.model_task = model_task
self.model_name = model_name
self.config_name = config_name
self.bucket = s3.S3Bucket(bucket_name='omatai-project')