Exemplo n.º 1
0
    def post(self):
        """
        Executes a prep job to create an image corpus for training.
        Use this method to start a prep job.
        """
        job_def = request.json
        job_def['process_json'] = True # Hardcode to process json file from project folder
        job = Job(job_def['name'],job_def)
        job.type = 'preprocess'
        dt = newdt.now()
        job.start_time = int(dt.timestamp()*1000)
        job.request = {'full_path': request.full_path,'remote_addr':request.remote_addr,'method':request.method}
        jb = aug_queue.enqueue(
             preprocess, job,job_timeout=-1,result_ttl=86400,ttl=-1)
        jb.meta['job_def'] = job_def
        dt = newdt.now()
        jb.meta['job_init_time'] = str(int(dt.timestamp()*1000))
        jb.status = 'Running'
        jb.save_meta()
        json_str = job.to_json_string()
        st = {
            'BUCKET' : job.bucket,
            'USE_GCS' : job.use_gcs,
            'ACCESS_KEY' : access_key,
            'SECRET_KEY' : secret_key,
            'S3_URL' : s3_url
        }
        storage = Storage(st)
        storage.upload_data(json_str,'jobs/running/{}_0_preprocess_r_{}.json'.format(str(job.start_time),jb.id),contentType='application/json')
        storage.upload_data(json_str,'jobs/all/{}_0_preprocess_r_{}.json'.format(str(job.start_time),jb.id),contentType='application/json')

        return {
            "status": jb.status,
            'job_id': jb.id,
            'meta':jb.meta},201
Exemplo n.º 2
0
    def post(self):
        """
        Executes a training.
        Use this method to start a training.
        """
        job_def = request.json
        job = Job(job_def['name'], job_def)
        job.type = 'train'
        dt = newdt.now()
        job.start_time = int(dt.timestamp() * 1000)
        job.request = {
            'full_path': request.full_path,
            'remote_addr': request.remote_addr,
            'method': request.method
        }
        if hasattr(job, 'ml_engine') and job.ml_engine:
            jb = train_queue.enqueue(train_mlengine,
                                     job,
                                     job_timeout=-1,
                                     result_ttl=-1)
        else:
            jb = train_queue.enqueue(train_job_method, job, job_timeout=-1)
        jb.meta['job_init_time'] = str(int(dt.timestamp() * 1000))
        jb.meta['job_def'] = job_def
        jb.save_meta()

        json_str = job.to_json_string()
        st = {
            'BUCKET': job.bucket,
            'USE_GCS': job.use_gcs,
            'ACCESS_KEY': access_key,
            'SECRET_KEY': secret_key,
            'S3_URL': s3_url
        }
        storage = Storage(st)
        storage.upload_data(json_str,
                            'jobs/running/{}_0_train_r_{}.json'.format(
                                str(job.start_time), jb.id),
                            contentType='application/json')
        storage.upload_data(json_str,
                            'jobs/all/{}_0_train_r_{}.json'.format(
                                str(job.start_time), jb.id),
                            contentType='application/json')
        return {
            "status": jb.get_status(),
            'job_id': jb.id,
            'meta': jb.meta
        }, 201