def getAlisaBucket(): """Get Alisa oss bucket, this function get params from env variables""" ep = os.getenv("SQLFLOW_OSS_ALISA_ENDPOINT") ak = os.getenv("SQLFLOW_OSS_AK") sk = os.getenv("SQLFLOW_OSS_SK") bucketName = os.getenv("SQLFLOW_OSS_ALISA_BUCKET") if ep == "" or ak == "" or sk == "": return SQLFlowDiagnostic( "should define SQLFLOW_OSS_ALISA_ENDPOINT, " "SQLFLOW_OSS_ALISA_BUCKET, SQLFLOW_OSS_AK, SQLFLOW_OSS_SK " "when using submitter alisa") return oss.get_bucket(bucketName, ak, sk, endpoint=ep)
def submit_optflow_job(train_table, result_table, fsl_file_content, solver, user_id): """ Submit the OptFlow job. Args: train_table (str): the source table name. result_table (str): the table name to save the solved results. fsl_file_content (str): the FSL file content to submit. solver (str): the solver used to solve the model. user_id (str): the user id. Returns: None """ project_name = train_table.split(".")[0] snapshot_id = os.getenv("SQLFLOW_OPTFLOW_SNAPSHOT_ID") if not snapshot_id: raise ValueError("SQLFLOW_OPTFLOW_SNAPSHOT_ID must be set") token = os.getenv("SQLFLOW_OPTFLOW_TOKEN") if not token: raise ValueError("SQLFLOW_OPTFLOW_TOKEN must be set") submit_job_url = os.getenv("SQLFLOW_OPTFLOW_SUBMIT_JOB_URL") if not submit_job_url: raise ValueError("SQLFLOW_OPTFLOW_SUBMIT_JOB_URL must be set") query_job_status_url = os.getenv("SQLFLOW_OPTFLOW_QUERY_JOB_STATUS_URL") if not query_job_status_url: raise ValueError("SQLFLOW_OPTFLOW_QUERY_JOB_STATUS_URL must be set") query_job_log_url = os.getenv("SQLFLOW_OPTFLOW_QUERY_JOB_LOG_URL") if not query_job_log_url: raise ValueError("SQLFLOW_OPTFLOW_QUERY_JOB_LOG_URL must be set") bucket_name = "sqlflow-optflow-models" bucket = get_bucket(bucket_name) try: bucket_info = bucket.get_bucket_info() except oss2.exceptions.NoSuchBucket: # Create bucket if not exists bucket.create_bucket() bucket_info = bucket.get_bucket_info() fsl_file_id = '{}.fsl'.format(uuid.uuid4()) bucket.put_object(fsl_file_id, fsl_file_content) should_delete_object = True try: bucket.put_object_acl(fsl_file_id, oss2.BUCKET_ACL_PUBLIC_READ) fsl_url = "http://{}.{}/{}".format(bucket_name, bucket_info.extranet_endpoint, fsl_file_id) input_params = { "input_table": train_table, "output_table": result_table, "fsl_path": fsl_url, "solver_name": solver, } json_data = { "userNumber": user_id, "projectName": project_name, "snapshotId": snapshot_id, "token": token, "inputParams": input_params, } response = requests.post(submit_job_url, json=json_data, headers=OPTFLOW_HTTP_HEADERS) response.raise_for_status() response_json = response.json() if not response_json['success']: raise ValueError("Job submission fails") print('Job submission succeeds') record_id = response_json['data']['recordId'] try: success = print_job_log_till_finish(query_job_status_url, query_job_log_url, record_id, user_id, token) if success: print("Job succeeds. Save solved result in {}.".format( result_table)) else: print("Job fails.") except: # noqa: E722 # FIXME(sneaxiy): we should not delete object if there is any # network error when querying job status and logs. But when # should we clean the object? should_delete_object = False six.reraise(*sys.exc_info()) finally: if should_delete_object: bucket.delete_object(fsl_file_id)