from ait_sdk.common.files.ait_manifest import AITManifest # do not remove
from ait_sdk.develop.ait_path_helper import AITPathHelper # do not remove
from ait_sdk.utils.logging import get_logger, log, get_log_path # do not remove
from ait_sdk.develop.annotation import measures, resources, downloads, ait_main # do not remove
# must use modules
# In[8]:
#########################################
# area:create manifest
# should edit
#########################################
if not is_ait_launch:
from ait_sdk.common.files.ait_manifest_generator import AITManifestGenerator
manifest_genenerator = AITManifestGenerator(current_dir)
manifest_genenerator.set_ait_name('eval_mean_difference_german_aif360')
manifest_genenerator.set_ait_description('''
detecting bias on credit decisions, data set is german_credit.
Available protected_attribute and privileged_classes example (See german.doc for details)
month(>=24.0),
credit_amount(>=3000.0),
investment_as_income_percentage(>=3.0),
residence_since(>=3.0),
age(>=25.0, defult),
number_of_credits(>=2.0),
people_liable_for(>=2.0)
''')
manifest_genenerator.set_ait_author('AIST')
manifest_genenerator.set_ait_email('')
manifest_genenerator.set_ait_version('0.1')
def test_write_output(self):
print('++ テスト開始')
aaa = AITManifestGenerator('./')
aaa.set_ait_name("set_ait_name")
aaa.set_ait_description("set_ait_description")
aaa.set_ait_author("set_ait_author")
aaa.set_ait_email("set_ait_email")
aaa.set_ait_version("0.1")
aaa.set_ait_quality("set_ait_quality")
aaa.set_ait_reference("set_ait_reference")
aaa.add_ait_inventories('name1', 'type1', 'description1', ['csv'],
'schema1')
aaa.add_ait_inventories('name2', 'type2', 'description2',
['gz', 'zip'], 'schema')
aaa.add_ait_parameters('name1', 'type1', 'description1',
'default_val1')
aaa.add_ait_parameters('name2', 'type2', 'description2')
aaa.add_ait_measures('name1', 'type1', 'description1', 'structure1')
aaa.add_ait_measures('name2', 'type2', 'description2', 'structure2')
aaa.add_ait_resources('name1', 'type1', 'description1')
aaa.add_ait_resources('name2', 'type2', 'description2')
aaa.add_ait_downloads('name1', 'description1')
aaa.add_ait_downloads('name2', 'description2')
aaa.write()
print('++ テスト終了')
from ait_sdk.common.files.ait_manifest import AITManifest # do not remove
from ait_sdk.develop.ait_path_helper import AITPathHelper # do not remove
from ait_sdk.utils.logging import get_logger, log, get_log_path # do not remove
from ait_sdk.develop.annotation import measures, resources, downloads, ait_main # do not remove
# must use modules
# In[8]:
#########################################
# area:create manifest
# should edit
#########################################
if not is_ait_launch:
from ait_sdk.common.files.ait_manifest_generator import AITManifestGenerator
manifest_genenerator = AITManifestGenerator(current_dir)
manifest_genenerator.set_ait_name('dev_template_local_docker')
manifest_genenerator.set_ait_description(
'AIT template (docker image regist to local)')
manifest_genenerator.set_ait_author('AIST')
manifest_genenerator.set_ait_email('')
manifest_genenerator.set_ait_version('0.1')
manifest_genenerator.set_ait_quality(
'https://airc.aist.go.jp/aiqm/quality/internal/Coverage_for_distinguished_problem_cases'
)
manifest_genenerator.set_ait_reference('')
manifest_genenerator.add_ait_inventories(
name='iris_data',
type_='dataset',
description='アヤメの分類データです',
format_=['csv'],
from ait_sdk.common.files.ait_manifest import AITManifest # do not remove
from ait_sdk.develop.ait_path_helper import AITPathHelper # do not remove
from ait_sdk.utils.logging import get_logger, log, get_log_path # do not remove
from ait_sdk.develop.annotation import measures, resources, downloads, ait_main # do not remove
# must use modules
# In[8]:
#########################################
# area:create manifest
# should edit
#########################################
if not is_ait_launch:
from ait_sdk.common.files.ait_manifest_generator import AITManifestGenerator
manifest_genenerator = AITManifestGenerator(current_dir)
manifest_genenerator.set_ait_name('eval_metamorphic_test_tf1.13')
manifest_genenerator.set_ait_description('''Metamorphic test.
Make sure can be classified in the same result as the original class be added a little processing to the original data.'''
)
manifest_genenerator.set_ait_author('AIST')
manifest_genenerator.set_ait_email('')
manifest_genenerator.set_ait_version('0.1')
manifest_genenerator.set_ait_quality(
'https://airc.aist.go.jp/aiqm/quality/internal/Robustness_of_trained_model'
)
manifest_genenerator.set_ait_reference('')
manifest_genenerator.add_ait_inventories(
name='mnist_dataset',
type_='dataset',
description=
from ait_sdk.common.files.ait_manifest import AITManifest # do not remove
from ait_sdk.develop.ait_path_helper import AITPathHelper # do not remove
from ait_sdk.utils.logging import get_logger, log, get_log_path # do not remove
from ait_sdk.develop.annotation import measures, resources, downloads, ait_main # do not remove
# must use modules
# In[ ]:
#########################################
# area:create manifest
# should edit
#########################################
if not is_ait_launch:
from ait_sdk.common.files.ait_manifest_generator import AITManifestGenerator
manifest_genenerator = AITManifestGenerator(current_dir)
manifest_genenerator.set_ait_name(
'eval_regression_analyze_coefficient_determination_tf2.3')
manifest_genenerator.set_ait_description(
'1つの目的変数、複数の説明変数で構築された重回帰分析のモデルの自由度調整済み決定係数を算出します。\n標本値（実測値、観測値）を y = {y(1), y(2), ...,y(N)}、\n回帰方程式による推定値を f = {f(1), f(2), ..., f(N)}、\n求める値をR^2、「bar(x)=xの平均」としたとき、\nR^2 ≡ 1 - ( Σ[i=1→N] ( y(i) - f(i) )^2 / ( ( Σ[j=1→N] ( y(i) - bar(y) ) )^2 )'
)
manifest_genenerator.set_ait_author('AIST')
manifest_genenerator.set_ait_email('')
manifest_genenerator.set_ait_version('0.1')
manifest_genenerator.set_ait_quality(
'https://airc.aist.go.jp/aiqm/quality/internal/Accuracy_of_trained_model'
)
manifest_genenerator.set_ait_reference('')
manifest_genenerator.add_ait_inventories(
name='trained_model',
type_='model',
from ait_sdk.utils.logging import get_logger, log, get_log_path # do not remove
from ait_sdk.develop.annotation import measures, resources, downloads, ait_main # do not remove
# must use modules
# In[8]:
#########################################
# area:create manifest
# should edit
#########################################
if not is_ait_launch:
from ait_sdk.common.files.ait_manifest_generator import AITManifestGenerator
manifest_genenerator = AITManifestGenerator(current_dir)
manifest_genenerator.set_ait_name('eval_regression_analyze_rmse_and_mae_tf2.3')
manifest_genenerator.set_ait_description('''Calculate the RMSE and MAE of a model of multiple regression analysis constructed with one objective variable and multiple explanatory variables.
Both RMSE (short for Root Mean Square Error) and MAE (short for Mean Absolute Error) represent the size of the averaged error.
Number of data n, true values: y(1),y(2),⋯,y(n) Predicted values: f(1),f(2),⋯,f(n) Let the values you seek be RMSE and MAE, respectively.
RMSE = √( ( ( 1 / n ) * ( Σ[i=1→n] ( (( f ( i ) - y ( i ) ) ^2 ) )
MAE = ( 1 / n ) * ( Σ[i=1→n] ( | f ( i ) - y ( i ) | )
''')
manifest_genenerator.set_ait_author('AIST')
manifest_genenerator.set_ait_email('')
manifest_genenerator.set_ait_version('0.1')
manifest_genenerator.set_ait_quality('https://airc.aist.go.jp/aiqm/quality/internal/Accuracy_of_trained_model')
manifest_genenerator.set_ait_reference('')
manifest_genenerator.add_ait_inventories(name='trained_model',
type_='model',
description='Tensorflow 2.3 model',
from ait_sdk.common.files.ait_manifest import AITManifest # do not remove
from ait_sdk.develop.ait_path_helper import AITPathHelper # do not remove
from ait_sdk.utils.logging import get_logger, log, get_log_path # do not remove
from ait_sdk.develop.annotation import measures, resources, downloads, ait_main # do not remove
# must use modules
# In[8]:
#########################################
# area:create manifest
# should edit
#########################################
if not is_ait_launch:
from ait_sdk.common.files.ait_manifest_generator import AITManifestGenerator
manifest_genenerator = AITManifestGenerator(current_dir)
manifest_genenerator.set_ait_name('eval_bdd100k_aicc_tf2.3')
manifest_genenerator.set_ait_description(
'''The image classification model infers the image data (.jpg).
Compare the inference result with the correct answer data (.json).
Output the coverage of the comparison result.
!!!Caution!!!
Please set the memory allocation of docker to 4GB or more.''')
manifest_genenerator.set_ait_author('AIST')
manifest_genenerator.set_ait_email('')
manifest_genenerator.set_ait_version('0.1')
manifest_genenerator.set_ait_quality(
'https://airc.aist.go.jp/aiqm/quality/internal/Accuracy_of_trained_model'
)
manifest_genenerator.set_ait_reference('')
manifest_genenerator.add_ait_inventories(
from ait_sdk.common.files.ait_manifest import AITManifest # do not remove
from ait_sdk.develop.ait_path_helper import AITPathHelper # do not remove
from ait_sdk.utils.logging import get_logger, log, get_log_path # do not remove
from ait_sdk.develop.annotation import measures, resources, downloads, ait_main # do not remove
# must use modules
# In[5]:
#########################################
# area:create manifest
# should edit
#########################################
if not is_ait_launch:
from ait_sdk.common.files.ait_manifest_generator import AITManifestGenerator
manifest_genenerator = AITManifestGenerator(current_dir)
manifest_genenerator.set_ait_name('generate_ca_pairwise')
manifest_genenerator.set_ait_description('''
The AIT is generate pair-wise combination for PICT.
''')
manifest_genenerator.set_ait_author('AIST')
manifest_genenerator.set_ait_email('')
manifest_genenerator.set_ait_version('0.1')
manifest_genenerator.set_ait_quality(
'https://airc.aist.go.jp/aiqm/quality/internal/Coverage_for_distinguished_problem_cases'
)
manifest_genenerator.set_ait_reference('')
manifest_genenerator.add_ait_inventories(
name='pair_wise_model',
type_='dataset',
description='''
from ait_sdk.common.files.ait_manifest import AITManifest # do not remove
from ait_sdk.develop.ait_path_helper import AITPathHelper # do not remove
from ait_sdk.utils.logging import get_logger, log, get_log_path # do not remove
from ait_sdk.develop.annotation import measures, resources, downloads, ait_main # do not remove
# must use modules
# In[5]:
#########################################
# area:create manifest
# should edit
#########################################
if not is_ait_launch:
from ait_sdk.common.files.ait_manifest_generator import AITManifestGenerator
manifest_genenerator = AITManifestGenerator(current_dir)
manifest_genenerator.set_ait_name('eval_find_ca_high_risk')
manifest_genenerator.set_ait_description(
'Evaluating quantity of high risk CA combination cases in BDD dataset)'
)
manifest_genenerator.set_ait_author('AIST')
manifest_genenerator.set_ait_email('')
manifest_genenerator.set_ait_version('0.1')
manifest_genenerator.set_ait_quality(
'https://airc.aist.go.jp/aiqm/quality/internal/Diversity_of_test_data')
manifest_genenerator.set_ait_reference('')
manifest_genenerator.add_ait_inventories(
name='Data',
type_='dataset',
description=
'Classification of different attributes related to autonomous driving scenarios',
from ait_sdk.common.files.ait_manifest import AITManifest # do not remove
from ait_sdk.develop.ait_path_helper import AITPathHelper # do not remove
from ait_sdk.utils.logging import get_logger, log, get_log_path # do not remove
from ait_sdk.develop.annotation import measures, resources, downloads, ait_main # do not remove
# must use modules
# In[8]:
#########################################
# area:create manifest
# should edit
#########################################
if not is_ait_launch:
from ait_sdk.common.files.ait_manifest_generator import AITManifestGenerator
manifest_genenerator = AITManifestGenerator(current_dir)
manifest_genenerator.set_ait_name('eval_dnn_coverage_tf1.13')
manifest_genenerator.set_ait_description('''
Calculate the neuron coverage for the input dataset given by the user, and display it in the form of a heat map. After that, a gradient for the input dataset is computed by backpropagation. Based on the gradient, the most efficient manipulation to input values in order for the neuron coverage to increase is selected. A number of data is removed from the dataset, and these removed data are converted to new data by the selected manipulation. Then, the new data is added to the dataset, and the coverage is recalculated by running the model with the updated dataset. Similarly, a gradient for the updated dataset is computed. These processes are repeated until the target coverage rate is achieved. Data manipulation algorithm is implemented by the user in Python.
''')
manifest_genenerator.set_ait_author('AIST')
manifest_genenerator.set_ait_email('')
manifest_genenerator.set_ait_version('0.1')
manifest_genenerator.set_ait_quality(
'https://airc.aist.go.jp/aiqm/quality/internal/Robustness_of_trained_model'
)
manifest_genenerator.set_ait_reference('')
manifest_genenerator.add_ait_inventories(
name='image_data',
type_='dataset',
description='MNIST image data',
from ait_sdk.utils.logging import get_logger, log, get_log_path # do not remove
from ait_sdk.develop.annotation import measures, resources, downloads, ait_main # do not remove
# must use modules
# In[8]:
#########################################
# area:create manifest
# should edit
#########################################
if not is_ait_launch:
from ait_sdk.common.files.ait_manifest_generator import AITManifestGenerator
manifest_genenerator = AITManifestGenerator(current_dir)
manifest_genenerator.set_ait_name('eval_mnist_acc_tf2.3')
manifest_genenerator.set_ait_description('Only Sequential API Model\n\n<QualityMeasurement>\nAccuracy=TP+TNTP+FP+FN+TN\nPrecision=TPTP+FP\nRecall=TPTP+FN\nF−measure=2Recall∗PrecisionRecall+Precision\nAUC\n\n<Resources>\nROC曲線\n混同行列\nNG予測画像')
manifest_genenerator.set_ait_author('AIST')
manifest_genenerator.set_ait_email('')
manifest_genenerator.set_ait_version('0.1')
manifest_genenerator.set_ait_quality('https://airc.aist.go.jp/aiqm/quality/internal/Accuracy_of_trained_model')
manifest_genenerator.set_ait_reference('')
manifest_genenerator.add_ait_inventories(name='trained_model',
type_='model',
description='Tensorflow 2.3で学習したモデル',
format_=['h5'],
schema='https://support.hdfgroup.org/HDF5/doc/')
manifest_genenerator.add_ait_inventories(name='test_set_images',
type_='dataset',
from ait_sdk.common.files.ait_manifest import AITManifest # do not remove
from ait_sdk.develop.ait_path_helper import AITPathHelper # do not remove
from ait_sdk.utils.logging import get_logger, log, get_log_path # do not remove
from ait_sdk.develop.annotation import measures, resources, downloads, ait_main # do not remove
# must use modules
# In[5]:
#########################################
# area:create manifest
# should edit
#########################################
if not is_ait_launch:
from ait_sdk.common.files.ait_manifest_generator import AITManifestGenerator
manifest_genenerator = AITManifestGenerator(current_dir)
manifest_genenerator.set_ait_name('eval_ca_distribution')
manifest_genenerator.set_ait_description(
'Evaluating distribution of dataset with expected distribution')
manifest_genenerator.set_ait_author('AIST')
manifest_genenerator.set_ait_email('')
manifest_genenerator.set_ait_version('0.1')
manifest_genenerator.set_ait_quality(
'https://airc.aist.go.jp/aiqm/quality/internal/Distribution_of_training_data'
)
manifest_genenerator.set_ait_reference('')
manifest_genenerator.add_ait_inventories(
'Data', 'dataset',
'Classification of different attributes related to autonomous driving scenarios',
['csv'], 'https://bdd-data.berkeley.edu/')
manifest_genenerator.add_ait_parameters(
from ait_sdk.common.files.ait_manifest import AITManifest # do not remove
from ait_sdk.develop.ait_path_helper import AITPathHelper # do not remove
from ait_sdk.utils.logging import get_logger, log, get_log_path # do not remove
from ait_sdk.develop.annotation import measures, resources, downloads, ait_main # do not remove
# must use modules
# In[ ]:
#########################################
# area:create manifest
# should edit
#########################################
if not is_ait_launch:
from ait_sdk.common.files.ait_manifest_generator import AITManifestGenerator
manifest_genenerator = AITManifestGenerator(current_dir)
manifest_genenerator.set_ait_name('eval_mnist_data_coverage')
manifest_genenerator.set_ait_description('''
Calculate coverage for the contour area and perimeter of the dataset MNIST image.\r\n
\r\n
𝐶𝑜𝑣[𝑥(𝑛)]=|𝑚𝑎𝑥{𝑥(𝑛)}−𝑚𝑖𝑛{𝑥(𝑛)}|/| 〖ℎ𝑖𝑔ℎ〗_𝑖−〖𝑙𝑜𝑤〗_𝑖 |
''')
manifest_genenerator.set_ait_author('AIST')
manifest_genenerator.set_ait_email('')
manifest_genenerator.set_ait_version('0.1')
manifest_genenerator.set_ait_quality(
'https://airc.aist.go.jp/aiqm/quality/internal/Diversity_of_test_data')
manifest_genenerator.set_ait_reference('')
manifest_genenerator.add_ait_inventories(
name='images',
type_='dataset',
from ait_sdk.common.files.ait_manifest import AITManifest # do not remove
from ait_sdk.develop.ait_path_helper import AITPathHelper # do not remove
from ait_sdk.utils.logging import get_logger, log, get_log_path # do not remove
from ait_sdk.develop.annotation import measures, resources, downloads, ait_main # do not remove
# must use modules
# In[5]:
#########################################
# area:create manifest
# should edit
#########################################
if not is_ait_launch:
from ait_sdk.common.files.ait_manifest_generator import AITManifestGenerator
manifest_genenerator = AITManifestGenerator(current_dir)
manifest_genenerator.set_ait_name('eval_find_ca_error')
manifest_genenerator.set_ait_description(
'Evaluating quantity of unsound CA combination cases in dataset')
manifest_genenerator.set_ait_author('AIST')
manifest_genenerator.set_ait_email('')
manifest_genenerator.set_ait_version('0.1')
manifest_genenerator.set_ait_quality(
'https://airc.aist.go.jp/aiqm/quality/internal/Coverage_for_distinguished_problem_cases'
)
manifest_genenerator.set_ait_reference('')
manifest_genenerator.add_ait_inventories(
name='ca_data',
type_='dataset',
description=
'''Classification of different attributes related to autonomous driving scenarios.
from ait_sdk.utils.logging import get_logger, log, get_log_path # do not remove
from ait_sdk.develop.annotation import measures, resources, downloads, ait_main # do not remove
# must use modules
# In[5]:
#########################################
# area:create manifest
# should edit
#########################################
if not is_ait_launch:
from ait_sdk.common.files.ait_manifest_generator import AITManifestGenerator
manifest_genenerator = AITManifestGenerator(current_dir)
manifest_genenerator.set_ait_name('eval_coverage_ca_pairwise')
manifest_genenerator.set_ait_description('it is calculates that Data-set has how much coverage for the combination of a given pair-wise.\n1.Find out if each pairwise row exists in the dataset\n2.A search pairwise pattern and a matching amount is output to a file.\n3.Calculate what percentage of all pairwise patterns are matched.\nBelow, restrictions\n1.The dataset must have the same columns as the pairwise combination.\n2.It is error if the required column does not exist in dataset.\n3.The dataset may have an extra column.\n4.It does not support regular expression search.\n"*" Is treated as one character "*".\n5.Pairwise and dataset may have null value.\nデータセットが特定のペアワイズの組み合わせに対してどの程度のカバレッジを持っているかを計算する。\n1.ペアワイズの各行が、データセットに存在するか検索する\n2.検索したペアワイズのパターンとマッチした件数をファイル出力する。\n3.マッチした件数が全ペアワイズパターンのうちの何パーセントか計算する。\n以下、制約事項\n1.データセットは、ペアワイズの組み合わせと同じカラムを持っている必要がある。\n2.データセットに、必要なカラムが存在しない場合はエラーになる。\n3.データセットは、余分なカラムを持っていてもよい。\n4.正規表現検索には対応しない。\n”＊”は”＊”という１文字として扱う。\n5.ペアワイズやデータセットにnullは存在してもよい。')
manifest_genenerator.set_ait_author('AIST')
manifest_genenerator.set_ait_email('')
manifest_genenerator.set_ait_version('0.1')
manifest_genenerator.set_ait_quality('https://airc.aist.go.jp/aiqm/quality/internal/Diversity_of_test_data')
manifest_genenerator.set_ait_reference('')
manifest_genenerator.add_ait_inventories(name='pairwise_list',
type_='dataset',
description='Pairwise_list.csv',
format_=['csv'],
schema='https://www.sciencedirect.com/topics/computer-science/pairwise-comparison')
manifest_genenerator.add_ait_inventories(name='target',
type_='dataset',
description='target.csv',
