def test_write_output(self):
print('++ テスト開始')
aaa = AITManifestGenerator('./')
aaa.set_ait_name("set_ait_name")
aaa.set_ait_description("set_ait_description")
aaa.set_ait_author("set_ait_author")
aaa.set_ait_email("set_ait_email")
aaa.set_ait_version("0.1")
aaa.set_ait_quality("set_ait_quality")
aaa.set_ait_reference("set_ait_reference")
aaa.add_ait_inventories('name1', 'type1', 'description1', ['csv'],
'schema1')
aaa.add_ait_inventories('name2', 'type2', 'description2',
['gz', 'zip'], 'schema')
aaa.add_ait_parameters('name1', 'type1', 'description1',
'default_val1')
aaa.add_ait_parameters('name2', 'type2', 'description2')
aaa.add_ait_measures('name1', 'type1', 'description1', 'structure1')
aaa.add_ait_measures('name2', 'type2', 'description2', 'structure2')
aaa.add_ait_resources('name1', 'type1', 'description1')
aaa.add_ait_resources('name2', 'type2', 'description2')
aaa.add_ait_downloads('name1', 'description1')
aaa.add_ait_downloads('name2', 'description2')
aaa.write()
print('++ テスト終了')
investment_as_income_percentage(>=3.0),
residence_since(>=3.0),
age(>=25.0, defult),
number_of_credits(>=2.0),
people_liable_for(>=2.0)
''')
manifest_genenerator.set_ait_author('AIST')
manifest_genenerator.set_ait_email('')
manifest_genenerator.set_ait_version('0.1')
manifest_genenerator.set_ait_quality(
'https://airc.aist.go.jp/aiqm/quality/internal/Distribution_of_training_data'
)
manifest_genenerator.set_ait_reference('')
manifest_genenerator.add_ait_inventories(
name='Data',
type_='dataset',
description='german credit data',
format_=['csv'],
schema='https://archive.ics.uci.edu/ml/datasets/')
manifest_genenerator.add_ait_parameters(name='protected_attribute',
type_='str',
description='protected attributee',
default_val='age')
manifest_genenerator.add_ait_parameters(name='privileged_classes',
type_='float',
description='privileged classes',
default_val='25.0')
manifest_genenerator.add_ait_measures(
name='mean_difference',
type_='float',
description='mean difference of metric fairness',
structure='single',
from ait_sdk.common.files.ait_manifest_generator import AITManifestGenerator
manifest_genenerator = AITManifestGenerator(current_dir)
manifest_genenerator.set_ait_name('dev_template_local_docker')
manifest_genenerator.set_ait_description(
'AIT template (docker image regist to local)')
manifest_genenerator.set_ait_author('AIST')
manifest_genenerator.set_ait_email('')
manifest_genenerator.set_ait_version('0.1')
manifest_genenerator.set_ait_quality(
'https://airc.aist.go.jp/aiqm/quality/internal/Coverage_for_distinguished_problem_cases'
)
manifest_genenerator.set_ait_reference('')
manifest_genenerator.add_ait_inventories(
name='iris_data',
type_='dataset',
description='アヤメの分類データです',
format_=['csv'],
schema='https://archive.ics.uci.edu/ml/datasets/iris')
manifest_genenerator.add_ait_parameters(
name='mean_column_name',
type_='str',
description='sepal.width\nsepal.length\npetal.width\npetal.length',
default_val='sepal.width')
manifest_genenerator.add_ait_measures(name='mean',
type_='float',
description='mean of select column',
structure='single',
min='0')
manifest_genenerator.add_ait_resources(name='pairplot',
type_='picture',
description='pairplot')
manifest_genenerator = AITManifestGenerator(current_dir)
manifest_genenerator.set_ait_name('eval_metamorphic_test_tf1.13')
manifest_genenerator.set_ait_description('''Metamorphic test.
Make sure can be classified in the same result as the original class be added a little processing to the original data.'''
)
manifest_genenerator.set_ait_author('AIST')
manifest_genenerator.set_ait_email('')
manifest_genenerator.set_ait_version('0.1')
manifest_genenerator.set_ait_quality(
'https://airc.aist.go.jp/aiqm/quality/internal/Robustness_of_trained_model'
)
manifest_genenerator.set_ait_reference('')
manifest_genenerator.add_ait_inventories(
name='mnist_dataset',
type_='dataset',
description=
'MNIST_dataset are train image, train label, test image, test label',
format_=['zip'],
schema='http://yann.lecun.com/exdb/mnist/')
manifest_genenerator.add_ait_inventories(
name='mnist_model',
type_='model',
description='MNIST_model',
format_=['zip'],
schema='https://github.com/hitachi-rd-yokohama/deep_saucer')
manifest_genenerator.add_ait_parameters(
name='Lap',
type_='int',
description='Input Data Conversion Number',
default_val='10')
manifest_genenerator.add_ait_parameters(
manifest_genenerator = AITManifestGenerator(current_dir)
manifest_genenerator.set_ait_name(
'eval_regression_analyze_coefficient_determination_tf2.3')
manifest_genenerator.set_ait_description(
'1つの目的変数、複数の説明変数で構築された重回帰分析のモデルの自由度調整済み決定係数を算出します。\n標本値（実測値、観測値）を y = {y(1), y(2), ...,y(N)}、\n回帰方程式による推定値を f = {f(1), f(2), ..., f(N)}、\n求める値をR^2、「bar(x)=xの平均」としたとき、\nR^2 ≡ 1 - ( Σ[i=1→N] ( y(i) - f(i) )^2 / ( ( Σ[j=1→N] ( y(i) - bar(y) ) )^2 )'
)
manifest_genenerator.set_ait_author('AIST')
manifest_genenerator.set_ait_email('')
manifest_genenerator.set_ait_version('0.1')
manifest_genenerator.set_ait_quality(
'https://airc.aist.go.jp/aiqm/quality/internal/Accuracy_of_trained_model'
)
manifest_genenerator.set_ait_reference('')
manifest_genenerator.add_ait_inventories(
name='trained_model',
type_='model',
description='Tensorflow 2.3で学習したモデル',
format_=['h5'],
schema='https://support.hdfgroup.org/HDF5/doc/')
manifest_genenerator.add_ait_inventories(
name='dataset_for_verification',
type_='dataset',
description='検証用データセット\n目的変数と説明変数のセットでラベルは必要',
format_=['csv'],
schema='uncreated')
manifest_genenerator.add_ait_parameters(name='target_variable',
type_='str',
description='目的変数',
default_val='')
manifest_genenerator.add_ait_measures(
name='degree_of_freedom_adjusted_coefficient_determination',
type_='float',
manifest_genenerator = AITManifestGenerator(current_dir)
manifest_genenerator.set_ait_name('eval_regression_analyze_rmse_and_mae_tf2.3')
manifest_genenerator.set_ait_description('''Calculate the RMSE and MAE of a model of multiple regression analysis constructed with one objective variable and multiple explanatory variables.
Both RMSE (short for Root Mean Square Error) and MAE (short for Mean Absolute Error) represent the size of the averaged error.
Number of data n, true values: y(1),y(2),⋯,y(n) Predicted values: f(1),f(2),⋯,f(n) Let the values you seek be RMSE and MAE, respectively.
RMSE = √( ( ( 1 / n ) * ( Σ[i=1→n] ( (( f ( i ) - y ( i ) ) ^2 ) )
MAE = ( 1 / n ) * ( Σ[i=1→n] ( | f ( i ) - y ( i ) | )
''')
manifest_genenerator.set_ait_author('AIST')
manifest_genenerator.set_ait_email('')
manifest_genenerator.set_ait_version('0.1')
manifest_genenerator.set_ait_quality('https://airc.aist.go.jp/aiqm/quality/internal/Accuracy_of_trained_model')
manifest_genenerator.set_ait_reference('')
manifest_genenerator.add_ait_inventories(name='trained_model',
type_='model',
description='Tensorflow 2.3 model',
format_=['h5'],
schema='HDF5')
manifest_genenerator.add_ait_inventories(name='dataset_for_verification',
type_='dataset',
description='Data set for verification requires label',
format_=['csv'],
schema='csv for verification')
manifest_genenerator.add_ait_parameters(name='target_variable',
type_='str',
description='target variable',
default_val='')
manifest_genenerator.add_ait_measures(name='RMSE',
type_='float',
description='The closer to 0, the smaller prediction error.',
structure='single',
manifest_genenerator.set_ait_description(
'''The image classification model infers the image data (.jpg).
Compare the inference result with the correct answer data (.json).
Output the coverage of the comparison result.
!!!Caution!!!
Please set the memory allocation of docker to 4GB or more.''')
manifest_genenerator.set_ait_author('AIST')
manifest_genenerator.set_ait_email('')
manifest_genenerator.set_ait_version('0.1')
manifest_genenerator.set_ait_quality(
'https://airc.aist.go.jp/aiqm/quality/internal/Accuracy_of_trained_model'
)
manifest_genenerator.set_ait_reference('')
manifest_genenerator.add_ait_inventories(
name='trained_model_checkpoint',
type_='model',
description='trained_model_checkpoint',
format_=['zip'],
schema='https://www.tensorflow.org/guide/saved_model')
manifest_genenerator.add_ait_inventories(
name='trained_model_graph',
type_='model',
description='trained_model_graph',
format_=['zip'],
schema='https://www.tensorflow.org/guide/saved_model')
manifest_genenerator.add_ait_inventories(
name='trained_model_protobuf',
type_='model',
description='trained_model_protobuf',
format_=['zip'],
schema='https://www.tensorflow.org/guide/saved_model')
manifest_genenerator.add_ait_inventories(
manifest_genenerator.set_ait_name('generate_ca_pairwise')
manifest_genenerator.set_ait_description('''
The AIT is generate pair-wise combination for PICT.
''')
manifest_genenerator.set_ait_author('AIST')
manifest_genenerator.set_ait_email('')
manifest_genenerator.set_ait_version('0.1')
manifest_genenerator.set_ait_quality(
'https://airc.aist.go.jp/aiqm/quality/internal/Coverage_for_distinguished_problem_cases'
)
manifest_genenerator.set_ait_reference('')
manifest_genenerator.add_ait_inventories(
name='pair_wise_model',
type_='dataset',
description='''
Model of pair-wise.
Define factors and constraints.
''',
format_=['txt'],
schema='https://github.com/Microsoft/pict/blob/master/doc/pict.md')
manifest_genenerator.add_ait_parameters(
name='order_combination',
type_='int',
description='Order of combinations.',
default_val='2')
manifest_genenerator.add_ait_parameters(name='seed',
type_='int',
description='''
Randomize generation, N - seed.
if you fix seed, please set it to 1 or more.
''',
manifest_genenerator = AITManifestGenerator(current_dir)
manifest_genenerator.set_ait_name('eval_find_ca_high_risk')
manifest_genenerator.set_ait_description(
'Evaluating quantity of high risk CA combination cases in BDD dataset)'
)
manifest_genenerator.set_ait_author('AIST')
manifest_genenerator.set_ait_email('')
manifest_genenerator.set_ait_version('0.1')
manifest_genenerator.set_ait_quality(
'https://airc.aist.go.jp/aiqm/quality/internal/Diversity_of_test_data')
manifest_genenerator.set_ait_reference('')
manifest_genenerator.add_ait_inventories(
name='Data',
type_='dataset',
description=
'Classification of different attributes related to autonomous driving scenarios',
format_=['csv'],
schema='https://bdd-data.berkeley.edu/')
manifest_genenerator.add_ait_inventories(
name='High_risk_CA_combinations',
type_='attribute set',
description=
'Combinations of different attribute values that are high risk situations in real life',
format_=['csv'],
schema='User given data')
manifest_genenerator.add_ait_measures(
name='SingleCount',
type_='int',
description='Number of high risk cases in simple combinations',
structure='single',
manifest_genenerator = AITManifestGenerator(current_dir)
manifest_genenerator.set_ait_name('eval_dnn_coverage_tf1.13')
manifest_genenerator.set_ait_description('''
Calculate the neuron coverage for the input dataset given by the user, and display it in the form of a heat map. After that, a gradient for the input dataset is computed by backpropagation. Based on the gradient, the most efficient manipulation to input values in order for the neuron coverage to increase is selected. A number of data is removed from the dataset, and these removed data are converted to new data by the selected manipulation. Then, the new data is added to the dataset, and the coverage is recalculated by running the model with the updated dataset. Similarly, a gradient for the updated dataset is computed. These processes are repeated until the target coverage rate is achieved. Data manipulation algorithm is implemented by the user in Python.
''')
manifest_genenerator.set_ait_author('AIST')
manifest_genenerator.set_ait_email('')
manifest_genenerator.set_ait_version('0.1')
manifest_genenerator.set_ait_quality(
'https://airc.aist.go.jp/aiqm/quality/internal/Robustness_of_trained_model'
)
manifest_genenerator.set_ait_reference('')
manifest_genenerator.add_ait_inventories(
name='image_data',
type_='dataset',
description='MNIST image data',
format_=['gz'],
schema='http://yann.lecun.com/exdb/mnist/')
manifest_genenerator.add_ait_inventories(
name='label',
type_='dataset',
description='MNIST label data',
format_=['gz'],
schema='http://yann.lecun.com/exdb/mnist/')
manifest_genenerator.add_ait_inventories(
name='tf_ckpt',
type_='model',
description='''Tensorflow model datas.\n
This is loaded by `tf.train.import_meta_graph`.''',
format_=['*'],
schema='https://github.com/tensorflow/models/tree/master/official')
#########################################
if not is_ait_launch:
from ait_sdk.common.files.ait_manifest_generator import AITManifestGenerator
manifest_genenerator = AITManifestGenerator(current_dir)
manifest_genenerator.set_ait_name('eval_mnist_acc_tf2.3')
manifest_genenerator.set_ait_description('Only Sequential API Model\n\n<QualityMeasurement>\nAccuracy=TP+TNTP+FP+FN+TN\nPrecision=TPTP+FP\nRecall=TPTP+FN\nF−measure=2Recall∗PrecisionRecall+Precision\nAUC\n\n<Resources>\nROC曲線\n混同行列\nNG予測画像')
manifest_genenerator.set_ait_author('AIST')
manifest_genenerator.set_ait_email('')
manifest_genenerator.set_ait_version('0.1')
manifest_genenerator.set_ait_quality('https://airc.aist.go.jp/aiqm/quality/internal/Accuracy_of_trained_model')
manifest_genenerator.set_ait_reference('')
manifest_genenerator.add_ait_inventories(name='trained_model',
type_='model',
description='Tensorflow 2.3で学習したモデル',
format_=['h5'],
schema='https://support.hdfgroup.org/HDF5/doc/')
manifest_genenerator.add_ait_inventories(name='test_set_images',
type_='dataset',
description='テスト画像セット（MNISTフォーマット）',
format_=['gz'],
schema='http://yann.lecun.com/exdb/mnist/')
manifest_genenerator.add_ait_inventories(name='test_set_labels',
type_='dataset',
description='テスト画像ラベル（MNISTフォーマット）',
format_=['gz'],
schema='http://yann.lecun.com/exdb/mnist/')
manifest_genenerator.add_ait_parameters(name='class_count',
type_='int',
if not is_ait_launch:
from ait_sdk.common.files.ait_manifest_generator import AITManifestGenerator
manifest_genenerator = AITManifestGenerator(current_dir)
manifest_genenerator.set_ait_name('eval_ca_distribution')
manifest_genenerator.set_ait_description(
'Evaluating distribution of dataset with expected distribution')
manifest_genenerator.set_ait_author('AIST')
manifest_genenerator.set_ait_email('')
manifest_genenerator.set_ait_version('0.1')
manifest_genenerator.set_ait_quality(
'https://airc.aist.go.jp/aiqm/quality/internal/Distribution_of_training_data'
)
manifest_genenerator.set_ait_reference('')
manifest_genenerator.add_ait_inventories(
'Data', 'dataset',
'Classification of different attributes related to autonomous driving scenarios',
['csv'], 'https://bdd-data.berkeley.edu/')
manifest_genenerator.add_ait_parameters(
'attribute_no', 'int',
'Total number of attribute for distibution analysis', '6')
manifest_genenerator.add_ait_parameters(
'dimension', 'int',
'Dimensions of how many attributes to combine for distibution analysis',
'2')
manifest_genenerator.add_ait_resources(
'distibution_csv', 'table',
'Table of distribution for each combination')
manifest_genenerator.add_ait_resources(
'distibution_plot', 'picture',
'Plot of distribution for each combination')
manifest_genenerator.add_ait_downloads('Log', 'AITLog')
manifest_genenerator = AITManifestGenerator(current_dir)
manifest_genenerator.set_ait_name('eval_mnist_data_coverage')
manifest_genenerator.set_ait_description('''
Calculate coverage for the contour area and perimeter of the dataset MNIST image.\r\n
\r\n
𝐶𝑜𝑣[𝑥(𝑛)]=|𝑚𝑎𝑥{𝑥(𝑛)}−𝑚𝑖𝑛{𝑥(𝑛)}|/| 〖ℎ𝑖𝑔ℎ〗_𝑖−〖𝑙𝑜𝑤〗_𝑖 |
''')
manifest_genenerator.set_ait_author('AIST')
manifest_genenerator.set_ait_email('')
manifest_genenerator.set_ait_version('0.1')
manifest_genenerator.set_ait_quality(
'https://airc.aist.go.jp/aiqm/quality/internal/Diversity_of_test_data')
manifest_genenerator.set_ait_reference('')
manifest_genenerator.add_ait_inventories(
name='images',
type_='dataset',
description='MNIST images',
format_=['gz'],
schema='http://fashion-mnist.s3-website.eu-central-1.amazonaws.com')
manifest_genenerator.add_ait_inventories(
name='labels',
type_='dataset',
description='MNIST labels',
format_=['gz'],
schema='http://fashion-mnist.s3-website.eu-central-1.amazonaws.com')
manifest_genenerator.add_ait_parameters(name='coverage_category',
type_='str',
description='''
Specify what coverage rate to be calculated.\n
Area = Tire area\n
ArcLength = contour perimeter length\n
Mean = Average pixel value\n"
manifest_genenerator = AITManifestGenerator(current_dir)
manifest_genenerator.set_ait_name('eval_find_ca_error')
manifest_genenerator.set_ait_description(
'Evaluating quantity of unsound CA combination cases in dataset')
manifest_genenerator.set_ait_author('AIST')
manifest_genenerator.set_ait_email('')
manifest_genenerator.set_ait_version('0.1')
manifest_genenerator.set_ait_quality(
'https://airc.aist.go.jp/aiqm/quality/internal/Coverage_for_distinguished_problem_cases'
)
manifest_genenerator.set_ait_reference('')
manifest_genenerator.add_ait_inventories(
name='ca_data',
type_='dataset',
description=
'''Classification of different attributes related to autonomous driving scenarios.
Need header.
Include header for Unsound_CA_combinations.
''',
format_=['csv'],
schema='CSV')
manifest_genenerator.add_ait_inventories(
name='Unsound_CA_combinations',
type_='attribute set',
description=
'Combinations of different attribute values that are not possible in real life',
format_=['csv'],
schema='User given data')
manifest_genenerator.add_ait_parameters(
name='PCA',
type_='str',
description='Primary conditional attribute',
# should edit
#########################################
if not is_ait_launch:
from ait_sdk.common.files.ait_manifest_generator import AITManifestGenerator
manifest_genenerator = AITManifestGenerator(current_dir)
manifest_genenerator.set_ait_name('eval_coverage_ca_pairwise')
manifest_genenerator.set_ait_description('it is calculates that Data-set has how much coverage for the combination of a given pair-wise.\n1.Find out if each pairwise row exists in the dataset\n2.A search pairwise pattern and a matching amount is output to a file.\n3.Calculate what percentage of all pairwise patterns are matched.\nBelow, restrictions\n1.The dataset must have the same columns as the pairwise combination.\n2.It is error if the required column does not exist in dataset.\n3.The dataset may have an extra column.\n4.It does not support regular expression search.\n"*" Is treated as one character "*".\n5.Pairwise and dataset may have null value.\nデータセットが特定のペアワイズの組み合わせに対してどの程度のカバレッジを持っているかを計算する。\n1.ペアワイズの各行が、データセットに存在するか検索する\n2.検索したペアワイズのパターンとマッチした件数をファイル出力する。\n3.マッチした件数が全ペアワイズパターンのうちの何パーセントか計算する。\n以下、制約事項\n1.データセットは、ペアワイズの組み合わせと同じカラムを持っている必要がある。\n2.データセットに、必要なカラムが存在しない場合はエラーになる。\n3.データセットは、余分なカラムを持っていてもよい。\n4.正規表現検索には対応しない。\n”＊”は”＊”という１文字として扱う。\n5.ペアワイズやデータセットにnullは存在してもよい。')
manifest_genenerator.set_ait_author('AIST')
manifest_genenerator.set_ait_email('')
manifest_genenerator.set_ait_version('0.1')
manifest_genenerator.set_ait_quality('https://airc.aist.go.jp/aiqm/quality/internal/Diversity_of_test_data')
manifest_genenerator.set_ait_reference('')
manifest_genenerator.add_ait_inventories(name='pairwise_list',
type_='dataset',
description='Pairwise_list.csv',
format_=['csv'],
schema='https://www.sciencedirect.com/topics/computer-science/pairwise-comparison')
manifest_genenerator.add_ait_inventories(name='target',
type_='dataset',
description='target.csv',
format_=['csv'],
schema='https://www.sciencedirect.com/topics/computer-science/pairwise-comparison')
manifest_genenerator.add_ait_measures(name='coverage',
type_='float',
description='coverage of all patterns are matched',
structure='single',
min='0',
max='1')
manifest_genenerator.add_ait_resources(name='matching_result',
type_='table',
