def cli_rename(api: GbdApi, args):
if not api.feature_exists(args.old_name):
eprint("Feature '{}' does not exist or is virtual".format(
args.old_name))
elif api.feature_exists(args.new_name):
eprint("Feature '{}' does already exist".format(args.new_name))
else:
api.rename_feature(args.old_name, args.new_name)
def cli_info(api: GbdApi, args):
if args.name is None:
for db_str in api.get_databases():
print("Database: {}".format(db_str))
print("Features: {}".format(" ".join(
api.get_material_features(db_str))))
print("Virtual: {}".format(" ".join(
api.get_virtual_features(db_str))))
else:
info = api.get_feature_info(args.name)
for key in info:
print("{}: {}".format(key, info[key]))
def cli_delete(api: GbdApi, args):
if api.feature_exists(args.name):
if (not args.hashes
or len(args.hashes) == 0) and not sys.stdin.isatty():
args.hashes = read_hashes()
if args.hashes and len(args.hashes) > 0:
if args.force or confirm(
"Delete attributes of given hashes from '{}'?".format(
args.name)):
api.remove_attributes(args.name, args.hashes)
elif args.force or confirm(
"Delete feature '{}' and all associated attributes?".format(
args.name)):
api.remove_feature(args.name)
else:
eprint("Feature '{}' does not exist or is virtual".format(args.name))
def get_all_attributes(hashvalue):
with GbdApi(app.config['database']) as gbd_api:
features = gbd_api.get_features()
info = dict([])
for feature in features:
values = gbd_api.search(feature, hashvalue)
info.update({feature: str(",".join(str(value) for value in values))})
return Response(json.dumps(info), status=200, mimetype="application/json")
def cli_get(api: GbdApi, args):
hashes = []
if not sys.stdin.isatty():
hashes = read_hashes()
resultset = api.query_search(args.query, hashes, args.resolve,
args.collapse, args.group_by)
for result in resultset:
print(args.separator.join([(str(item or '')) for item in result]))
def get_attribute(feature, hashvalue):
with GbdApi(app.config['database']) as gbd_api:
try:
values = gbd_api.search(feature, hashvalue)
if len(values) == 0:
return Response("No feature associated with this hash", status=404, mimetype="text/plain")
return str(",".join(str(value) for value in values))
except ValueError as err:
return Response("Value Error: {}".format(err), status=500, mimetype="text/plain")
def get_file(hashvalue, filename):
with GbdApi(app.config['database']) as gbd_api:
values = gbd_api.search("local", hashvalue)
if len(values) == 0:
return Response("No according file found in our database", status=404, mimetype="text/plain")
try:
path = values.pop()
return send_file(path, as_attachment=True, attachment_filename=os.path.basename(path))
except FileNotFoundError:
return Response("Files temporarily not accessible", status=404, mimetype="text/plain")
def list_features(database):
with GbdApi(app.config['database']) as gbd_api:
if database is None:
available_features = sorted(gbd_api.get_features())
available_features.remove("local")
return Response(json.dumps(available_features), status=200, mimetype="application/json")
elif database not in list(map(basename, gbd_api.get_databases())):
return Response("Database does not exist in the running instance of GBD server", status=404,
mimetype="text/plain")
else:
target_database = list(filter(lambda x: basename(x) == database, gbd_api.get_databases()))[0]
return Response(json.dumps(gbd_api.get_features(target_database)), status=200, mimetype="application/json")
def set_tag(hash, name, value):
pat = re.compile(r"^[a-zA-Z0-9_]*$")
if not (pat.match(hash) and pat.match(name) and pat.match(value)):
return Response("Input violates restriction to alpha-numeric characters and underline", status=406, mimetype="text/plain")
with GbdApi(app.config['database']) as gbd_api:
try:
if gbd_api.get_feature_size("tags") > 10*gbd_api.get_feature_size("local"):
return Response("Too many tags, cleanup required", status=503, mimetype="text/plain")
else:
gbd_api.set_tag(name, value, [hash])
return Response("Successfully set tag {}={} for {}".format(name, value, hash), status=201, mimetype="text/plain")
except ValueError as err:
return Response("Rejected: {}".format(err), status=406, mimetype="text/plain")
def get_url_file():
with GbdApi(app.config['database']) as gbd_api:
query = request.form.get('query')
try:
result = gbd_api.query_search(query, [], ["filename"])
except ValueError as err:
return Response("Feature not found", status=400, mimetype="text/plain")
content = "\n".join(
[flask.url_for("get_file", hashvalue=row[0], filename=row[1], _external=True) for row in result])
app.logger.info('Sending URL file to {}'.format(request.remote_addr))
file_name = "query_result.uri"
return Response(content, mimetype='text/uri-list',
headers={"Content-Disposition": "attachment; filename=\"{}\"".format(file_name),
"filename": "{}".format(file_name)})
def get_database_file(database):
with GbdApi(app.config['database']) as gbd_api:
if database is None:
return send_file(gbd_api.get_databases()[0],
as_attachment=True,
attachment_filename=os.path.basename(gbd_api.get_databases()[0]),
mimetype='application/x-sqlite3')
elif database not in list(map(basename, gbd_api.get_databases())):
return Response("Database does not exist in the running instance of GBD server", status=404,
mimetype="text/plain")
else:
return send_file(list(filter(lambda x: basename(x) == database, gbd_api.get_databases()))[0],
as_attachment=True,
attachment_filename=database,
mimetype='application/x-sqlite3')
def quick_search_results():
with GbdApi(app.config['database']) as gbd_api:
query = request.form.get('query')
selected_features = list(filter(lambda x: x != '', request.form.get('selected_features').split(',')))
if not len(selected_features):
selected_features.append("filename")
available_features = sorted(gbd_api.get_features())
available_features.remove("local")
features = sorted(list(set(available_features) & set(selected_features)))
try:
rows = list(gbd_api.query_search(query, [], features))
features.insert(0, "GBDhash")
result = list(dict((features[index], row[index]) for index in range(0, len(features))) for row in rows)
return Response(json.dumps(result), status=200, mimetype="application/json")
except ValueError as err:
return Response(str(err), status=400, mimetype="text/plain")
def get_csv_file():
with GbdApi(app.config['database']) as gbd_api:
query = request.form.get('query')
selected_features = list(filter(lambda x: x != '', request.form.get('selected_features').split(',')))
if not len(selected_features):
selected_features.append("filename")
try:
results = gbd_api.query_search(query, [], selected_features)
except ValueError as err:
return Response("Feature not found", status=400, mimetype="text/plain")
headers = ["hash"] + selected_features
content = "\n".join([" ".join([str(entry) for entry in result]) for result in results])
app.logger.info('Sending CSV file to {}'.format(request.remote_addr))
file_name = "query_result.csv"
return Response(" ".join(headers) + "\n" + content, mimetype='text/csv',
headers={"Content-Disposition": "attachment; filename=\"{}\"".format(file_name),
"filename": "{}".format(file_name)})
def cli_info_set(api: GbdApi, args):
api.meta_set(args.feature, args.name, args.value)
def cli_init_sanitize(api: GbdApi, args):
api.bootstrap("sanitation_info", args.hashes)
def cli_create(api: GbdApi, args):
if not api.feature_exists(args.name):
api.create_feature(args.name, args.unique)
else:
eprint("Feature '{}' does already exist".format(args.name))
def cli_init_ct(api: GbdApi, args):
api.bootstrap("clause_types", args.hashes)
def cli_init_dsh(api: GbdApi, args):
api.bootstrap("degree_sequence_hash", args.hashes)
def cli_import(api: GbdApi, args):
path = os.path.abspath(args.path)
api.import_file(path, args.key, args.source, args.target)
def cli_init_local(api: GbdApi, args):
path = os.path.abspath(args.path)
api.init_database(path)
def main():
parser = argparse.ArgumentParser(
description='Access and maintain the global benchmark database.')
parser.add_argument('-d',
"--db",
help='Specify database to work with',
default=os.environ.get('GBD_DB'),
nargs='?')
parser.add_argument('-j',
"--jobs",
help='Specify number of parallel jobs',
default=1,
nargs='?')
parser.add_argument(
'-s',
"--separator",
choices=[" ", ",", ";"],
default=" ",
help="Feature separator (delimiter used in import and output)")
parser.add_argument(
'-t',
"--join-type",
choices=["INNER", "OUTER", "LEFT"],
default="LEFT",
help="Join Type: treatment of missing values in queries")
parser.add_argument(
'-v',
'--verbose',
action='store_true',
help='Print additional (or diagnostic) information to stderr')
subparsers = parser.add_subparsers(help='Available Commands:')
# INITIALIZATION AND BOOTSTRAPPING
parser_init = subparsers.add_parser('init', help='Initialize Database')
parser_init_subparsers = parser_init.add_subparsers(
help='Select Initialization Procedure:')
# init local paths:
parser_init_local = parser_init_subparsers.add_parser(
'local', help='Initialize Local Hash/Path Entries')
parser_init_local.add_argument('path',
type=directory_type,
help="Path to benchmarks")
parser_init_local.set_defaults(func=cli_init_local)
# init clause types:
parser_init_ct = parser_init_subparsers.add_parser(
'clause_types', help='Initialize Clause-Type Tables')
parser_init_ct.add_argument('hashes', help='Hashes', nargs='+')
parser_init_ct.set_defaults(func=cli_init_ct)
# init degree_sequence_hash:
parser_init_dsh = parser_init_subparsers.add_parser(
'degree_sequence_hash', help='Initialize Degree-Sequence Hash')
parser_init_dsh.add_argument('hashes', help='Hashes', nargs='+')
parser_init_dsh.set_defaults(func=cli_init_dsh)
# init sanitation info
parser_init_sanitize = parser_init_subparsers.add_parser(
'sanitize', help='Check Instances, Store Sanitation Info')
parser_init_sanitize.add_argument('hashes', help='Hashes', nargs='+')
parser_init_sanitize.set_defaults(func=cli_init_sanitize)
# GBD HASH
parser_hash = subparsers.add_parser('hash',
help='Print hash for a single file')
parser_hash.add_argument('path',
type=file_type,
help="Path to one benchmark")
parser_hash.set_defaults(func=cli_hash)
# GET/SET ATTRIBUTES
parser_get = subparsers.add_parser(
'get', help='Get data by query (or hash-list via stdin)')
parser_get.add_argument(
'query',
help=
'Specify a query-string (e.g. "variables > 100 and path like %%mp1%%")',
nargs='?')
parser_get.add_argument('-r',
'--resolve',
help='List of features to resolve against',
nargs='+')
parser_get.add_argument(
'-c',
'--collapse',
default='group_concat',
choices=['group_concat', 'min', 'max', 'avg', 'count', 'sum'],
help='Treatment of multiple values per hash (or grouping value resp.)')
parser_get.add_argument('-g',
'--group_by',
default='hash',
help='Group by specified attribute value')
parser_get.set_defaults(func=cli_get)
parser_set = subparsers.add_parser(
'set',
help=
'Set specified attribute-value for given hashes (via argument or stdin)'
)
parser_set.add_argument('hashes', help='Hashes', nargs='*')
parser_set.add_argument('-n',
'--name',
type=column_type,
help='Feature name',
required=True)
parser_set.add_argument('-v',
'--value',
help='Attribute value',
required=True)
parser_set.add_argument('-f',
'--force',
action='store_true',
help='Overwrite existing unique values')
parser_set.set_defaults(func=cli_set)
# IMPORT DATA FROM CSV
parser_import = subparsers.add_parser(
'import', help='Import attributes from csv-file')
parser_import.add_argument('path', type=file_type, help="Path to csv-file")
parser_import.add_argument('-k',
'--key',
type=column_type,
help="Name of the key column (gbd-hash)",
required=True)
parser_import.add_argument('-s',
'--source',
help="Name of source column in csv-file",
required=True)
parser_import.add_argument('-t',
'--target',
type=column_type,
help="Name of target column (in database)",
required=True)
parser_import.set_defaults(func=cli_import)
# CREATE/DELETE/MODIFY FEATURES
parser_create = subparsers.add_parser('create',
help='Create a new feature')
parser_create.add_argument('name',
type=column_type,
help='Name of feature')
parser_create.add_argument(
'-u',
'--unique',
help='Unique constraint: specify default-value of feature')
parser_create.set_defaults(func=cli_create)
parser_delete = subparsers.add_parser(
'delete',
help=
'Delete all values assiociated with given hashes (via argument or stdin) or remove feature if no hashes are given'
)
parser_delete.add_argument('hashes', help='Hashes', nargs='*')
parser_delete.add_argument('name',
type=column_type,
help='Name of feature')
parser_delete.add_argument('-f',
'--force',
action='store_true',
help='Do not ask for confirmation')
parser_delete.set_defaults(func=cli_delete)
parser_rename = subparsers.add_parser('rename', help='Rename feature')
parser_rename.add_argument('old_name',
type=column_type,
help='Old name of feature')
parser_rename.add_argument('new_name',
type=column_type,
help='New name of feature')
parser_rename.set_defaults(func=cli_rename)
# HANDLE META-FEATURES (e.g. specify runtime meta-data like timeout/memout/machine)
parser_info = subparsers.add_parser(
'info', help='Print info about available features')
parser_info.add_argument('name',
type=column_type,
help='Print info about specified feature',
nargs='?')
parser_info.set_defaults(func=cli_info)
parser_info_set = subparsers.add_parser('info_set',
help='Set feature meta-attributes')
parser_info_set.add_argument('feature',
type=column_type,
help='Feature name')
parser_info_set.add_argument('-n',
'--name',
type=column_type,
help='Meta-feature name',
required=True)
parser_info_set.add_argument('-v',
'--value',
help='Meta-feature value',
required=True)
parser_info_set.set_defaults(func=cli_info_set)
parser_info_clear = subparsers.add_parser(
'info_clear', help='Clear feature meta-attributes')
parser_info_clear.add_argument('feature',
type=column_type,
help='Feature name')
parser_info_clear.add_argument('-n',
'--name',
type=column_type,
help='Meta-feature name')
parser_info_clear.set_defaults(func=cli_info_clear)
# SCORE CALCULATION
parser_par2 = subparsers.add_parser(
'par2', help='Calculate PAR-2 score for given runtime feature')
parser_par2.add_argument(
'query',
help=
'Specify a query-string (e.g. "variables > 100 and path like %%mp1%%")',
nargs='?')
parser_par2.add_argument('name',
type=column_type,
help='Name of runtime feature')
parser_par2.set_defaults(func=cli_par2)
# EVALUATE ARGUMENTS
args = parser.parse_args()
if not args.db:
eprint("""No database path is given.
A database path can be given in two ways:
-- by setting the environment variable GBD_DB
-- by giving a path via --db=[path]
A database file containing some attributes of instances used in the SAT Competitions can be obtained at http://gbd.iti.kit.edu/getdatabase
Initialize your database with local paths to your benchmark instances by using the init-command. """
)
elif len(sys.argv) > 1:
try:
with GbdApi(args.db, int(args.jobs), args.separator,
args.join_type, args.verbose) as api:
args.func(api, args)
except AttributeError as e:
eprint(e)
else:
parser.print_help()
def cli_hash(api: GbdApi, args):
path = os.path.abspath(args.path)
print(GbdApi.hash_file(path))
def list_databases():
with GbdApi(app.config['database']) as gbd_api:
return Response(json.dumps(list(map(basename,
gbd_api.get_databases()))),
status=200,
mimetype="application/json")
def cli_info_clear(api: GbdApi, args):
api.meta_clear(args.feature, args.name)
def quick_search():
with GbdApi(app.config['database']) as gbd_api:
pass
return render_template('index.html')
def cli_par2(api: GbdApi, args):
api.calculate_par2_score(args.query, args.name)
def cli_set(api: GbdApi, args):
if (not args.hashes or len(args.hashes) == 0) and not sys.stdin.isatty():
args.hashes = read_hashes()
api.set_attribute(args.name, args.value, args.hashes, args.force)