def get_notebook_modules_info(repository_id):
with connect() as session:
repository = session.query(Repository).filter(
Repository.id == repository_id).first()
notebook_modules = []
if repository is None:
return notebook_modules
for name in repository.notebook_names:
if not name:
continue
notebook = session.query(Notebook).filter(
Notebook.repository_id == repository.id,
Notebook.name == name,
).first()
nb_modules_query = (
"SELECT * "
"FROM notebook_modules "
"WHERE notebook_id = {} AND repository_id = {}").format(
notebook.id, repository_id)
nb_modules = session.execute(nb_modules_query)
nb_module = nb_modules.fetchone()
notebook_obj = {}
notebook_obj['name'] = notebook.name
if nb_module is not None:
if 'any_any' in nb_module:
notebook_obj['modules'] = nb_module.any_any
if 'any_any_count' in nb_module:
notebook_obj['totalmodules'] = nb_module.any_any_count
notebook_modules.append(notebook_obj)
return notebook_modules
def main():
"""Main function"""
script_name = os.path.basename(__file__)[:-3]
parser = argparse.ArgumentParser(
description='Calculate MD5 hashes and the presence of exercise keywords')
parser.add_argument('-v', '--verbose', type=int, default=config.VERBOSE,
help='increase output verbosity')
parser.add_argument('-i', '--interval', type=int, nargs=2,
default=config.REPOSITORY_INTERVAL,
help='repository id interval')
parser.add_argument('-c', '--count', action='store_true',
help='count results')
parser.add_argument('-r', '--reverse', action='store_true',
help='iterate in reverse order')
parser.add_argument('--check', type=str, nargs='*',
default={'all', script_name, script_name + '.py'},
help='check name in .exit')
args = parser.parse_args()
config.VERBOSE = args.verbose
status = None
if not args.count:
status = StatusLogger(script_name)
status.report()
with connect() as session, savepid():
apply(
session,
status,
args.count,
args.interval,
args.reverse,
set(args.check)
)
def get_cell_modules_info(repository_id):
with connect() as session:
query = ("SELECT * "
"FROM cell_modules "
"WHERE repository_id = {}").format(repository_id)
cell_modules = pd.read_sql(query, session.connection())
return cell_modules
def get_repository_nb_info(repository_id):
with connect() as session:
repository = session.query(Repository).filter(
Repository.id == repository_id).first()
notebooks = []
if repository is None:
return notebooks
for name in repository.notebook_names:
if not name:
continue
notebook = session.query(Notebook).filter(
Notebook.repository_id == repository.id,
Notebook.name == name,
).first()
notebook_obj = {}
notebook_obj['name'] = notebook.name
notebook_obj['id'] = notebook.id
notebook_obj['nbformat'] = notebook.nbformat
notebook_obj['kernel'] = notebook.kernel
notebook_obj['language'] = notebook.language
notebook_obj['language_version'] = notebook.language_version
notebook_obj['max_execution_count'] = notebook.max_execution_count
notebook_obj['total_cells'] = notebook.total_cells
notebook_obj['code_cells'] = notebook.code_cells
notebook_obj[
'code_cells_with_output'] = notebook.code_cells_with_output
notebook_obj['markdown_cells'] = notebook.markdown_cells
notebook_obj['raw_cells'] = notebook.raw_cells
notebook_obj[
'unknown_cell_formats'] = notebook.unknown_cell_formats
notebook_obj['empty_cells'] = notebook.empty_cells
notebooks.append(notebook_obj)
return notebooks
def get_repository_info(repository_id):
with connect() as session:
repository_obj = session.query(Repository).filter(
Repository.id == repository_id).first()
repositories = {}
if repository_obj is None:
return repositories
notebooks_info = []
repositories[
"Link"] = "https://" + repository_obj.domain + "/" + repository_obj.repository
repositories["No. of Notebooks"] = repository_obj.notebooks_count
repositories["Setup Count"] = repository_obj.setups_count
repositories["Requirement Count"] = repository_obj.requirements_count
if repository_obj.setups_count > 0:
repositories["Setup"] = repository_obj.setups
elif repository_obj.requirements_count > 0:
repositories["Requirement"] = repositories[
"Link"] + '/tree/master/' + repository_obj.requirements
repositories["Valid Python Notebooks"] = valid_python_notebooks(
repository_id)
repositories["Invalid Python Notebooks"] = invalid_python_notebooks(
repository_id)
repositories[
"Count of notebooks without execution count"] = get_nb_without_execution_count(
repository_id)
repositories[
"Count of notebooks with execution count"] = get_nb_with_execution_count(
repository_id)
return repositories
def get_notebook_execution_info(repository_id):
with connect() as session:
repository = session.query(Repository).filter(
Repository.id == repository_id).first()
notebook_execution = []
if repository is None:
return notebook_execution
for name in repository.notebook_names:
if not name:
continue
notebook = session.query(Notebook).filter(
Notebook.repository_id == repository.id,
Notebook.name == name,
).first()
query = ("SELECT * "
"FROM executions "
"WHERE notebook_id = {} AND repository_id = {}").format(
notebook.id, repository_id)
executions = session.execute(query)
execution = executions.fetchone()
if execution is not None:
notebook_obj = {}
notebook_obj['name'] = notebook.name
if execution["reason"] is None:
notebook_obj['executionreason'] = "Success"
else:
notebook_obj['executionreason'] = execution["reason"]
notebook_obj['diffoncell'] = execution["diff"]
notebook_obj['diffcount'] = execution["diff_count"]
notebook_obj['duration'] = human_readable_duration(
execution["duration"])
notebook_obj['msg'] = execution["msg"]
notebook_execution.append(notebook_obj)
return notebook_execution
def get_reproduced_nb(repository_id):
with connect() as session:
execution_query = (
"SELECT id, notebook_id, mode, reason, msg, diff, cell, count, diff_count, timeout, duration, processed, skip "
"FROM executions "
"WHERE repository_id = {}").format(repository_id)
executions = pd.read_sql(execution_query, session.connection())
total_notebooks = get_total_notebooks(repository_id)
if total_notebooks == 0:
return
validity_notebooks = get_valid_invalid_nb(repository_id)
failed_installations = get_failed_installations(executions)
non_declared_dependencies = get_non_declared_dependencies(executions)
repro_exceptions = get_repro_exceptions(executions)
installed_dp = executions[(executions["processed"] > 0)
& (executions["mode"] == 3)]
non_declared_dp = executions[(executions["processed"] > 0)
& (executions["mode"] == 5)]
combined = pd.concat([installed_dp, non_declared_dp])
total_combined = len(combined)
repro_excluded_nbformat = len(
combined[combined["reason"] == "<Read notebook error>"])
combined = combined[combined["reason"] != "<Read notebook error>"]
with_exceptions = combined[combined["processed"] & 4 == 4]
exception_error = get_exception_error(combined, executions)
repro_timeout = get_repro_timeout(combined, total_notebooks)
nb_finished_unfinished_executions = get_nb_finished_unfinished_executions(
executions)
nb_results_difference = get_nb_results_difference(executions)
nb_execution_count = get_nb_execution_count(repository_id)
nb_output_cell_count = get_nb_output_cell_count(repository_id)
return json.dumps({
'failed_installations': failed_installations,
'non_declared_dependencies': non_declared_dependencies,
'repro_exceptions': repro_exceptions,
'exception_error': exception_error,
'nb_finished_unfinished_executions':
nb_finished_unfinished_executions,
'nb_results_difference': nb_results_difference,
'validity_notebooks': validity_notebooks,
'nb_execution_count': nb_execution_count,
'nb_output_cell_count': nb_output_cell_count
})
def get_total_notebooks(repository_id):
with connect() as session:
nb_query = ("SELECT count(id) "
"FROM notebooks "
"WHERE NOT (kernel = 'no-kernel' AND nbformat = '0') "
"AND total_cells != 0 "
"AND repository_id = {} ").format(repository_id)
result = session.execute(nb_query)
total_notebooks = result.scalar()
return total_notebooks
def get_nb_with_execution_count(repository_id):
with connect() as session:
query = ("SELECT * "
"FROM notebooks "
"WHERE repository_id = {}").format(repository_id)
notebooks = pd.read_sql(query, session.connection())
with_execution_count = notebooks[notebooks.max_execution_count >= -0]
nb_with_execution_count = len(with_execution_count)
return nb_with_execution_count
def main():
"""Main function"""
script_name = os.path.basename(__file__)[:-3]
parser = argparse.ArgumentParser(
description="Compress processed repositories")
parser.add_argument("-v",
"--verbose",
type=int,
default=config.VERBOSE,
help="increase output verbosity")
parser.add_argument("-z",
"--compression",
type=str,
default=config.COMPRESSION,
help="compression algorithm")
parser.add_argument("-e",
"--retry-errors",
action='store_true',
help="retry errors")
parser.add_argument("-i",
"--interval",
type=int,
nargs=2,
default=config.REPOSITORY_INTERVAL,
help="id interval")
parser.add_argument("-c",
"--count",
action='store_true',
help="count results")
parser.add_argument('-r',
'--reverse',
action='store_true',
help='iterate in reverse order')
parser.add_argument('-k',
'--keep-uncompressed',
action='store_true',
help='keep uncompressed files')
parser.add_argument('--check',
type=str,
nargs='*',
default={'all', script_name, script_name + '.py'},
help='check name in .exit')
args = parser.parse_args()
config.VERBOSE = args.verbose
status = None
if not args.count:
status = StatusLogger(script_name)
status.report()
config.COMPRESSION = args.compression
with connect() as session, savepid():
apply(session, status, args.keep_uncompressed, args.count,
args.interval, args.reverse, set(args.check))
def get_cell_type(repository_id):
with connect() as session:
query = ("SELECT * "
"FROM cells "
"WHERE repository_id = {}").format(repository_id)
cells = pd.read_sql(query, session.connection())
if not cells.empty:
fig, counts = display_counts(cells["cell_type"].value_counts(),
width=5,
show_values=True,
plot=False,
title='Types of cells in Notebooks')
return fig
def get_nblanguage_version(repository_id):
with connect() as session:
query = ("SELECT * "
"FROM notebooks "
"WHERE repository_id = {}").format(repository_id)
notebooks = pd.read_sql(query, session.connection())
if not notebooks.empty:
series = notebooks.groupby(["language_version", "language"
]).count()['kernel'].unstack()
series.plot(kind="bar",
title='Programming Language Version of Notebooks')
fig = plt.gcf()
return fig
def main():
"""Main function"""
script_name = os.path.basename(__file__)[:-3]
parser = argparse.ArgumentParser(
description="Extract requirement files from registered repositories")
parser.add_argument("-v",
"--verbose",
type=int,
default=config.VERBOSE,
help="increase output verbosity")
parser.add_argument("-n",
"--repositories",
type=int,
default=None,
nargs="*",
help="repositories ids")
parser.add_argument("-i",
"--interval",
type=int,
nargs=2,
default=config.REPOSITORY_INTERVAL,
help="id interval")
parser.add_argument("-e",
"--retry-errors",
action='store_true',
help="retry errors")
parser.add_argument("-c",
"--count",
action='store_true',
help="count results")
parser.add_argument('-r',
'--reverse',
action='store_true',
help='iterate in reverse order')
parser.add_argument('--check',
type=str,
nargs='*',
default={'all', script_name, script_name + '.py'},
help='check name in .exit')
args = parser.parse_args()
config.VERBOSE = args.verbose
status = None
if not args.count:
status = StatusLogger(script_name)
status.report()
with connect() as session, savepid():
apply(session, status, args.repositories or True,
0 if args.retry_errors else consts.R_REQUIREMENTS_ERROR,
args.count, args.interval, args.reverse, set(args.check))
def get_nblanguage(repository_id):
with connect() as session:
query = ("SELECT * "
"FROM notebooks "
"WHERE repository_id = {}").format(repository_id)
notebooks = pd.read_sql(query, session.connection())
if not notebooks.empty:
fig, counts = display_counts(
notebooks["language"].value_counts(),
width=5,
show_values=True,
plot=False,
title='Programming Language of Notebooks')
return fig
def github_crawler(github_url):
status = None
count = None
interval = None
reverse = None
check = 'all'
keep_uncompressed = 'False'
dispatches = set()
script_name = None
skip_env = False
skip_extract = 0
dry_run = 0
status = StatusLogger(script_name)
status.report()
with connect() as session, mount_basedir(), savepid():
repository = load_repository.load_repository_from_url(
session, github_url)
s1_notebooks_and_cells.apply(
SafeSession(session,
interrupted=consts.N_STOPPED), status, [repository.id]
or True, consts.R_N_ERROR, count, interval, reverse, set(check))
s2_requirement_files.apply(session, status, [repository.id] or True,
consts.R_REQUIREMENTS_ERROR, count,
interval, reverse, set(check))
s3_compress.apply(session, status, keep_uncompressed, count, interval,
reverse, set(check))
s4_markdown_features.apply(session, status, consts.C_PROCESS_ERROR,
count, interval, reverse, set(check))
s5_extract_files.apply(session, status, consts.R_COMPRESS_ERROR, count,
interval, reverse, set(check))
s6_cell_features.apply(SafeSession(session), status, dispatches, True,
consts.C_PROCESS_ERROR, consts.C_SYNTAX_ERROR,
consts.C_TIMEOUT, count, interval, reverse,
set(check))
result = s7_execute_repositories.apply(
session, repository.id, status, script_name, config.EXECUTION_MODE,
config.WITH_EXECUTION, config.WITH_DEPENDENCY,
consts.R_COMPRESS_ERROR, 3, consts.R_TROUBLESOME,
consts.R_UNAVAILABLE_FILES, skip_env, skip_extract, dry_run,
mode_rules, s7_execute_repositories.notebook_exec_mode, count,
interval, reverse, set(check))
p0_local_possibility.apply(session, status, count, interval, reverse,
set(check))
p1_notebook_aggregate.apply(session, status, consts.N_AGGREGATE_ERROR,
count, interval, reverse, set(check))
p2_sha1_exercises.apply(session, status, count, interval, reverse,
set(check))
return repository.id
def get_repository_notebook(repository_id, notebook_id):
with connect() as session:
filters = [Repository.id == repository_id]
repository = session.query(Repository).filter(*filters).first()
if repository is not None:
notebook_filters = [
Notebook.id == notebook_id,
Notebook.repository_id == repository_id
]
notebook_query = session.query(Notebook).filter(
*notebook_filters).first()
notebook_name = notebook_query.name
repository_path = repository.path
repository_name = repository.repository
return repository_name, repository_path, notebook_name
def get_repository_requirements_info(repository_id):
with connect() as session:
query = ("SELECT name, reqformat, content "
"FROM requirement_files "
"WHERE repository_id = {}").format(repository_id)
requirement_files = pd.read_sql(query, session.connection())
if requirement_files["reqformat"].item() == 'requirements.txt':
requirements = []
requirement_files_content = requirement_files["content"].item()
for rq in requirement_files_content.split('\r\n'):
requirements.append(rq)
return requirements
return requirement_files["content"]
def valid_python_notebooks(repository_id):
with connect() as session:
query = ("SELECT * "
"FROM notebooks "
"WHERE repository_id = {}").format(repository_id)
notebooks = pd.read_sql(query, session.connection())
valid_python_notebooks_count = len(
notebooks[(notebooks["language"] == "python")
& (notebooks["language_version"] != "unknown")
& ~((notebooks["kernel"] == "no-kernel")
& (notebooks["nbformat"] == "0"))
& (notebooks["total_cells"] != 0)
& (np.bitwise_and(notebooks["processed"], 16) == 0)])
return valid_python_notebooks_count
def main():
"""Main function"""
parser = argparse.ArgumentParser(description="Load Repository by URL")
parser.add_argument("url", type=str,
help="repository URL")
parser.add_argument("-v", "--verbose", type=int, default=config.VERBOSE,
help="increase output verbosity")
parser.add_argument("-b", "--branch", type=str,
help="specific branch")
parser.add_argument("-c", "--commit", type=str,
help="specific commit")
parser.add_argument("-e", "--clone-existing", action='store_true',
help="clone even if repository exists")
args = parser.parse_args()
config.VERBOSE = args.verbose
with connect() as session, mount_basedir(), savepid():
load_repository_from_url(
session, args.url, args.branch, args.commit, args.clone_existing
)
def main():
"""Main function"""
register_surrogateescape()
script_name = os.path.basename(__file__)[:-3]
parser = argparse.ArgumentParser(
description='Execute repositories')
parser.add_argument('-v', '--verbose', type=int, default=config.VERBOSE,
help='increase output verbosity')
parser.add_argument('-e', '--retry-errors', action='store_true',
help='retry errors')
parser.add_argument('-i', '--interval', type=int, nargs=2,
default=config.REPOSITORY_INTERVAL,
help='repository id interval')
parser.add_argument('-c', '--count', action='store_true',
help='count results')
parser.add_argument('-r', '--reverse', action='store_true',
help='iterate in reverse order')
parser.add_argument('--check', type=str, nargs='*',
default={'all', script_name, script_name + '.py'},
help='check name in .exit')
args = parser.parse_args()
config.VERBOSE = args.verbose
status = None
if not args.count:
status = StatusLogger(script_name)
status.report()
with connect() as session, savepid():
apply(
session,
status,
0 if args.retry_errors else consts.R_COMPRESS_ERROR,
args.count,
args.interval,
args.reverse,
set(args.check)
)
def get_cell_info(repository_id, notebook_id):
with connect() as session:
filters = [
Cell.repository_id == repository_id,
Cell.notebook_id == notebook_id,
]
query = (session.query(Cell).filter(*filters))
notebook_cells = []
for cell in query:
print("cell")
notebook_cell_obj = {}
notebook_cell_obj['id'] = cell.id
notebook_cell_obj['repository_id'] = cell.repository_id
notebook_cell_obj['notebook_id'] = cell.notebook_id
notebook_cell_obj['index'] = cell.index
notebook_cell_obj['cell_type'] = cell.cell_type
notebook_cell_obj['execution_count'] = cell.execution_count
notebook_cell_obj['lines'] = cell.lines
notebook_cell_obj['output_formats'] = cell.output_formats
notebook_cell_obj['source'] = cell.source
notebook_cell_obj['python'] = cell.python
notebook_cells.append(notebook_cell_obj)
return notebook_cells
def get_nb_output_cell_count(repository_id):
with connect() as session:
nb_query = (
"SELECT sum(code_cells) as sum_code_cells, sum(code_cells_with_output) as sum_code_cells_with_output "
"FROM notebooks "
"WHERE repository_id = {} ").format(repository_id)
notebooks = session.execute(nb_query).fetchone()
total_code_cells = notebooks.sum_code_cells
total_code_cells_with_output = notebooks.sum_code_cells_with_output
total_code_cells_without_output = total_code_cells - total_code_cells_with_output
nb_output_cell_count_data = [
total_code_cells_with_output, total_code_cells_without_output
]
nb_output_cell_count_labels = [
'Code Cells with Output', 'Code Cells without Output'
]
nb_output_cell_count = {
'data': nb_output_cell_count_data,
'labels': nb_output_cell_count_labels,
'title': 'Notebooks with Code cells with and without Output',
}
return nb_output_cell_count
def get_notebook(repository_id, notebook_id):
with connect() as session:
nbconvert_rdf = ''
name = ''
filters = [Repository.id == repository_id]
repository = session.query(Repository).filter(*filters).first()
notebook_filters = [
Notebook.id == notebook_id, Notebook.repository_id == repository_id
]
notebook_query = session.query(Notebook).filter(
*notebook_filters).first()
name = notebook_query.name
with mount_basedir():
if repository.path.exists():
execution_path = (config.EXECUTION_DIR / repository.hash_dir2)
if os.path.exists(execution_path):
notebook_path = execution_path
else:
notebook_path = repository.path
try:
with open(str(notebook_path / name)) as ofile:
notebook = ofile.read()
nbtordfconverter = nb2rdf.NBToRDFConverter()
notebook_json = nbformat.reads(notebook, as_version=4)
nbconvert_rdf = nbtordfconverter.convert_to_rdf(
name, notebook_json)
output_file_extension = 'ttl'
output_file = os.path.join(
repository.path, name + "." + output_file_extension)
open(output_file, 'w').write(str(nbconvert_rdf))
return str(nbconvert_rdf), name
except OSError as e:
vprint(3, "Failed to open notebook {}".format(e))
return str(nbconvert_rdf), name
def main():
"""Main function"""
script_name = os.path.basename(__file__)[:-3]
parser = argparse.ArgumentParser(
description="Extract notebooks from registered repositories")
parser.add_argument("-v",
"--verbose",
type=int,
default=config.VERBOSE,
help="increase output verbosity")
parser.add_argument("-n",
"--repositories",
type=int,
default=None,
nargs="*",
help="repositories ids")
parser.add_argument("-s",
"--status",
type=int,
default=consts.R_COMPRESS_OK +
consts.R_COMMIT_MISMATCH,
help="has processed status")
parser.add_argument("-z",
"--no",
type=int,
default=0,
help="does not have status")
parser.add_argument("-e",
"--retry-errors",
action='store_true',
help="retry errors")
parser.add_argument("-t",
"--retry-timeout",
action='store_true',
help="retry timeout")
parser.add_argument("-i",
"--interval",
type=int,
nargs=2,
default=config.REPOSITORY_INTERVAL,
help="id interval")
parser.add_argument("-c",
"--count",
action='store_true',
help="count results")
parser.add_argument('-r',
'--reverse',
action='store_true',
help='iterate in reverse order')
parser.add_argument('--check',
type=str,
nargs='*',
default={'all', script_name, script_name + '.py'},
help='check name in .exit')
args = parser.parse_args()
config.VERBOSE = args.verbose
status = None
if not args.count:
status = StatusLogger(script_name)
status.report()
with connect() as session, savepid():
apply(session, status, args.repositories or True, args.status, args.no,
args.count, args.interval, args.reverse, set(args.check))
def get_execution_order_json(repository_id, notebook_id, cell_types):
with connect() as session:
# Get Execution Order of Cells in a Notebook of a Repository
# Filter on the type of notebook cell
filters = [
Cell.repository_id == repository_id,
Cell.notebook_id == notebook_id,
]
if int(cell_types) == consts.CODE_CELL_TYPE:
filters += [Cell.cell_type == 'code']
elif int(cell_types) == consts.OTHER_CELL_TYPE:
filters += [Cell.cell_type != 'code']
query = (session.query(Cell).filter(*filters))
# Create a dictionary with Cell Index and Cell Execution Count
notebook_cells_json = []
notebook_cell_obj = {}
for cell in query:
if cell.execution_count is not None:
notebook_cell_obj[cell.index] = int(cell.execution_count)
notebook_cells_json = {
k: v
for k, v in sorted(notebook_cell_obj.items(),
key=lambda item: item[1])
}
# Sort the Execution order based on Execution Count
execution_order = []
for k, v in sorted(notebook_cell_obj.items(),
key=lambda item: item[1]):
execution_order.append(k)
# JSON for the execution order
execution_order_json = {"nodes": [], "links": []}
node_name = ''
node_label = 'Cell'
node_id = ''
source = ''
target = ''
node_obj = {}
link_obj = {"source": '', "target": '', "type": "followedby"}
for cell in query:
node_obj = {}
node_name = cell.execution_count
node_id = cell.index
if node_name:
node_obj["name"] = node_name
else:
node_obj["name"] = "-"
node_obj["id"] = node_id
node_obj["label"] = "Cell " + str(node_id)
execution_order_json["nodes"].append(node_obj)
for item, next_item in zip(execution_order, execution_order[1:]):
link_obj = {}
link_obj["source"] = item
link_obj["target"] = next_item
link_obj["type"] = 'followedby'
execution_order_json["links"].append(link_obj)
return execution_order_json
def main():
"""Main function"""
register_surrogateescape()
script_name = os.path.basename(__file__)[:-3]
parser = argparse.ArgumentParser(description='Execute repositories')
parser.add_argument('-v',
'--verbose',
type=int,
default=config.VERBOSE,
help='increase output verbosity')
parser.add_argument("-n",
"--notebooks",
type=int,
default=None,
nargs="*",
help="notebooks ids")
parser.add_argument('-e',
'--retry-errors',
action='store_true',
help='retry errors')
parser.add_argument('-s',
'--retry-syntaxerrors',
action='store_true',
help='retry syntax errors')
parser.add_argument('-t',
'--retry-timeout',
action='store_true',
help='retry timeout')
parser.add_argument('-i',
'--interval',
type=int,
nargs=2,
default=config.REPOSITORY_INTERVAL,
help='repository id interval')
parser.add_argument('-c',
'--count',
action='store_true',
help='count results')
parser.add_argument('-r',
'--reverse',
action='store_true',
help='iterate in reverse order')
parser.add_argument('--check',
type=str,
nargs='*',
default={'all', script_name, script_name + '.py'},
help='check name in .exit')
args = parser.parse_args()
config.VERBOSE = args.verbose
status = None
if not args.count:
status = StatusLogger(script_name)
status.report()
dispatches = set()
with savepid():
with connect() as session:
apply(SafeSession(session), status, dispatches, args.notebooks
or True, 0 if args.retry_errors else consts.C_PROCESS_ERROR,
0 if args.retry_syntaxerrors else consts.C_SYNTAX_ERROR,
0 if args.retry_timeout else consts.C_TIMEOUT, args.count,
args.interval, args.reverse, set(args.check))
pos_apply(dispatches, args.retry_errors, args.retry_timeout,
args.verbose)
def main():
"""Main function"""
script_name = os.path.basename(__file__)[:-3]
parser = argparse.ArgumentParser(description="Execute repositories")
parser.add_argument("-v",
"--verbose",
type=int,
default=config.VERBOSE,
help="increase output verbosity")
parser.add_argument("-e",
"--retry-errors",
action='store_true',
help="retry errors")
parser.add_argument("-f",
"--discover-deleted",
action='store_true',
help="try to discover deleted files")
parser.add_argument("-i",
"--interval",
type=int,
nargs=2,
default=config.REPOSITORY_INTERVAL,
help="repository id interval")
parser.add_argument("-z",
"--retry-troublesome",
action='store_true',
help="retry troublesome")
parser.add_argument(
"-x",
"--with-execution",
type=int,
default=config.WITH_EXECUTION,
help="-1: without execution; 0: all; 1: with execution")
parser.add_argument(
"-d",
"--with-dependency",
type=int,
default=config.WITH_DEPENDENCY,
help="-1: without dependency; 0: all; 1: with dependency")
parser.add_argument(
"-m",
"--execution-mode",
type=int,
default=config.EXECUTION_MODE,
help="-1: auto; 1: cellorder, 2: dependencies, 4: anaconda")
parser.add_argument("-c",
"--count",
action='store_true',
help="count results")
parser.add_argument(
"--dry-run",
type=int,
default=0,
help="dry-run level. 0 runs everything. 1 does not execute. "
"2 does not install dependencies. 3 does not extract files. "
"4 does not prepare conda environment")
parser.add_argument('-r',
'--reverse',
action='store_true',
help='iterate in reverse order')
parser.add_argument('--check',
type=str,
nargs='*',
default={'all', script_name, script_name + '.py'},
help='check name in .exit')
parser.add_argument("--skip-env",
action='store_true',
help="skip environment")
parser.add_argument("--skip-extract",
action='store_true',
help="skip extraction")
args = parser.parse_args()
config.VERBOSE = args.verbose
status = None
if not args.count:
status = StatusLogger(script_name)
status.report()
with connect() as session, savepid():
apply(session, status, script_name, args.execution_mode,
args.with_execution, args.with_dependency,
0 if args.retry_errors else consts.R_COMPRESS_ERROR,
1 if args.retry_errors else 3,
0 if args.retry_troublesome else consts.R_TROUBLESOME,
0 if args.discover_deleted else consts.R_UNAVAILABLE_FILES,
args.skip_env, args.skip_extract, args.dry_run,
mode_rules_cell_order, notebook_exec_mode_cell_order, args.count,
args.interval, args.reverse, set(args.check))
