def make_spectra(project_name, projectdir, versions, remake):
versions = list(versions)
assert len(versions) > 0
original_dir = os.getcwd()
os.chdir(projectdir)
project = projects.get_project(project_name, versions[0])
test_runner.initialize_directory(project)
runner = test_runner.TestRunner(project)
runner.load_correct_outputs()
os.chdir(original_dir)
for v in versions:
project = projects.get_project(project_name, v)
outfile = get_spectra_file(project.name, project.version)
exists = os.path.exists(outfile)
if exists and not remake:
print("Spectra already exists for {0}. Skipping".format(v))
continue
os.chdir(projectdir)
print("Running version {0}".format(v))
runner.set_project(project)
run_to_result = runner.get_buggy_version_results()
os.chdir(original_dir)
if not os.path.exists(SPECTRA_DIR):
os.mkdir(SPECTRA_DIR)
outfile = get_spectra_file(project.name, project.version)
print("Saving results to {0}".format(outfile))
run_result.save(outfile, run_to_result)
def import_ref(project_id, ref_name):
if not config.validation.is_valid_project_id(project_id):
raise ValueError('Invalid project id "%s"' % project_id)
if not config.validation.is_valid_ref_name(ref_name):
raise ValueError('Invalid ref name "%s"' % ref_name)
project = projects.get_project(project_id)
if project is None:
raise NotFoundError('project %s not found' % project_id)
if project.config_location.storage_type != GITILES_LOCATION_TYPE:
raise Error('project %s is not a Gitiles project' % project_id)
loc = gitiles.Location.parse_resolve(project.config_location.url)
ref = None
for r in projects.get_refs([project_id])[project_id] or ():
if r.name == ref_name:
ref = r
if ref is None:
raise NotFoundError(
('ref "%s" is not found in project %s. '
'Possibly it is not declared in projects/%s:refs.cfg') %
(ref_name, project_id, project_id))
cfg = get_gitiles_config()
loc = loc._replace(
treeish=ref_name,
path=ref.config_path or cfg.ref_config_default_path,
)
_import_config_set('projects/%s/%s' % (project_id, ref_name), loc)
def set_calc_status(user_id, project_codename, calc_name, status):
project = projects.get_project(user_id, project_codename)
if not project:
raise RuntimeError("Unknown project " + project_codename)
g_db = core.api_util.DatabaseContext.get_conn()
if status == STATUS_RUNNING:
g_db.execute(
"""UPDATE calculations
SET status = %s,
last_start_date = now()
WHERE project_uid = %s
AND name = %s
AND delete_date IS NULL""",
[status, project_codename, calc_name])
elif status == STATUS_STOPPED:
g_db.execute(
"""UPDATE calculations
SET status = %s,
last_stop_date = now()
WHERE project_uid = %s
AND name = %s
AND delete_date IS NULL""",
[status, project_codename, calc_name])
else:
g_db.execute(
"""UPDATE calculations
SET status = %s
WHERE project_uid = %s
AND name = %s
AND delete_date IS NULL""",
[status, project_codename, calc_name])
def import_ref(project_id, ref_name):
if not config.validation.is_valid_project_id(project_id):
raise ValueError('Invalid project id "%s"' % project_id)
if not config.validation.is_valid_ref_name(ref_name):
raise ValueError('Invalid ref name "%s"' % ref_name)
project = projects.get_project(project_id)
if project is None:
raise NotFoundError('project %s not found' % project_id)
if project.HasField('gitiles_location'):
loc = _make_gitiles_location(project.gitiles_location)
else:
raise Error('project %s does not have a gitiles location' % project_id)
ref = None
for r in projects.get_refs([project_id])[project_id] or ():
if r.name == ref_name:
ref = r
if ref is None:
raise NotFoundError(
('ref "%s" is not found in project %s. '
'Possibly it is not declared in projects/%s:refs.cfg') %
(ref_name, project_id, project_id))
cfg = get_gitiles_config()
loc = loc._replace(
treeish=ref_name,
path=ref.config_path or cfg.ref_config_default_path,
)
_import_config_set('projects/%s/%s' % (project_id, ref_name), loc)
def import_project(project_id):
if not config.validation.is_valid_project_id(project_id):
raise ValueError('Invalid project id: %s' % project_id)
project = projects.get_project(project_id)
if project is None:
raise NotFoundError('project %s not found' % project_id)
if project.config_location.storage_type != GITILES_LOCATION_TYPE:
raise Error('project %s is not a Gitiles project' % project_id)
loc = gitiles.Location.parse_resolve(project.config_location.url)
# Adjust location
cfg = get_gitiles_config()
if not loc.treeish or loc.treeish == 'HEAD':
loc = loc._replace(treeish=cfg.project_config_default_ref)
loc = loc._replace(
path=loc.path.strip('/') or cfg.project_config_default_path,
)
# Update project repo info.
repo_url = str(loc._replace(treeish=None, path=None))
projects.update_import_info(
project_id, projects.RepositoryType.GITILES, repo_url)
_import_config_set('projects/%s' % project_id, loc)
def import_ref(project_id, ref_name):
if not config.validation.is_valid_project_id(project_id):
raise ValueError('Invalid project id "%s"' % project_id)
if not config.validation.is_valid_ref_name(ref_name):
raise ValueError('Invalid ref name "%s"' % ref_name)
project = projects.get_project(project_id)
if project is None:
raise NotFoundError('project %s not found' % project_id)
if project.config_location.storage_type != GITILES_LOCATION_TYPE:
raise Error('project %s is not a Gitiles project' % project_id)
loc = gitiles.Location.parse_resolve(project.config_location.url)
ref = projects.get_ref(project_id, ref_name)
if ref is None:
raise NotFoundError(
('ref "%s" is not found in project %s. '
'Possibly it is not declared in projects/%s:refs.cfg') %
(ref_name, project_id, project_id))
cfg = get_gitiles_config()
loc = loc._replace(
treeish=ref_name,
path=ref.config_path or cfg.ref_config_default_path,
)
_import_config_set('projects/%s/%s' % (project_id, ref_name), loc)
def import_project(project_id):
if not config.validation.is_valid_project_id(project_id):
raise ValueError('Invalid project id: %s' % project_id)
config_set = 'projects/%s' % project_id
project = projects.get_project(project_id)
if project is None:
raise NotFoundError('project %s not found' % project_id)
if project.config_location.storage_type != GITILES_LOCATION_TYPE:
raise Error('project %s is not a Gitiles project' % project_id)
try:
loc = _resolved_location(project.config_location.url)
except gitiles.TreeishResolutionError:
@ndb.transactional
def txn():
key = ndb.Key(storage.ConfigSet, config_set)
if key.get():
logging.warning(
'treeish was not resolved in URL "%s" => delete project',
project.config_location.url)
key.delete()
txn()
return
# Update project repo info.
repo_url = str(loc._replace(treeish=None, path=None))
projects.update_import_info(project_id, projects.RepositoryType.GITILES,
repo_url)
_import_config_set(config_set, loc, project_id)
def import_project(project_id, loc=None):
if not config.validation.is_valid_project_id(project_id):
raise ValueError('Invalid project id: %s' % project_id)
if loc is None:
project = projects.get_project(project_id)
if project is None:
raise NotFoundError('project %s not found' % project_id)
if project.config_location.storage_type != GITILES_LOCATION_TYPE:
raise Error('project %s is not a Gitiles project' % project_id)
loc = gitiles.Location.parse_resolve(project.config_location.url)
# Adjust location
cfg = get_gitiles_config()
if not loc.treeish or loc.treeish == 'HEAD':
loc = loc._replace(treeish=cfg.project_config_default_ref)
loc = loc._replace(
path=loc.path.strip('/') or cfg.project_config_default_path,
)
# Update project repo info.
repo_url = str(loc._replace(treeish=None, path=None))
projects.update_import_info(
project_id, projects.RepositoryType.GITILES, repo_url)
_import_config_set('projects/%s' % project_id, loc)
def create_payment(id):
project_information = projects.get_project(id)
subproject_list = subprojects.list_subprojects(id)
category_list = categories.list_categories(id)
return render_template("createpayment.html",
project_information=project_information,
subproject_list=subproject_list,
category_list=category_list)
def get_mesh(user_id, project_codename, mesh_name, include_deleted=False):
project = projects.get_project(user_id, project_codename)
if not project:
raise RuntimeError("Unknown project " + project_codename)
query = """SELECT * FROM meshes WHERE project_uid = %s AND name = %s"""
if not include_deleted:
query += " AND delete_date IS NULL"
g_db = core.api_util.DatabaseContext.get_conn()
return g_db.execute(query, [project_codename, mesh_name]).fetchone()
def get_calc(user_id, project_codename, calc_id, include_deleted=False):
g_db = core.api_util.DatabaseContext.get_conn()
project = projects.get_project(user_id, project_codename)
if not project:
raise RuntimeError("Unknown project " + project_codename)
query = """SELECT * FROM calculations WHERE project_uid = %s AND id = %s"""
if not include_deleted:
query += " AND delete_date IS NULL"
return g_db.execute(query, [project_codename, calc_id]).fetchone()
def set_job(user_id, project_codename, calc_name, job_id):
project = projects.get_project(user_id, project_codename)
if not project:
raise RuntimeError("Unknown project " + project_codename)
g_db = core.api_util.DatabaseContext.get_conn()
g_db.execute(
"""UPDATE calculations
SET job_id = %s
WHERE project_uid = %s
AND name = %s
AND delete_date IS NULL""",
[job_id, project_codename, calc_name])
def save_status_file(user_id, project_codename, calc_id, status_file_id):
project = projects.get_project(user_id, project_codename)
if not project:
raise RuntimeError("Unknown project " + project_codename)
g_db = core.api_util.DatabaseContext.get_conn()
g_db.execute(
"""UPDATE calculations
SET status_file_id = %s
WHERE project_uid = %s
AND id = %s
AND delete_date IS NULL""",
[status_file_id, project_codename, calc_id])
def set_mesh_status(user_id, project_codename, mesh_name, status):
project = projects.get_project(user_id, project_codename)
if not project:
raise RuntimeError("Unknown project " + project_codename)
g_db = core.api_util.DatabaseContext.get_conn()
g_db.execute(
"""UPDATE meshes
SET status = %s
WHERE project_uid = %s
AND name = %s
AND delete_date IS NULL""",
[status, project_codename, mesh_name])
def create_calc(user_id, project_codename, mesh_id, calc_name, overwrite=True):
g_db = core.api_util.DatabaseContext.get_conn()
with pg_util.Transaction(g_db):
project = projects.get_project(user_id, project_codename)
if not project:
raise RuntimeError("Unknown project " + project_codename)
if overwrite:
delete_calc(user_id, project_codename, calc_name)
calc_id = g_db.execute(
"INSERT INTO calculations (project_uid, mesh_id, name) VALUES (%s, %s, %s) RETURNING id",
[project_codename, mesh_id, calc_name]).fetchval()
return g_db.execute("SELECT * FROM calculations WHERE id = %s",
[calc_id]).fetchone()
def create_mesh(user_id, project_codename, mesh_name, overwrite=True):
g_db = core.api_util.DatabaseContext.get_conn()
with pg_util.Transaction(g_db):
project = projects.get_project(user_id, project_codename)
if not project:
raise RuntimeError("Unknown project " + project_codename)
if overwrite:
log.warning("Deleting mesh " + mesh_name + " from overwrite")
delete_mesh(user_id, project_codename, mesh_name)
mesh_id = g_db.execute(
"INSERT INTO meshes (project_uid, name) VALUES (%s, %s) RETURNING id",
[project_codename, mesh_name]).fetchval()
return g_db.execute("SELECT * FROM meshes WHERE id = %s",
[mesh_id]).fetchone()
def create_categories(id):
project_information = projects.get_project(id)
getid = projects.get_userid(id)
userid1 = getid[0]
userid2 = users.user_id()
if not userid1 == userid2:
return render_template(
"error.html",
message=
"Ei oikeutta muokata osa-alueita. Vain tapahtuman järjestäjällä on oikeus muokata kategorioita."
)
else:
return render_template("createcategories.html",
project_information=project_information)
def save_calc_param_file(user_id, project_codename, calc_name, param_file):
project = projects.get_project(user_id, project_codename)
if not project:
raise RuntimeError("Unknown project " + project_codename)
g_db = core.api_util.DatabaseContext.get_conn()
with projects.try_append_file_to_project(user_id, project_codename,
param_file) as project_file:
file_id = project_file['id']
g_db.execute(
"""UPDATE calculations
SET params_file_id = %s
WHERE project_uid = %s
AND name = %s
AND delete_date IS NULL""",
[file_id, project_codename, calc_name])
def delete_calc(user_id, project_codename, calc_name):
running_calc_status = [STATUS_PENDING, STATUS_RUNNING]
project = projects.get_project(user_id, project_codename)
if not project:
raise RuntimeError("Unknown project " + project_codename)
g_db = core.api_util.DatabaseContext.get_conn()
calculations = g_db.execute(
"""SELECT *
FROM calculations
WHERE project_uid = %s
AND name = %s
AND delete_date IS NULL""",
[project_codename, calc_name]).fetchall()
# Cancelling jobs
running_job_ids = []
for calc in calculations:
if calc['status'] in running_calc_status and calc['job_id'] is not None:
running_job_ids.append(calc['job_id'])
if running_job_ids:
log.info("Killing jobs " + repr(running_job_ids) +
" from deleting calc " + calc_name + "of project " +
project_codename)
jobs.cancel_jobs(running_job_ids)
for calc in calculations:
status_file_id = calc['status_file_id']
result_file_id = calc['result_file_id']
iterations_file_id = calc['iterations_file_id']
reduce_file_id = calc['reduce_file_id']
internal_file_id = calc['internal_file_id']
if status_file_id is not None:
projects.remove_file_from_project(user_id, project_codename,
status_file_id)
if result_file_id is not None:
projects.remove_file_from_project(user_id, project_codename,
result_file_id)
if iterations_file_id is not None:
projects.remove_file_from_project(user_id, project_codename,
iterations_file_id)
if reduce_file_id is not None:
projects.remove_file_from_project(user_id, project_codename,
reduce_file_id)
if internal_file_id is not None:
projects.remove_file_from_project(user_id, project_codename,
internal_file_id)
pg_util.delete_with_date(g_db, "calculations", calc['id'])
def edit_subprojects(id):
project_information = projects.get_project(id)
subproject_list = subprojects.list_subprojects(id)
getid = projects.get_userid(id)
userid1 = getid[0]
userid2 = users.user_id()
if not userid1 == userid2:
return render_template(
"error.html",
message=
"Ei oikeutta muokata osa-alueita. Vain tapahtuman järjestäjällä on oikeus muokata osa-alueita."
)
else:
return render_template("editsubprojects.html",
project_information=project_information,
subproject_list=subproject_list)
def get_calc_status_file(user_id, project_codename, calc_id):
project = projects.get_project(user_id, project_codename)
if not project:
raise RuntimeError("Unknown project " + project_codename)
g_db = core.api_util.DatabaseContext.get_conn()
result = g_db.execute(
"""SELECT status_file_id
FROM calculations
WHERE project_uid = %s
AND id = %s
AND delete_date IS NULL""",
[project_codename, calc_id]).fetchone()
if not result or result['status_file_id'] is None:
return None
return projects.get_file_by_id(user_id, project_codename,
int(result['status_file_id']))
def project(id):
project_information = projects.get_project(id)
creator_id = project_information[2]
user_information = users.user_name(creator_id)
subproject_list = subprojects.list_subprojects(id)
category_list = categories.list_categories(id)
other_payment_list = payments.list_other_payments(id)
user_payment_list = payments.list_user_payments(id)
grandtotal = subprojects.get_grandtotal(id)
headlines = ('Saaja', 'Summa', 'Viesti', 'Osa-alue', 'Pvm')
if user_payment_list == []:
headlines = []
if grandtotal is None:
grandtotal = 0
payment_grandtotal = payments.get_payment_grandtotal(id)
if payment_grandtotal is None:
payment_grandtotal = 0
user_payment_total = payments.user_payment_total(users.user_id(), id)
if user_payment_total is None:
user_payment_total = 0
count_payments = payments.count_payments(id)
if creator_id == users.user_id():
return render_template("project.html",
project_information=project_information,
user_information=user_information,
subproject_list=subproject_list,
category_list=category_list,
other_payment_list=other_payment_list,
user_payment_list=user_payment_list,
grandtotal=grandtotal,
payment_grandtotal=payment_grandtotal,
user_payment_total=user_payment_total,
count_payments=count_payments,
headlines=headlines)
else:
return render_template("projectguest.html",
project_information=project_information,
user_information=user_information,
subproject_list=subproject_list,
category_list=category_list,
other_payment_list=other_payment_list,
user_payment_list=user_payment_list,
grandtotal=grandtotal,
payment_grandtotal=payment_grandtotal,
user_payment_total=user_payment_total,
headlines=headlines)
def save_mesh_files(user_id,
project_codename,
mesh_name,
mesh_file_id,
preview_file_id=None):
project = projects.get_project(user_id, project_codename)
if not project:
raise RuntimeError("Unknown project " + project_codename)
g_db = core.api_util.DatabaseContext.get_conn()
g_db.execute(
"""UPDATE meshes
SET result_file_id = %s,
preview_file_id = %s
WHERE project_uid = %s
AND name = %s
AND delete_date IS NULL""",
[mesh_file_id, preview_file_id, project_codename, mesh_name])
def get_project_calculations(user_id,
project_codename,
include_deleted=False,
offset=0,
limit=None,
order=None):
project = projects.get_project(user_id, project_codename)
if not project:
raise RuntimeError("Unknown project " + project_codename)
query = """SELECT * FROM calculations WHERE project_uid = %s"""
if not include_deleted:
query += " AND delete_date IS NULL"
if order:
query += " " + order.to_sql()
if limit:
query += " LIMIT " + str(int(limit))
if offset:
query += " OFFSET " + str(int(offset))
g_db = core.api_util.DatabaseContext.get_conn()
return g_db.execute(query, [project_codename]).fetchall()
def import_project(project_id):
if not config.validation.is_valid_project_id(project_id):
raise ValueError('Invalid project id: %s' % project_id)
config_set = 'projects/%s' % project_id
project = projects.get_project(project_id)
if project is None:
raise NotFoundError('project %s not found' % project_id)
if project.HasField('gitiles_location'):
loc = _make_gitiles_location(project.gitiles_location)
else:
raise Error('project %s does not have a gitiles location' % project_id)
# Update project repo info.
repo_url = str(loc._replace(treeish=None, path=None))
projects.update_import_info(project_id, projects.RepositoryType.GITILES,
repo_url)
_import_config_set(config_set, loc, project_id)
def save_result_files(user_id,
project_codename,
calc_name,
result_file_id,
iterations_file_id=None,
reduce_file_id=None,
internal_file_id=None):
project = projects.get_project(user_id, project_codename)
if not project:
raise RuntimeError("Unknown project " + project_codename)
g_db = core.api_util.DatabaseContext.get_conn()
if internal_file_id:
g_db.execute(
"""UPDATE calculations
SET result_file_id = %s,
iterations_file_id = %s,
reduce_file_id = %s,
internal_file_id = %s
WHERE project_uid = %s
AND name = %s
AND delete_date IS NULL""", [
result_file_id, iterations_file_id, reduce_file_id,
internal_file_id, project_codename, calc_name
])
else:
g_db.execute(
"""UPDATE calculations
SET result_file_id = %s,
iterations_file_id = %s,
reduce_file_id = %s
WHERE project_uid = %s
AND name = %s
AND delete_date IS NULL""", [
result_file_id, iterations_file_id, reduce_file_id,
project_codename, calc_name
])
def get_project(self,pid):
return get_project(self.bc.BC_ACCOUNT_NUM,pid)
def delete_mesh(user_id, project_codename, mesh_name):
running_calc_status = [calc.STATUS_PENDING, calc.STATUS_RUNNING]
running_mesh_status = [STATUS_PENDING, STATUS_RUNNING]
project = projects.get_project(user_id, project_codename)
if not project:
raise RuntimeError("Unknown project " + project_codename)
g_db = core.api_util.DatabaseContext.get_conn()
proj_meshes = g_db.execute(
"""SELECT * FROM meshes WHERE project_uid = %s AND name = %s AND delete_date IS NULL""",
[project_codename, mesh_name]).fetchall()
# Cancelling computations
mesh_ids = [m['id'] for m in proj_meshes]
if not mesh_ids:
return
calculations = g_db.execute(
"""SELECT *
FROM calculations
WHERE project_uid = %s
AND mesh_id IN (""" +
", ".join(["%s"] * len(mesh_ids)) + """)
AND delete_date IS NULL""",
[project_codename] + mesh_ids).fetchall()
running_job_ids = [
m['job_id'] for m in proj_meshes if m['status'] in running_mesh_status
]
running_job_ids += [
c['job_id'] for c in calculations if c['status'] in running_calc_status
]
if running_job_ids:
log.info("killing jobs " + repr(running_job_ids) +
" from deleting mesh " + mesh_name + " of project " +
project_codename)
jobs.cancel_jobs(running_job_ids)
# Cleaning calculations
for proj_calc in calculations:
status_file_id = proj_calc['status_file_id']
result_file_id = proj_calc['result_file_id']
iterations_file_id = proj_calc['iterations_file_id']
reduce_file_id = proj_calc['reduce_file_id']
internal_file_id = proj_calc['internal_file_id']
if status_file_id is not None:
projects.remove_file_from_project(user_id, project_codename,
status_file_id)
if result_file_id is not None:
projects.remove_file_from_project(user_id, project_codename,
result_file_id)
if iterations_file_id is not None:
projects.remove_file_from_project(user_id, project_codename,
iterations_file_id)
if reduce_file_id is not None:
projects.remove_file_from_project(user_id, project_codename,
reduce_file_id)
if internal_file_id is not None:
projects.remove_file_from_project(user_id, project_codename,
internal_file_id)
pg_util.delete_with_date(g_db, "calculations", proj_calc['id'])
# Cleaning meshes
for proj_mesh in proj_meshes:
file_id = proj_mesh['result_file_id']
if file_id is not None:
projects.remove_file_from_project(user_id, project_codename,
file_id)
pg_util.delete_with_date(g_db, "meshes", proj_mesh['id'])