def follow_publication(name, url):
'''Track a new publication by adding it to the database.'''
db_session.begin()
new_publication = Publication(name=name, url=url)
db_session.add(new_publication)
db_session.commit()
return "Following publication {} at {} ".format(name, url)
def start():
request_data: dict = request_to_json(request)
conf: Artifact = setup_workdir(request_data)
db_session.begin()
db_session.flush()
step = db_session.query(SimulationStep).get(conf.step_id)
task = run_simulation.delay(conf.id)
step.celery_id = task.id
db_session.commit()
return jsonify(step)
def delete_stuff(q):
'''Generic removal function. Receives a query,
and return an error or success message according
to how the query went.'''
try:
db_session.begin()
db_session.delete(q.one())
db_session.commit()
return "Deleted."
except NoResultFound:
return "No such publication."
def setup_workdir(request_data: dict) -> Artifact:
"""
Creates new dir for simulation and places simulation configuration file in it
:param request_data: Flask request with configuration
:return: tuple with path to workdir and saved configuration
"""
conf_name = get_conf_name(request_data['configurationName'])
conf = request_data['configuration']
start_time = datetime.datetime.utcnow()
simulation = Simulation(started_utc=start_time,
name="test_simulation",
current_step="CLI",
status='ONGOING')
db_session.add(simulation)
db_session.flush()
step = SimulationStep(started_utc=start_time,
origin="CLI",
simulation_id=simulation.id,
status='ONGOING')
db_session.add(step)
db_session.flush()
workdir_path = create_workdir(simulation.id)
conf_path = '{}/{}'.format(workdir_path, conf_name)
simulation.workdir = workdir_path
simulation.current_step_id = step.id
with open(conf_path, 'w+') as f:
json.dump(conf, f, indent=2)
configuration = Artifact(size_kb=os.path.getsize(conf_path),
path=conf_path,
created_utc=start_time,
step_id=step.id,
name=conf_name,
file_type='JSON',
simulation_id=simulation.id)
simulation.artifacts.append(configuration)
step.artifacts.append(configuration)
simulation.steps.append(step)
db_session.begin()
db_session.add_all([configuration, step, simulation])
db_session.flush()
db_session.commit()
return configuration
def return_donation_commitee_id_from_name(name):
try:
committee = db_session.query(Committee)\
.filter(Committee.committee_name == name).one()
except Exception as e:
committee = Committee()
committee.committee_name = name
db_session.add(committee)
db_session.commit()
db_session.begin()
return committee.id
def return_filing_period_id_from_name(name):
try:
filing_period = db_session.query(PoliticalDonationFilingPeriod)\
.filter(PoliticalDonationFilingPeriod.period_name == name).one()
except Exception as e:
filing_period = PoliticalDonationFilingPeriod()
filing_period.period_name = name
db_session.add(filing_period)
db_session.commit()
db_session.begin()
return filing_period.id
def return_employer_name_id_from_name(name):
try:
employer_name = db_session.query(PoliticalDonationEmployerName)\
.filter(PoliticalDonationEmployerName.employer_name == name).one()
except Exception as e:
employer_name = PoliticalDonationEmployerName()
employer_name.employer_name = name
db_session.add(employer_name)
db_session.commit()
db_session.begin()
return employer_name.id
def return_contributor_type_id_from_name(name):
try:
contributor_type = db_session.query(ContributorType)\
.filter(ContributorType.type_name == name).one()
except Exception as e:
contributor_type = ContributorType()
contributor_type.type_name = name
db_session.add(contributor_type)
db_session.commit()
db_session.begin()
return contributor_type.id
def return_employer_occupation_id_from_name(name):
try:
occupation = db_session.query(PoliticalDonationEmployerOccupation)\
.filter(PoliticalDonationEmployerOccupation.occupation_name == name).one()
except Exception as e:
occupation = PoliticalDonationEmployerOccupation()
occupation.occupation_name = name
db_session.add(occupation)
db_session.commit()
db_session.begin()
return occupation.id
def return_race_id_from_name_and_district(name, district):
try:
race = db_session.query(Race)\
.filter(Race.race_name == name)\
.filter(Race.race_district == district).one()
except Exception as e:
race = Race()
race.race_name = name
race.race_district = district
db_session.add(office)
db_session.commit()
db_session.begin()
return office.id
def modify_word(id_w, proper, forbid_all, forbidden):
'''Modify a word in the database. Change its proper
status, the fact that its forbidden for all or some
publications.'''
# First, let's udpate the word table
db_session.query(Word).filter(Word.id == id_w).update({"proper": proper})
# Then, remove every forbidden properties for this word
for forb in db_session.query(Forbidden).filter(Forbidden.word_id == id_w):
db_session.delete(forbidden_all)
# Then if forbid_all : insert only word_id
db_session.begin()
if forbid_all:
newForbidden = Forbidden(word_id = id_w)
db_session.add(newForbidden)
else:
for fpub in forbidden:
newForbidden = Forbidden(word_id = id_w, publication_id = fpub)
db_session.add(newForbidden)
db_session.commit()
return "Updated."
def save_results_and_cleanup(plots, simulation_id: int, step_id: int,
workdir: str):
output_path: str = workdir + "/plots/"
plots: List[Artifact] = index_plots(output_path, simulation_id, step_id)
reports: List[Artifact] = index_reports(workdir + "/reports/",
simulation_id, step_id)
simulation: Simulation = db_session.query(Simulation).get(simulation_id)
step: SimulationStep = db_session.query(SimulationStep).get(step_id)
db_session.flush()
db_session.begin()
end_time = datetime.datetime.utcnow()
simulation.finished_utc = end_time
simulation.status = 'SUCCESS'
step.finished_utc = end_time
step.status = 'SUCCESS'
db_session.add_all(plots)
db_session.add_all(reports)
db_session.add_all([simulation, step])
db_session.commit()
cleanup(workdir)
return workdir
def reports_step(self, simulation_id: int) -> None:
start_time = datetime.datetime.utcnow()
simulation: Simulation = db_session.query(Simulation).get(simulation_id)
step: SimulationStep = SimulationStep(started_utc=start_time,
origin="REPORT",
simulation_id=simulation.id,
status='ONGOING')
db_session.add(step)
db_session.flush()
workdir: str = simulation.workdir
output_path: str = workdir + "/plots/"
if not os.path.exists(output_path):
os.makedirs(output_path)
simulation.current_step = "REPORT"
simulation.current_step_id = step.id
db_session.begin()
db_session.add_all([simulation, step])
db_session.commit()
result = chain(
all_clones_plot_stats.s(workdir),
all_clones_mullerplot.s(),
noise_plot_stats.s(),
major_clones_plot_stats.s(),
major_clones_mullerplot.s(),
mullerplot.s(),
mutation_histogram.s(),
simulation_report.s(),
build_cell_model.s(),
save_results_and_cleanup.s(simulation_id, step.id, workdir),
).apply_async()
return result
def mark_ongoing_as_failed():
"""
Marks all ongoing simulations and steps as failed. This method runs on app init
and any simulation that has status 'ONGOING' in db, is treated as failed
:return:
"""
db_session.begin()
db_session.query(Simulation).filter(Simulation.status == 'ONGOING').update(
{
Simulation.status: 'FAILURE',
Simulation.finished_utc: datetime.datetime.utcnow()
})
db_session.commit()
db_session.begin()
db_session.query(SimulationStep).filter(
SimulationStep.status == 'ONGOING').update({
SimulationStep.status:
'FAILURE',
SimulationStep.finished_utc:
datetime.datetime.utcnow()
})
db_session.commit()
return
from models import VotesmartCandidate
file_location = '/home/chris/projects/philly/git/political-id-match/id_matrix.csv'
#print(file_location)
line_num = 0
csvfile = open(file_location, 'r')
csvreader = csv.reader(csvfile, quotechar='"')
header_row = ['votesmart_candidate_id', 'crp_id', 'fec_id', 'nimsp_candidate_id', 'nimsp_entity_id', 'firstname', \
'nickname', 'middlename', 'lastname', 'suffix', 'office_state_id', 'election_state_id']
db_session.begin()
# Delete donations from this year
sql_query = "TRUNCATE votesmart_candidates"
results = db_session.execute(sql_query)
for row in csvreader:
line_num += 1
if len(row) != len(header_row):
print("ERROR: ", line_num, len(row), row)
continue
row_dict = dict(zip(header_row, row))
def cli_step(self, artifact_id: int) -> int:
"""
Celery task for generating CLI output from configuration.
Ideally, here I'd like to be able to pass Artifact object, so I don't need to get it from database
but due to the way that serializers in celery work, it is not possible. If passed, the Artifact object
would be serialized to list of its properties: ['id', 'created_utc', 'size_kb', 'path']. The proper
way to do it would be to set celery serializer to already used in flask AlchemyEncoder
see:
- https://stackoverflow.com/questions/21631878/celery-is-there-a-way-to-write-custom-json-encoder-decoder
:param artifact_id:
:param self:
:return: the id of created cli artifact
"""
conf: Artifact = db_session.query(Artifact).get(artifact_id)
step: SimulationStep = db_session.query(SimulationStep).get(conf.step_id)
simulation: Simulation = db_session.query(Simulation).get(
conf.simulation_id)
step.celery_id = self.request.id
step.status = 'ONGOING'
db_session.begin()
runtime_info: CliRuntimeInfo = CliRuntimeInfo(memory=0,
cpu=0,
step_id=step.id)
db_session.add(runtime_info)
db_session.commit()
executor: BaseExecutor = get_cli_executor()
executor.execute(conf)
while executor.is_finished is not True:
# Setting runtime_info like: runtime_info = executor.status seems to cause executor info not updating in
# SimulationStep, when querying @simulation_api.route('/step/<step_id>') endpoint
# TODO - find out whether step id is assigned incorrectly to configuration Artifact, or assigning another
# sqlalchemy object to runtime_info overrides some internal property that should not be overridden
# (ex. _sa_instance_state)
db_session.begin()
runtime_info.cpu = executor.status.cpu
runtime_info.memory = executor.status.memory
runtime_info.progress = executor.status.progress
db_session.commit()
sleep(POLLING_PERIOD)
db_session.begin()
result: Artifact = executor.result
log: Artifact = executor.log
if executor.status.error is not None:
step.status = 'FAILURE'
simulation.status = 'FAILURE'
simulation.finished_utc = datetime.datetime.utcnow()
step.finished_utc = datetime.datetime.utcnow()
else:
step.status = 'SUCCESS'
step.finished_utc = datetime.datetime.utcnow()
result.simulation_id = step.simulation_id
log.simulation_id = step.simulation_id
runtime_info.memory = 0
runtime_info.cpu = 0
runtime_info.progress = 100
db_session.add_all([result, log])
db_session.commit()
return result.id
def analyzer_step(self, artifact_id: int) -> int:
print('analyzer artifact id', artifact_id)
cli_out: Artifact = db_session.query(Artifact).get(artifact_id)
print('analyzer artifact id', cli_out.__dict__)
simulation: Simulation = db_session.query(Simulation).get(cli_out.simulation_id)
start_time = datetime.datetime.utcnow()
step: SimulationStep = SimulationStep(started_utc=start_time, origin="ANALYZER", simulation_id=simulation.id,
status='ONGOING')
db_session.add(step)
db_session.flush()
step.celery_id = self.request.id
simulation.current_step = "ANALYZER"
simulation.current_step_id = step.id
db_session.begin()
db_session.add_all([simulation, step])
db_session.commit()
db_session.begin()
runtime_info: AnalyzerRuntimeInfo = AnalyzerRuntimeInfo(
progress=0,
is_finished=False,
step_id=step.id
)
db_session.add(runtime_info)
db_session.commit()
executor: BaseExecutor = get_analyzer_executor()
print('Starting executor')
executor.execute(cli_out)
print('Starting polling..')
while executor.is_finished is not True:
db_session.begin()
print('status', executor.status.__dict__)
runtime_info.progress = executor.status.progress
runtime_info.error = executor.status.error
db_session.commit()
sleep(settings.SIMBAD_ANALYZER_POLLING_PERIOD)
db_session.begin()
result: List[Artifact] = list(map(lambda path: Artifact(path=path), executor.result))
for artifact in result:
artifact.step_id = step.id
artifact.name = path_leaf(artifact.path)
artifact.file_type = file_extension(artifact.path)
artifact.created_utc = datetime.datetime.fromtimestamp(os.path.getmtime(artifact.path))
artifact.simulation_id = step.simulation_id
artifact.size_kb = os.path.getsize(artifact.path)
if executor.status.error is not None:
print('Executor: ', executor.status.__dict__)
print('Result', result)
step.status = 'FAILURE'
simulation.status = 'FAILURE'
simulation.finished_utc = datetime.datetime.utcnow()
step.finished_utc = datetime.datetime.utcnow()
runtime_info.error = executor.status.error
db_session.add_all(result)
db_session.commit()
sleep(settings.SIMBAD_ANALYZER_POLLING_PERIOD)
raise RevokeChainRequested('Analyzer step failed')
step.finished_utc = datetime.datetime.utcnow()
step.status = 'SUCCESS'
runtime_info.progress = 100
db_session.add_all(result)
db_session.commit()
return simulation.id