Example #1
0
def follow_publication(name, url):
    '''Track a new publication by adding it to the database.'''
    db_session.begin()
    new_publication = Publication(name=name, url=url)
    db_session.add(new_publication)
    db_session.commit()
    return "Following publication {} at {} ".format(name, url)
Example #2
0
def start():
    request_data: dict = request_to_json(request)
    conf: Artifact = setup_workdir(request_data)
    db_session.begin()
    db_session.flush()
    step = db_session.query(SimulationStep).get(conf.step_id)
    task = run_simulation.delay(conf.id)
    step.celery_id = task.id
    db_session.commit()
    return jsonify(step)
Example #3
0
def delete_stuff(q):
    '''Generic removal function. Receives a  query,
    and return an error or success message according
    to how the query went.'''
    try:
        db_session.begin()
        db_session.delete(q.one())
        db_session.commit()
        return "Deleted."
    except NoResultFound:
        return "No such publication."
Example #4
0
def setup_workdir(request_data: dict) -> Artifact:
    """
    Creates new dir for simulation and places simulation configuration file in it
    :param request_data: Flask request with configuration
    :return: tuple with path to workdir and saved configuration
    """
    conf_name = get_conf_name(request_data['configurationName'])
    conf = request_data['configuration']

    start_time = datetime.datetime.utcnow()

    simulation = Simulation(started_utc=start_time,
                            name="test_simulation",
                            current_step="CLI",
                            status='ONGOING')
    db_session.add(simulation)
    db_session.flush()
    step = SimulationStep(started_utc=start_time,
                          origin="CLI",
                          simulation_id=simulation.id,
                          status='ONGOING')
    db_session.add(step)
    db_session.flush()

    workdir_path = create_workdir(simulation.id)
    conf_path = '{}/{}'.format(workdir_path, conf_name)

    simulation.workdir = workdir_path
    simulation.current_step_id = step.id

    with open(conf_path, 'w+') as f:
        json.dump(conf, f, indent=2)

    configuration = Artifact(size_kb=os.path.getsize(conf_path),
                             path=conf_path,
                             created_utc=start_time,
                             step_id=step.id,
                             name=conf_name,
                             file_type='JSON',
                             simulation_id=simulation.id)
    simulation.artifacts.append(configuration)
    step.artifacts.append(configuration)
    simulation.steps.append(step)

    db_session.begin()
    db_session.add_all([configuration, step, simulation])
    db_session.flush()
    db_session.commit()

    return configuration
Example #5
0
def return_donation_commitee_id_from_name(name):

    try:

        committee = db_session.query(Committee)\
            .filter(Committee.committee_name == name).one()

    except Exception as e:

        committee = Committee()
        committee.committee_name = name

        db_session.add(committee)        
        db_session.commit()
        db_session.begin()

    return committee.id
Example #6
0
def return_filing_period_id_from_name(name):

    try:

        filing_period = db_session.query(PoliticalDonationFilingPeriod)\
            .filter(PoliticalDonationFilingPeriod.period_name == name).one()

    except Exception as e:

        filing_period = PoliticalDonationFilingPeriod()
        filing_period.period_name = name

        db_session.add(filing_period)        
        db_session.commit()
        db_session.begin()

    return filing_period.id
Example #7
0
def return_employer_name_id_from_name(name):

    try:

        employer_name = db_session.query(PoliticalDonationEmployerName)\
            .filter(PoliticalDonationEmployerName.employer_name == name).one()

    except Exception as e:

        employer_name = PoliticalDonationEmployerName()
        employer_name.employer_name = name

        db_session.add(employer_name)        
        db_session.commit()
        db_session.begin()

    return employer_name.id
Example #8
0
def return_contributor_type_id_from_name(name):

    try:

        contributor_type = db_session.query(ContributorType)\
            .filter(ContributorType.type_name == name).one()

    except Exception as e:

        contributor_type = ContributorType()
        contributor_type.type_name = name

        db_session.add(contributor_type)        
        db_session.commit()
        db_session.begin()

    return contributor_type.id
Example #9
0
def return_employer_occupation_id_from_name(name):

    try:

        occupation = db_session.query(PoliticalDonationEmployerOccupation)\
            .filter(PoliticalDonationEmployerOccupation.occupation_name == name).one()

    except Exception as e:

        occupation = PoliticalDonationEmployerOccupation()
        occupation.occupation_name = name

        db_session.add(occupation)        
        db_session.commit()
        db_session.begin()

    return occupation.id
Example #10
0
def return_race_id_from_name_and_district(name, district):

    try:

        race = db_session.query(Race)\
            .filter(Race.race_name == name)\
            .filter(Race.race_district == district).one()

    except Exception as e:

        race = Race()
        race.race_name = name
        race.race_district = district

        db_session.add(office)        
        db_session.commit()
        db_session.begin()

    return office.id
Example #11
0
def modify_word(id_w, proper, forbid_all, forbidden):
    '''Modify a word in the database. Change its proper
    status, the fact that its forbidden for all or some
    publications.'''
    # First, let's udpate the word table
    db_session.query(Word).filter(Word.id == id_w).update({"proper": proper})
    # Then, remove every forbidden properties for this word
    for forb in db_session.query(Forbidden).filter(Forbidden.word_id == id_w):
        db_session.delete(forbidden_all)
    # Then if forbid_all : insert only word_id
    db_session.begin()
    if forbid_all:
        newForbidden = Forbidden(word_id = id_w)
        db_session.add(newForbidden)
    else:
        for fpub in forbidden:
            newForbidden = Forbidden(word_id = id_w, publication_id = fpub)
            db_session.add(newForbidden)
    db_session.commit()
    return "Updated."
Example #12
0
def save_results_and_cleanup(plots, simulation_id: int, step_id: int,
                             workdir: str):
    output_path: str = workdir + "/plots/"
    plots: List[Artifact] = index_plots(output_path, simulation_id, step_id)
    reports: List[Artifact] = index_reports(workdir + "/reports/",
                                            simulation_id, step_id)

    simulation: Simulation = db_session.query(Simulation).get(simulation_id)
    step: SimulationStep = db_session.query(SimulationStep).get(step_id)
    db_session.flush()

    db_session.begin()
    end_time = datetime.datetime.utcnow()
    simulation.finished_utc = end_time
    simulation.status = 'SUCCESS'
    step.finished_utc = end_time
    step.status = 'SUCCESS'
    db_session.add_all(plots)
    db_session.add_all(reports)
    db_session.add_all([simulation, step])
    db_session.commit()
    cleanup(workdir)
    return workdir
Example #13
0
def reports_step(self, simulation_id: int) -> None:
    start_time = datetime.datetime.utcnow()

    simulation: Simulation = db_session.query(Simulation).get(simulation_id)
    step: SimulationStep = SimulationStep(started_utc=start_time,
                                          origin="REPORT",
                                          simulation_id=simulation.id,
                                          status='ONGOING')
    db_session.add(step)
    db_session.flush()

    workdir: str = simulation.workdir
    output_path: str = workdir + "/plots/"

    if not os.path.exists(output_path):
        os.makedirs(output_path)

    simulation.current_step = "REPORT"
    simulation.current_step_id = step.id

    db_session.begin()
    db_session.add_all([simulation, step])
    db_session.commit()

    result = chain(
        all_clones_plot_stats.s(workdir),
        all_clones_mullerplot.s(),
        noise_plot_stats.s(),
        major_clones_plot_stats.s(),
        major_clones_mullerplot.s(),
        mullerplot.s(),
        mutation_histogram.s(),
        simulation_report.s(),
        build_cell_model.s(),
        save_results_and_cleanup.s(simulation_id, step.id, workdir),
    ).apply_async()
    return result
Example #14
0
def mark_ongoing_as_failed():
    """
    Marks all ongoing simulations and steps as failed. This method runs on app init
    and any simulation that has status 'ONGOING' in db, is treated as failed
    :return:
    """
    db_session.begin()
    db_session.query(Simulation).filter(Simulation.status == 'ONGOING').update(
        {
            Simulation.status: 'FAILURE',
            Simulation.finished_utc: datetime.datetime.utcnow()
        })
    db_session.commit()

    db_session.begin()
    db_session.query(SimulationStep).filter(
        SimulationStep.status == 'ONGOING').update({
            SimulationStep.status:
            'FAILURE',
            SimulationStep.finished_utc:
            datetime.datetime.utcnow()
        })
    db_session.commit()
    return
Example #15
0
from models import VotesmartCandidate

file_location = '/home/chris/projects/philly/git/political-id-match/id_matrix.csv'

#print(file_location)

line_num = 0

csvfile = open(file_location, 'r')

csvreader = csv.reader(csvfile, quotechar='"')

header_row = ['votesmart_candidate_id', 'crp_id', 'fec_id', 'nimsp_candidate_id', 'nimsp_entity_id', 'firstname', \
    'nickname', 'middlename', 'lastname', 'suffix', 'office_state_id', 'election_state_id']

db_session.begin()

# Delete donations from this year
sql_query = "TRUNCATE votesmart_candidates"

results = db_session.execute(sql_query)

for row in csvreader:

    line_num += 1

    if len(row) != len(header_row):
        print("ERROR: ", line_num, len(row), row)
        continue

    row_dict = dict(zip(header_row, row))
Example #16
0
def cli_step(self, artifact_id: int) -> int:
    """
    Celery task for generating CLI output from configuration.
    Ideally, here I'd like to be able to pass Artifact object, so I don't need to get it from database
    but due to the way that serializers in celery work, it is not possible. If passed, the Artifact object
    would be serialized to list of its properties: ['id', 'created_utc', 'size_kb', 'path']. The proper
    way to do it would be to set celery serializer to already used in flask AlchemyEncoder
    see:
        - https://stackoverflow.com/questions/21631878/celery-is-there-a-way-to-write-custom-json-encoder-decoder

    :param artifact_id:
    :param self:
    :return: the id of created cli artifact
    """
    conf: Artifact = db_session.query(Artifact).get(artifact_id)
    step: SimulationStep = db_session.query(SimulationStep).get(conf.step_id)
    simulation: Simulation = db_session.query(Simulation).get(
        conf.simulation_id)
    step.celery_id = self.request.id
    step.status = 'ONGOING'
    db_session.begin()
    runtime_info: CliRuntimeInfo = CliRuntimeInfo(memory=0,
                                                  cpu=0,
                                                  step_id=step.id)
    db_session.add(runtime_info)
    db_session.commit()

    executor: BaseExecutor = get_cli_executor()
    executor.execute(conf)

    while executor.is_finished is not True:
        # Setting runtime_info like: runtime_info = executor.status seems to cause executor info not updating in
        # SimulationStep, when querying @simulation_api.route('/step/<step_id>') endpoint
        # TODO - find out whether step id is assigned incorrectly to configuration Artifact, or assigning another
        #  sqlalchemy object to runtime_info overrides some internal property that should not be overridden
        #  (ex. _sa_instance_state)
        db_session.begin()
        runtime_info.cpu = executor.status.cpu
        runtime_info.memory = executor.status.memory
        runtime_info.progress = executor.status.progress
        db_session.commit()
        sleep(POLLING_PERIOD)

    db_session.begin()
    result: Artifact = executor.result
    log: Artifact = executor.log
    if executor.status.error is not None:
        step.status = 'FAILURE'
        simulation.status = 'FAILURE'
        simulation.finished_utc = datetime.datetime.utcnow()
        step.finished_utc = datetime.datetime.utcnow()
    else:
        step.status = 'SUCCESS'
        step.finished_utc = datetime.datetime.utcnow()

    result.simulation_id = step.simulation_id
    log.simulation_id = step.simulation_id
    runtime_info.memory = 0
    runtime_info.cpu = 0
    runtime_info.progress = 100
    db_session.add_all([result, log])
    db_session.commit()

    return result.id
Example #17
0
def analyzer_step(self, artifact_id: int) -> int:
    print('analyzer artifact id', artifact_id)
    cli_out: Artifact = db_session.query(Artifact).get(artifact_id)
    print('analyzer artifact id', cli_out.__dict__)

    simulation: Simulation = db_session.query(Simulation).get(cli_out.simulation_id)
    start_time = datetime.datetime.utcnow()
    step: SimulationStep = SimulationStep(started_utc=start_time, origin="ANALYZER", simulation_id=simulation.id,
                                          status='ONGOING')
    db_session.add(step)
    db_session.flush()
    step.celery_id = self.request.id
    simulation.current_step = "ANALYZER"
    simulation.current_step_id = step.id

    db_session.begin()
    db_session.add_all([simulation, step])
    db_session.commit()

    db_session.begin()
    runtime_info: AnalyzerRuntimeInfo = AnalyzerRuntimeInfo(
        progress=0,
        is_finished=False,
        step_id=step.id
    )

    db_session.add(runtime_info)
    db_session.commit()

    executor: BaseExecutor = get_analyzer_executor()
    print('Starting executor')
    executor.execute(cli_out)
    print('Starting polling..')

    while executor.is_finished is not True:
        db_session.begin()
        print('status', executor.status.__dict__)
        runtime_info.progress = executor.status.progress
        runtime_info.error = executor.status.error
        db_session.commit()
        sleep(settings.SIMBAD_ANALYZER_POLLING_PERIOD)

    db_session.begin()

    result: List[Artifact] = list(map(lambda path: Artifact(path=path), executor.result))

    for artifact in result:
        artifact.step_id = step.id
        artifact.name = path_leaf(artifact.path)
        artifact.file_type = file_extension(artifact.path)
        artifact.created_utc = datetime.datetime.fromtimestamp(os.path.getmtime(artifact.path))
        artifact.simulation_id = step.simulation_id
        artifact.size_kb = os.path.getsize(artifact.path)

    if executor.status.error is not None:
        print('Executor: ', executor.status.__dict__)
        print('Result', result)
        step.status = 'FAILURE'
        simulation.status = 'FAILURE'
        simulation.finished_utc = datetime.datetime.utcnow()
        step.finished_utc = datetime.datetime.utcnow()
        runtime_info.error = executor.status.error
        db_session.add_all(result)
        db_session.commit()
        sleep(settings.SIMBAD_ANALYZER_POLLING_PERIOD)
        raise RevokeChainRequested('Analyzer step failed')

    step.finished_utc = datetime.datetime.utcnow()
    step.status = 'SUCCESS'
    runtime_info.progress = 100
    db_session.add_all(result)
    db_session.commit()

    return simulation.id