예제 #1
0
def generate_feed(uid):
    subscriptions = Subscription.query(Subscription.uid == uid).fetch(200)
    subscription_urls = [sub.url for sub in subscriptions if sub.url]
    if len(subscription_urls) > 0:
        sources = Source.query(Source.url.IN(subscription_urls)).order(-Source.most_recent_article_added_date).fetch(len(subscription_urls))
        
        source_jsons = {}
        for source_json in memcache.get_multi([source.feed_cache_key() for source in sources]).itervalues():
            source_jsons[source_json['id']] = source_json
        
        to_fetch = [source for source in sources if source.key.id() not in source_jsons]
        print 'HITS {0} TO_FETCH {1}'.format(len(source_jsons), len(to_fetch))
        if len(to_fetch):
            source_promises = [src.json(include_articles=True, article_limit=FEED_ARTICLE_LIMIT, return_promise=True) for src in to_fetch]
            for promise in source_promises:
                data = promise()
                source_jsons[data['id']] = data
        
        # put the cache keys:
        if len(to_fetch):
            memcache.set_multi({source.feed_cache_key(): source_jsons[source.key.id()] for source in to_fetch if (source.key.id() in source_jsons)})
        
        source_json = [source_jsons[source.key.id()] for source in sources if source.key.id() in source_jsons]
    else:
        source_json = []
    return {
        "sources": source_json
    }
예제 #2
0
파일: cleaning.py 프로젝트: reactiv/thesis
def fix_sections_legislation(docpath, sourcename):
    source = session.query(Source).filter(Source.name == sourcename).first()
    if source is None:
        source = Source(name=sourcename, type='legislation')
        session.add(source)
        session.commit()

    # conjoin the subitems of clauses to their parent
    # create sections from the parents
    to_fix = session.query(Section).filter(Section.docpath.like(docpath))
    nc = None
    ns = None
    for s in to_fix:
        if s.name != 'blp':
            for c in s.clauses:
                if '-' not in c.name: # new section
                    if ns is not None:
                        ns.clauses.append(nc)
                        session.add(ns)
                        nc = None

                    ns = Section(name=c.content_html, docpath=s.docpath, source_id=source.id)
                elif c.name.count('-') == 1: # new clause
                    if nc is not None:
                        ns.clauses.append(nc)

                    text = c.content_html
                    nc = RawClause(header=c.header, content_html=text, cleaned=text)
                else:
                    # continuation clause
                    text = '\n#{}#'.format(c.header) + c.content_html
                    nc.content_html += text
                    nc.cleaned += text
    session.commit()
예제 #3
0
def featured_sources_by_category(category=None):
    q = Source.query(Source.featured_priority < 1)
    if category: q = q.filter(Source.categories == category)
    q = q.order(Source.featured_priority)
    sources = q.fetch(400)

    categories = util.unique_ordered_list(util.flatten(s.categories for s in sources))
    if category and category not in categories: categories.append(category)
    
    category_order = {category: i for i, category in enumerate(["Newspapers", "Culture", "Politics", "Tech", "Humor", "Local", "Longform"])}
    categories.sort(key=lambda x: category_order.get(x, 99999))

    sources_by_category = defaultdict(list)
    for source in sources:
        for category in source.categories:
            sources_by_category[category].append(source)

    max_items_per_category = 60 if category else 15
    for category, items in sources_by_category.items():
        sources_by_category[category] = items[:min(len(items), max_items_per_category)]

    category_jsons = []
    for category in categories:
        category_jsons.append({"id": category, "name": category, "sources": [s.json() for s in sources_by_category[category]]})

    return category_jsons
예제 #4
0
 def post(self):
     taskqueue.Queue('sources').purge()
     time.sleep(1) # TODO: anything but this; we need to wait 1 seconds, but how?
     
     for source in Source.query():
         source.most_recent_article_added_date = None
         source.enqueue_fetch(rand=True)
     self.response.write('done')
예제 #5
0
    def test_propagate_short(self):
        grid = Grid(8, 8)
        grid.add_source(Source(2, 1))
        grid.propagate(9)

        self.assertEqual(grid[0], [999.0, 999.0, 999.0, 999.0])
        self.assertEqual(grid[1], [1, 1, 1, 999.0])
        self.assertEqual(grid[2], [1, 0, 1, 999.0])
        self.assertEqual(grid[3], [1, 1, 1, 999.0])
예제 #6
0
 def test_propagate(self):
     grid = Grid(8, 8)
     grid.add_source(Source(2, 1))
     grid.propagate()
     print (grid)
     self.assertEqual(grid[0], [2, 2, 2, 2])
     self.assertEqual(grid[1], [1, 1, 1, 2])
     self.assertEqual(grid[2], [1, 0, 1, 2])
     self.assertEqual(grid[3], [1, 1, 1, 2])
예제 #7
0
 def post(self):
     url = canonical_url(self.request.get('url'))
     source_id = Source.id_for_source(url)
     source = ndb.Key(Source, source_id).get()
     while True:
         articles = Article.query(Article.source == source.key).order(-Article.added_date, Article.added_order).fetch(limit=100, keys_only=True)
         if len(articles) == 0: break
         ndb.delete_multi(articles)
     source.key.delete()
     self.response.write('Done')
예제 #8
0
def ensure_source(url, suppress_immediate_fetch=False):
    url = canonical_url(url)
    source_id = Source.id_for_source(url)
    source, inserted = get_or_insert(Source, source_id)
    if inserted:
        source.url = url
        source.put()
        source.enqueue_fetch()
    if inserted and not suppress_immediate_fetch:
        source.fetch_now()
    return source
예제 #9
0
파일: cleaning.py 프로젝트: reactiv/thesis
def generate_sources():
    book_sections = session.query(Section).filter(Section.docpath != "")
    names = set()
    for s in book_sections:
        name = get_section_name(s)
        if name != '' and name not in names:
            print(name)
            session.add(Source(name=name, type='fca_sourcebook'))
            names.add(name)

    session.commit()
예제 #10
0
def save_source(topic, date, windowsize, source):
    for ss in source:
        item = Source(topic, date, windowsize, ss)
        item_exist = db.session.query(Source).filter(Source.topic==topic ,\
                                                     Source.date==date ,\
                                                     Source.windowsize==windowsize ,\
                                                     Source.userid==ss).first()
        if item_exist:
            db.session.delete(item_exist)
        db.session.add(item)
        db.session.commit()
    print 'success save source'
예제 #11
0
파일: batch.py 프로젝트: agrover/BandRadar
def _foo():
    """test, do not use"""
    amazon_src = Source.byName("amazon")
    done_artists = Artist.select(Artist.q.recordings_updated != None)
    for artist in done_artists:
        for record in artist.recordings:
            if record.source == amazon_src:
                record.destroySelf()
        for recording in amazon.recordings(artist.name):
            Recording(
                name=recording["name"], by=artist, url=recording["url"], img_url=recording["img_url"], source=amazon_src
            )
예제 #12
0
파일: batch.py 프로젝트: agrover/BandRadar
def recording_artist_update(artist):
    # remove old entries
    for record in Recording.selectBy(by=artist):
        record.destroySelf()

    # add new entries (if any)
    for recording in cdbaby.recordings(artist.name):
        Recording(
            name=recording["name"],
            by=artist,
            url=recording["url"],
            img_url=recording["img_url"],
            source=Source.byName("cdbaby"),
        )
    for recording in amazon.recordings(artist.name):
        Recording(
            name=recording["name"],
            by=artist,
            url=recording["url"],
            img_url=recording["img_url"],
            source=Source.byName("amazon"),
        )
예제 #13
0
def create_bot():
    """Adds bot to DB."""

    name = request.form.get('name')
    desc = request.form.get('description')
    data_source = request.form.get('source')
    content_type = request.form.get('type')
    icon = request.form.get('icon')

    if content_type == "text_file":
        data_source = []
        file_list = request.files.getlist("text_file")
        for file in file_list:
            filename = secure_filename(file.filename)
            file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
            data_source.append("uploads/" + filename)

    content = process_source(content_type, data_source)

    if content == False:
        flash('error in bot creation! make sure your sources are correct?')
        return redirect("/")

    else:
        source = Source(content_type=content_type,
                        content_source=data_source,
                        content=content)

        db.session.add(source)
        db.session.commit()

        bot = Bot(bot_name=name,
                    creator_id = session['user_id'],
                    bot_description=desc,
                    bot_icon=icon,
                    source_id=source.source_id)

        db.session.add(bot)
        db.session.commit()

        flash('bot planted!')


        return redirect("/bot/" + str(bot.bot_id))
예제 #14
0
    def new_source(self, e):
        self.glwindow.sources.append(Source())  # 新建信号源实例
        self.glwindow.update()

        index = self.stackedSource.count()
        new = SourceControl(index, self.glwindow, self)
        self.stackedSource.addWidget(new)
        self.source_selector.addItem('wifi_' + str(self.source_count))
        self.source_count += 1

        self.source_selector.setCurrentIndex(index)
        self.stackedSource.setCurrentIndex(index)

        self.stackedSource.widget(index).source_delete.connect(
            self.on_source_delete)  # 绑定事件
        self.stackedSource.widget(index).source_modified.connect(
            self.on_modified)
        self.glwindow.active_source = index
        self.glwindow.active_building = -1
예제 #15
0
파일: seed.py 프로젝트: jtrieu2/bot.garden
def add_sources():

    print("Sources")

    # Deletes any existing data in the users table
    Source.query.delete()

    for line in open("seed_files/sources.txt"):
        line = line.rstrip()
        content_type, content_source = line.split("|")

        content = process_source(content_type, content_source)

        source = Source(content_type=content_type,
                        content_source=content_source,
                        content=content)

        db.session.add(source)

    db.session.commit()
예제 #16
0
def load_sources():
    """Load real sources into database."""

    print "Sources"
    Source.query.delete()

    src_dict = {
        'mtup': 'Meetup.com',
        'evtb': 'Eventbrite.com',
        'sfpk': 'SF Parks and Recreation'
    }

    for k, v in src_dict.items():
        src_id = k
        name = v

        source = Source(src_id=src_id, name=name)
        db.session.add(source)

    db.session.commit()
예제 #17
0
def main():
    filename = config("savefilename")
    lr = 0.0001

    this_config = dict(csv_file=config("csv_file"),
                       img_path=config("image_path"),
                       learning_rate=lr,
                       num_classes=4,
                       batchsize=64)

    wandb.init(project="prob_fix", name=filename, config=this_config)

    tr_loader, va_loader, te_loader, _ = get_train_val_test_loaders(
        task="default", batch_size=config("net.batch_size"))

    print('successfully loading!')

    model = Source()
    criterion = torch.nn.CrossEntropyLoss()

    optimizer = torch.optim.Adam(model.parameters(), lr=lr)
    print("Number of float-valued parameters:", count_parameters(model))

    model, start_epoch, stats = restore_checkpoint(model,
                                                   config("cnn.checkpoint"))

    axes = utils.make_training_plot()
    prolist = []

    evaluate_epoch(axes,
                   tr_loader,
                   va_loader,
                   te_loader,
                   model,
                   criterion,
                   start_epoch,
                   stats,
                   prolist,
                   multiclass=True)

    # initial val loss for early stopping
    prev_val_loss = stats[0][1]

    # TODO: define patience for early stopping
    patience = 5
    curr_patience = 0
    #

    # Loop over the entire dataset multiple times
    # for epoch in range(start_epoch, config('cnn.num_epochs')):
    epoch = start_epoch

    lowest_val_loss = 1
    train_auroc = 0
    test_auroc = 0
    lowest_round = epoch
    while curr_patience < patience:
        # Train model
        train_epoch(tr_loader, model, criterion, optimizer)

        # Evaluate model
        evaluate_epoch(axes,
                       tr_loader,
                       va_loader,
                       te_loader,
                       model,
                       criterion,
                       epoch + 1,
                       stats,
                       prolist,
                       multiclass=True)

        # Save model parameters
        save_checkpoint(model, epoch + 1, config("net.checkpoint"), stats)

        # update early stopping parameters
        curr_patience, prev_val_loss = early_stopping(stats, curr_patience,
                                                      prev_val_loss)

        epoch += 1
        if (prev_val_loss < lowest_val_loss):
            lowest_val_loss = prev_val_loss
            lowest_round = epoch

    pickle.dump(prolist, open("base_pro.pck", "wb"))
    print("Finished Training")
    # Save figure and keep plot open
    print("the lowest round: ", lowest_round)
    # utils.save_cnn_training_plot()
    # utils.save_cnn_other()
    utils.hold_training_plot()
예제 #18
0
    def import_to_db(self, event):
        flag_for_review = False
        venue_name = self.venue_name_fix(event['venue']['name'])
        try:
            v = Venue.byNameI(venue_name)
        except SQLObjectNotFound:
            v = Venue(name=venue_name, added_by=identity.current.user)
            flag_for_review = True
        if event['venue'].has_key('phone'):
            phone = event['venue']['phone']
            if not len(phone) >= 8:
                phone = "503-" + phone
            p = validators.PhoneNumber()
            try:
                event['venue']['phone'] = p.to_python(phone)
            except:
                event['venue']['phone'] = None
        self._set_optional_fields(v, event['venue'], ("address", "phone", "zip_code",
            "url", "description"))

        event_name = self.event_name_fix(event['name'])
        event_date = event["date"]
        event_time = event.get("time")
        if event_time:
            event_time = event_time.lower()
        # check same venue, date, time
        db_events = Event.selectBy(date=event_date,
            time=event_time, venue=v)
        if db_events.count():
            e = db_events[0]
            new_event = False
        else:
            # no time? still could be skippable, if event name is the same
            db_events = Event.selectBy(date=event_date,
                name=event_name, venue=v)
            if db_events.count():
                e = db_events[0]
                new_event = False
            else:
                e = Event(venue=v, name=event_name,
                    date=event_date, time=event_time,
                    added_by=identity.current.user)
                new_event = True
                try:
                    s = Source.byName(event["source"])
                except SQLObjectNotFound:
                    s = Source(name=event["source"])
                    flag_for_review = True
                e.addSource(s)
        self._set_optional_fields(e, event, ("cost", "ages", "url",
            "description", "ticket_url"))

        for artist in self.artists_clean(event['artists']):
            try:
                a = Artist.byNameI(artist)
                # artist was a similar, but now it's listed as having an event
                # so approve it
                if not a.approved:
                    a.approved = datetime.now()
            except SQLObjectNotFound:
                a = Artist(name=artist, added_by=identity.current.user)
                flag_for_review = True
            if not e.id in [existing.id for existing in a.events]:
                a.addEvent(e)

        # flag all events from certain unreliable sources
        if event["source"] in ("ticketswest", "upcoming", "lastfm"):
            flag_for_review = True

        if new_event and not flag_for_review:
            e.approved = datetime.now()
        elif flag_for_review:
            e.approved = None
        return (new_event, flag_for_review)
예제 #19
0
def fill_basics():
    with app.app_context():
        Police1 = Police(police_dept_id=1, name="User_Input")
        Police2 = Police(police_dept_id=2,
                         name="San Franciso Police Department",
                         city="San Francisco",
                         state="CA")
        Police3 = Police(police_dept_id=3,
                         name="Oakland Police Department",
                         city="Oakland",
                         state="CA")
        Police4 = Police(police_dept_id=4,
                         name="Alameda County Sheriff's Department",
                         city="Oakland",
                         state="CA")
        Police5 = Police(police_dept_id=5,
                         name="Santa Clara County Sheriff's Office",
                         city="San Jose",
                         state="CA")
        Police6 = Police(police_dept_id=6,
                         name="Fremont Police Department",
                         city="Fremont",
                         state="CA")
        Police7 = Police(police_dept_id=7,
                         name="San Leandro Police Department",
                         city="San Leandro",
                         state="CA")
        Police8 = Police(police_dept_id=8,
                         name="San Pablo Police Department",
                         city="San Pablo",
                         state="CA")
        Police9 = Police(police_dept_id=9,
                         name="New York Police Department",
                         city="New York",
                         state="NY")
        user = User(password="******",
                    username="******",
                    fname="Test123",
                    lname="Test123",
                    email="*****@*****.**",
                    description="Former Escort",
                    created_at=datetime.now(),
                    edited_at=datetime.now())
        db.session.add_all([
            Police1, Police2, Police3, Police4, Police5, Police6, Police7,
            Police8, Police9, user
        ])
        db.session.commit()
        source2 = Source(
            source_id=3,
            s_name="DataSF",
            police_dept_id=2,
            s_description="San Franciso Police API",
            url="https://data.sfgov.org/resource/cuks-n6tp.json?$limit=50000",
            s_type="gov api")
        source3 = Source(source_id=2,
                         s_name="DataSF",
                         police_dept_id=2,
                         s_description="San Franciso Police API",
                         url="https://data.sfgov.org/resource/PdId.json",
                         s_type="gov api")
        source = Source(source_id=1, s_name="User_report")
        source4 = Source(
            source_id=4,
            s_name="oaklandnet_90_days",
            police_dept_id=3,
            s_description="Oakland Police API Last 90 Days",
            url="ftp://crimewatchdata.oaklandnet.com/crimePublicData.csv",
            s_type="gov api")
        source5 = Source(
            source_id=5,
            s_name="socrata",
            police_dept_id=4,
            s_description="Alameda County Sheriff's Department API",
            url=
            "https://moto.data.socrata.com/resource/bvi2-5rde.json?$where=incident_description%20like%20%27%25PROST%25%27",
            s_type="gov api")
        source6 = Source(
            source_id=6,
            s_name="socrata",
            police_dept_id=5,
            s_description="Santa Clara County Sheriff's Office API",
            url=
            "https://moto.data.socrata.com/resource/wrmr-tdyp.json?$where=incident_description%20like%20%27%25PROST%25%27",
            s_type="gov api")
        source7 = Source(
            source_id=7,
            s_name="socrata",
            police_dept_id=6,
            s_description="FPD API",
            url=
            "https://moto.data.socrata.com/resource/nnzs-rxi5.json?$where=incident_description%20like%20%27%25PROST%25%27",
            s_type="gov api")
        source8 = Source(
            source_id=8,
            s_name="socrata",
            police_dept_id=7,
            s_description="SLPD API",
            url=
            "https://moto.data.socrata.com/resource/6nbc-apvm.json?$where=incident_description%20like%20%27%25PROST%25%27",
            s_type="gov api")
        source9 = Source(
            source_id=9,
            s_name="socrata",
            police_dept_id=8,
            s_description="SPPD API",
            url=
            "https://moto.data.socrata.com/resource/tsdt-igxn.json?$where=incident_description%20like%20%27%25PROST%25%27",
            s_type="gov api")
        source10 = Source(
            source_id=10,
            s_name="socrata",
            police_dept_id=9,
            s_description="NYPD Socrata API Recent Data",
            url=
            "https://data.cityofnewyork.us/resource/7x9x-zpz6.json?$where=pd_desc%20like%20%27%25PROST%25%27",
            s_type="gov api")
        source11 = Source(
            source_id=11,
            s_name="socrata",
            police_dept_id=9,
            s_description="NYPD Socrata API Historic Data",
            url=
            "https://data.cityofnewyork.us/resource/9s4h-37hy.json?$where=pd_desc%20like%20%27%25PROST%25%27&$limit=50000",
            s_type="gov api")
        db.session.add_all([
            source2, source3, source4, source5, source6, source7, source8,
            source9, source10, source11
        ])
        db.session.commit()
예제 #20
0
def parse_source(row, name=None, type_=None, url=None, provenance=None):
    """Parse a row into a Source object.
    
    Parameters
    ----------
    row : dict-like 
        Keys are column headings and values are the row values of a metadata 
        file.
    name : str, optional
        Name of data source.
    type_ : str, optional
        Type of data source.
    url : str, optional
        URL of data source.
    provenance : Provenance, optional
        Provenance object to be linked to the data source.
    
    Raises
    ------
    Exception : if there are no columns ending in 'study_id'.
    Exception : if no source name could be found for the given row and no name 
    argument was given.

    Returns
    -------
    Source
    """
    source = Source()
    # Initialize default values for attributes
    source_name = str(name) if name else None
    source_type = str(type_) if type_ else None
    source_url = str(url) if url else None

    # Find source_name in 'study_id' column
    for key in row.keys():
        match = re.match(r'(.*)study_id$', key)
        if match:
            if match.group(1):
                source_name = match.group(1).rstrip('_')
            break
    else:
        raise Exception('The metadata file must contain exactly one column '
                        'ending in "study_id".')
    # Replace source_name with 'source' column value if it exists
    new_source_name = row.get('source', '')
    if not re_missing.match(new_source_name):
        source_name = new_source_name
    # If source_name could not be initialized
    if not source_name:
        raise Exception('No source name could be found in the "study_id" '
                        'column heading, or as a valid value in the "source" '
                        'column of the metadata_file, and no source name '
                        'argument was provided.')

    # Replace source_type with 'source_type' column value if it exists
    new_source_type = row.get('source_type', '')
    if not re_missing.match(new_source_type):
        source_type = new_source_type
    if not source_type:
        raise Exception('No source type could be found in the "source_type" '
                        'column of the metadata_file, and no source type_ '
                        'argument was provided.')

    # Replace source_url with 'source_url' column value if it exists
    new_source_url = row.get('source_url', '')
    if not re_missing.match(new_source_url):
        source_url = new_source_url
    if not source_url:
        raise Exception('No source url could be found in the "source_url" '
                        'column of the metadata_file, and no source url'
                        'argument was provided.')

    # Assign to source attributes
    source.name = source_name
    source.type_ = source_type
    source.url = source_url
    if provenance:
        source.provenances.append(provenance)
    return source
예제 #21
0
파일: gl_window.py 프로젝트: Antonovvv/Y-Fi
                        self.buildings[active].move('forward')
                        self.building_z_changed.emit(self.buildings[active].position[2])
                    elif key == Qt.Key_Down:
                        self.buildings[active].move('back')
                        self.building_z_changed.emit(self.buildings[active].position[2])
                    elif key == Qt.Key_Left:
                        self.buildings[active].move('left')
                        self.building_x_changed.emit(self.buildings[active].position[0])
                    elif key == Qt.Key_Right:
                        self.buildings[active].move('right')
                        self.building_x_changed.emit(self.buildings[active].position[0])


if __name__ == '__main__':
    glutInit()
    displayMode = GLUT_DOUBLE | GLUT_ALPHA | GLUT_DEPTH
    glutInitDisplayMode(displayMode)

    app = QApplication(sys.argv)

    building = Building(floors=5, rooms=5, floor_thickness=1, room_length=12, room_width=6, room_height=5, wall_thickness=0.2)
    glwindow = GLWindow()
    source = Source()
    glwindow.sources.append(source)
    # source2 = Source(pos_x=10)
    # glwindow.sources.append(source2)

    glwindow.setGeometry(200, 200, 960, 680)
    glwindow.show()
    sys.exit(app.exec_())
예제 #22
0
class SampleParserTest(unittest.TestCase):
    sample_test_file = './data/test_data/samp_metadata/sample1.txt'

    row = OrderedDict([
        ('sample_name', '317.F10'), ('age', '22'), ('age_unit', 'years'),
        ('altitude', '0'), ('anatomical_body_site', 'FMA:Palm'),
        ('anonymized_name', 'F10'), ('body_habitat', 'UBERON:skin'),
        ('body_product', 'UBERON:sebum'),
        ('body_site', 'UBERON:zone of skin of hand'),
        ('collection_date', '11/12/2006'),
        ('country', 'GAZ:United States of America'), ('depth', '0'),
        ('description', 'human skin metagenome'), ('dna_extracted', 'true'),
        ('dominant_hand', ''), ('elevation', '1591.99'),
        ('env_biome', 'ENVO:human-associated habitat'),
        ('env_feature', 'ENVO:human-associated habitat'),
        ('host_common_name', 'human'), ('host_subject_id', 'F1'),
        ('host_taxid', '9606'), ('latitude', '40'), ('longitude', '-105'),
        ('palm_size', ''), ('physical_specimen_remaining', 'false'),
        ('public', 'true'), ('qiita_study_id', '317'),
        ('sample_type', 'XXQIITAXX'), ('sex', 'female'),
        ('time_since_last_wash', '0'),
        ('title',
         'The influence of sex handedness and washing on the diversity of hand surface bacteriaS1_V160'
         )
    ])
    dayfirst_dict = {'collection_date': False}

    # TODO Update details of source (when necessary)
    source1 = Source(name='qiita',
                     type_='Database (Public)',
                     url='https://qiita.ucsd.edu/study/description/0')
    experiment1 = Experiment(source=source1, orig_study_id='317')
    subject1 = Subject(
        source=source1,
        orig_study_id='317',
        orig_subject_id='F1',
        sex='female',
        country='United States of America',
        race=None,
        csection=None,
        disease=None,
        dob=None,
    )
    subject2 = Subject(
        source=source1,
        orig_study_id='317',
        orig_subject_id='F2',
        sex='female',
        country='United States of America',
        race=None,
        csection=None,
        disease=None,
        dob=None,
    )
    sampling_site = SamplingSite(
        uberon_habitat_term='UBERON:skin',
        uberon_product_term='UBERON:sebum',
        uberon_site_term='UBERON:zone of skin of hand',
        env_biom_term='ENVO:human-associated habitat',
        env_feature_term='ENVO:human-associated habitat')
    sampling_time = Time(timestamp=datetime.datetime(2006, 11, 12),
                         uncertainty=None,
                         date=datetime.date(2006, 11, 12),
                         time=None,
                         year=2006,
                         month=11,
                         day=12,
                         hour=None,
                         minute=None,
                         second=None,
                         season='autumn')
    sample1 = Sample(source=source1,
                     orig_study_id='317',
                     orig_subject_id='F1',
                     orig_sample_id='317.F10',
                     age_units=ureg.years,
                     age=22.0,
                     latitude=40.0,
                     longitude=-105.0,
                     elevation=1591.99,
                     height_units=ureg.metres,
                     height=None,
                     weight_units=ureg.kilograms,
                     weight=None,
                     bmi=None,
                     sample_date=datetime.date(2006, 11, 12),
                     sample_time=None,
                     sampling_site=sampling_site,
                     sampling_time=sampling_time)
    sample2 = Sample(source=source1,
                     orig_study_id='317',
                     orig_subject_id='F1',
                     orig_sample_id='317.F12',
                     age_units=ureg.years,
                     age=22.0,
                     latitude=40.0,
                     longitude=-105.0,
                     elevation=1591.99,
                     height_units=ureg.metres,
                     height=None,
                     weight_units=ureg.kilograms,
                     weight=None,
                     bmi=None,
                     sample_date=datetime.date(2006, 11, 12),
                     sample_time=None,
                     sampling_site=sampling_site,
                     sampling_time=sampling_time)
    sample3 = Sample(source=source1,
                     orig_study_id='317',
                     orig_subject_id='F2',
                     orig_sample_id='317.F20',
                     age_units=ureg.years,
                     age=None,
                     latitude=40.0,
                     longitude=-105.0,
                     elevation=1591.99,
                     height_units=ureg.metres,
                     height=None,
                     weight_units=ureg.kilograms,
                     weight=None,
                     bmi=None,
                     sample_date=datetime.date(2006, 11, 12),
                     sample_time=None,
                     sampling_site=sampling_site,
                     sampling_time=sampling_time)
    # Not necessary to establish these relationships for purpose of
    # test_parse_objects:
    sample1._subject = subject1
    sample2._subject = subject1
    sample3._subject = subject2
    subject1._samples = {sample1, sample2}
    subject2._samples = {sample3}
    experiment1._subjects = {subject1, subject2}
    experiment1._samples = {sample1, sample2, sample3}

    def test_parse_objects(self):
        experiment_ids = parse_objects(self.sample_test_file)
        self.assertIn('317', experiment_ids)
        experiment = experiment_ids['317']
        self.assertEqual(self.experiment1, experiment)
        self.assertIn(self.subject1, experiment.subjects)
        self.assertIn(self.subject2, experiment.subjects)
        self.assertIn(self.sample1, experiment.samples)
        self.assertIn(self.sample2, experiment.samples)
        self.assertIn(self.sample3, experiment.samples)

    # TODO: We will have to test without the source keyword at some point.
    def test_parse_sample(self):
        self.maxDiff = None
        blacklist_attrs = [
            '_sa_instance_state', 'source', 'counts', '_experiments',
            '_subject', '_preparations'
        ]
        sample = parse_sample(self.row,
                              self.dayfirst_dict,
                              source=self.source1)
        sample_attrs = set((key, value)
                           for key, value in sample.__dict__.items()
                           if key not in blacklist_attrs)
        expected_attrs = set((key, value)
                             for key, value in self.sample1.__dict__.items()
                             if key not in blacklist_attrs)
        self.assertEqual(sample_attrs, expected_attrs)
        self.assertEqual(sample.source, self.source1)
        self.assertEqual(sample.counts, self.sample1.counts)
        # When sample is parsed, it is not yet associated with subject/experiments
        self.assertEqual(sample._subject, None)
        self.assertEqual(sample._experiments, set())
        self.assertEqual(sample._preparations, set())

    def test_parse_subject(self):
        self.maxDiff = None
        blacklist_attrs = [
            '_sa_instance_state', 'source', 'counts', 'perturbation_facts',
            '_experiments', '_samples', '_perturbations'
        ]
        subject = parse_subject(self.row, source=self.source1)
        subject_attrs = set((key, value)
                            for key, value in subject.__dict__.items()
                            if key not in blacklist_attrs)
        expected_attrs = set((key, value)
                             for key, value in self.subject1.__dict__.items()
                             if key not in blacklist_attrs)
        self.assertEqual(subject_attrs, expected_attrs)
        self.assertEqual(subject.source, self.source1)
        self.assertEqual(subject.counts, self.subject1.counts)
        self.assertEqual(subject.perturbation_facts,
                         self.subject1.perturbation_facts)
        # When subject is parsed, it is not yet associated with samples/experiments
        self.assertEqual(subject._experiments, set())
        self.assertEqual(subject._samples, set())
        self.assertEqual(subject._perturbations, set())

    def test_parse_processing(self):
        self.maxDiff = None
        processing1 = Processing(parent=None,
                                 parameter_values='{}',
                                 orig_prep_id='577',
                                 orig_proc_id='2593')
        processing2 = Processing(parent=processing1,
                                 parameter_values='{'
                                 '"barcode_type":"golay_12",'
                                 '"command":"Split libraries (QIIMEq2 1.9.1)",'
                                 '"disable_bc_correction":"False",'
                                 '"disable_primers":"False",'
                                 '"generated on":"2016-01-14 17:01",'
                                 '"input_data":"2593",'
                                 '"max_ambig":"6",'
                                 '"max_barcode_errors":"1.5",'
                                 '"max_homopolymer":"6",'
                                 '"max_primer_mismatch":"0",'
                                 '"max_seq_len":"1000",'
                                 '"min_qual_score":"25",'
                                 '"min_seq_len":"200",'
                                 '"qual_score_window":"0",'
                                 '"reverse_primer_mismatches":"0",'
                                 '"reverse_primers":"disable",'
                                 '"trim_seq_length":"False",'
                                 '"truncate_ambi_bases":"False"'
                                 '}',
                                 orig_prep_id='577',
                                 orig_proc_id='310')
        processing3 = Processing(
            parent=processing2,
            parameter_values='{'
            '"command":"Pick closed-reference OTUs (QIIMEq2 1.9.1)",'
            '"generated on":"2015-06-30 14:06",'
            '"input_data":"310",'
            '"reference-seq":"/databases/gg/13_8/rep_set/97_otus.fasta",'
            '"reference-tax":"/databases/gg/13_8/taxonomy/97_otu_taxonomy.txt",'
            '"similarity":"0.97",'
            '"sortmerna_coverage":"0.97",'
            '"sortmerna_e_value":"1",'
            '"sortmerna_max_pos":"10000",'
            '"threads":"1"'
            '}',
            orig_prep_id='577',
            orig_proc_id='2594')
        expected_processings = {
            '2593': processing1,
            '310': processing2,
            '2594': processing3
        }
        processings = parse_processings('./data/test_data/proc1.json')
        # TODO: Implement workflows and parents as mocks?
        blacklist_attrs = ['_sa_instance_state', 'workflows', 'parent']
        for proc_id, processing in processings.items():
            self.assertIn(proc_id, expected_processings)
            processing_attrs = set(
                (key, value) for key, value in processing.__dict__.items()
                if key not in blacklist_attrs)
            expected_attrs = set(
                (key, value) for key, value in
                expected_processings[proc_id].__dict__.items()
                if key not in blacklist_attrs)
            self.assertEqual(processing_attrs, expected_attrs)
예제 #23
0
파일: liUtility.py 프로젝트: chenm001/leap
 def __findBuildPath(path):
     return Source.findBuildPath(path, modulePath)
예제 #24
0
파일: liUtility.py 프로젝트: chenm001/leap
 def __findBuildPath(path):
     return Source.findBuildPath(path, modulePath)
예제 #25
0
    _ = gcam.forward(xi)
    gcam.backward(ids=torch.tensor([[target_class]]).to(device))
    regions = gcam.generate(target_layer=target_layer)
    activation = regions.detach()
    save_gradcam(
        np.squeeze(activation),
        utils.denormalize_image(np.squeeze(xi.numpy()).transpose(1, 2, 0)),
        axarr,
        i,
    )


if __name__ == "__main__":
    # Attempts to restore from checkpoint
    print("Loading cnn...")
    model = Source()
    model, start_epoch, _ = restore_checkpoint(model,
                                               config("net.checkpoint"),
                                               force=True)

    tr_loader, va_loader, te_loader, _ = get_train_val_test_loaders(
        task="target",
        batch_size=config("net.batch_size"),
    )
    for i in range(40):
        plt.clf()
        f, axarr = plt.subplots(1, 2)
        visualize_input(i, axarr)
        visualize_layer1_activations(i, axarr)
        plt.close()
예제 #26
0
 def get(self):
     vars = {
         "featured_sources": Source.query().filter(Source.featured_priority >= 0).order(-Source.featured_priority).fetch()
     }
     self.response.write(template('sources.html', vars))