def generate_feed(uid):
subscriptions = Subscription.query(Subscription.uid == uid).fetch(200)
subscription_urls = [sub.url for sub in subscriptions if sub.url]
if len(subscription_urls) > 0:
sources = Source.query(Source.url.IN(subscription_urls)).order(-Source.most_recent_article_added_date).fetch(len(subscription_urls))
source_jsons = {}
for source_json in memcache.get_multi([source.feed_cache_key() for source in sources]).itervalues():
source_jsons[source_json['id']] = source_json
to_fetch = [source for source in sources if source.key.id() not in source_jsons]
print 'HITS {0} TO_FETCH {1}'.format(len(source_jsons), len(to_fetch))
if len(to_fetch):
source_promises = [src.json(include_articles=True, article_limit=FEED_ARTICLE_LIMIT, return_promise=True) for src in to_fetch]
for promise in source_promises:
data = promise()
source_jsons[data['id']] = data
# put the cache keys:
if len(to_fetch):
memcache.set_multi({source.feed_cache_key(): source_jsons[source.key.id()] for source in to_fetch if (source.key.id() in source_jsons)})
source_json = [source_jsons[source.key.id()] for source in sources if source.key.id() in source_jsons]
else:
source_json = []
return {
"sources": source_json
}
def fix_sections_legislation(docpath, sourcename):
source = session.query(Source).filter(Source.name == sourcename).first()
if source is None:
source = Source(name=sourcename, type='legislation')
session.add(source)
session.commit()
# conjoin the subitems of clauses to their parent
# create sections from the parents
to_fix = session.query(Section).filter(Section.docpath.like(docpath))
nc = None
ns = None
for s in to_fix:
if s.name != 'blp':
for c in s.clauses:
if '-' not in c.name: # new section
if ns is not None:
ns.clauses.append(nc)
session.add(ns)
nc = None
ns = Section(name=c.content_html, docpath=s.docpath, source_id=source.id)
elif c.name.count('-') == 1: # new clause
if nc is not None:
ns.clauses.append(nc)
text = c.content_html
nc = RawClause(header=c.header, content_html=text, cleaned=text)
else:
# continuation clause
text = '\n#{}#'.format(c.header) + c.content_html
nc.content_html += text
nc.cleaned += text
session.commit()
def featured_sources_by_category(category=None):
q = Source.query(Source.featured_priority < 1)
if category: q = q.filter(Source.categories == category)
q = q.order(Source.featured_priority)
sources = q.fetch(400)
categories = util.unique_ordered_list(util.flatten(s.categories for s in sources))
if category and category not in categories: categories.append(category)
category_order = {category: i for i, category in enumerate(["Newspapers", "Culture", "Politics", "Tech", "Humor", "Local", "Longform"])}
categories.sort(key=lambda x: category_order.get(x, 99999))
sources_by_category = defaultdict(list)
for source in sources:
for category in source.categories:
sources_by_category[category].append(source)
max_items_per_category = 60 if category else 15
for category, items in sources_by_category.items():
sources_by_category[category] = items[:min(len(items), max_items_per_category)]
category_jsons = []
for category in categories:
category_jsons.append({"id": category, "name": category, "sources": [s.json() for s in sources_by_category[category]]})
return category_jsons
def post(self):
taskqueue.Queue('sources').purge()
time.sleep(1) # TODO: anything but this; we need to wait 1 seconds, but how?
for source in Source.query():
source.most_recent_article_added_date = None
source.enqueue_fetch(rand=True)
self.response.write('done')
def test_propagate_short(self):
grid = Grid(8, 8)
grid.add_source(Source(2, 1))
grid.propagate(9)
self.assertEqual(grid[0], [999.0, 999.0, 999.0, 999.0])
self.assertEqual(grid[1], [1, 1, 1, 999.0])
self.assertEqual(grid[2], [1, 0, 1, 999.0])
self.assertEqual(grid[3], [1, 1, 1, 999.0])
def test_propagate(self):
grid = Grid(8, 8)
grid.add_source(Source(2, 1))
grid.propagate()
print (grid)
self.assertEqual(grid[0], [2, 2, 2, 2])
self.assertEqual(grid[1], [1, 1, 1, 2])
self.assertEqual(grid[2], [1, 0, 1, 2])
self.assertEqual(grid[3], [1, 1, 1, 2])
def post(self):
url = canonical_url(self.request.get('url'))
source_id = Source.id_for_source(url)
source = ndb.Key(Source, source_id).get()
while True:
articles = Article.query(Article.source == source.key).order(-Article.added_date, Article.added_order).fetch(limit=100, keys_only=True)
if len(articles) == 0: break
ndb.delete_multi(articles)
source.key.delete()
self.response.write('Done')
def ensure_source(url, suppress_immediate_fetch=False):
url = canonical_url(url)
source_id = Source.id_for_source(url)
source, inserted = get_or_insert(Source, source_id)
if inserted:
source.url = url
source.put()
source.enqueue_fetch()
if inserted and not suppress_immediate_fetch:
source.fetch_now()
return source
def generate_sources():
book_sections = session.query(Section).filter(Section.docpath != "")
names = set()
for s in book_sections:
name = get_section_name(s)
if name != '' and name not in names:
print(name)
session.add(Source(name=name, type='fca_sourcebook'))
names.add(name)
session.commit()
def save_source(topic, date, windowsize, source):
for ss in source:
item = Source(topic, date, windowsize, ss)
item_exist = db.session.query(Source).filter(Source.topic==topic ,\
Source.date==date ,\
Source.windowsize==windowsize ,\
Source.userid==ss).first()
if item_exist:
db.session.delete(item_exist)
db.session.add(item)
db.session.commit()
print 'success save source'
def _foo():
"""test, do not use"""
amazon_src = Source.byName("amazon")
done_artists = Artist.select(Artist.q.recordings_updated != None)
for artist in done_artists:
for record in artist.recordings:
if record.source == amazon_src:
record.destroySelf()
for recording in amazon.recordings(artist.name):
Recording(
name=recording["name"], by=artist, url=recording["url"], img_url=recording["img_url"], source=amazon_src
)
def recording_artist_update(artist):
# remove old entries
for record in Recording.selectBy(by=artist):
record.destroySelf()
# add new entries (if any)
for recording in cdbaby.recordings(artist.name):
Recording(
name=recording["name"],
by=artist,
url=recording["url"],
img_url=recording["img_url"],
source=Source.byName("cdbaby"),
)
for recording in amazon.recordings(artist.name):
Recording(
name=recording["name"],
by=artist,
url=recording["url"],
img_url=recording["img_url"],
source=Source.byName("amazon"),
)
def create_bot():
"""Adds bot to DB."""
name = request.form.get('name')
desc = request.form.get('description')
data_source = request.form.get('source')
content_type = request.form.get('type')
icon = request.form.get('icon')
if content_type == "text_file":
data_source = []
file_list = request.files.getlist("text_file")
for file in file_list:
filename = secure_filename(file.filename)
file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
data_source.append("uploads/" + filename)
content = process_source(content_type, data_source)
if content == False:
flash('error in bot creation! make sure your sources are correct?')
return redirect("/")
else:
source = Source(content_type=content_type,
content_source=data_source,
content=content)
db.session.add(source)
db.session.commit()
bot = Bot(bot_name=name,
creator_id = session['user_id'],
bot_description=desc,
bot_icon=icon,
source_id=source.source_id)
db.session.add(bot)
db.session.commit()
flash('bot planted!')
return redirect("/bot/" + str(bot.bot_id))
def new_source(self, e):
self.glwindow.sources.append(Source()) # 新建信号源实例
self.glwindow.update()
index = self.stackedSource.count()
new = SourceControl(index, self.glwindow, self)
self.stackedSource.addWidget(new)
self.source_selector.addItem('wifi_' + str(self.source_count))
self.source_count += 1
self.source_selector.setCurrentIndex(index)
self.stackedSource.setCurrentIndex(index)
self.stackedSource.widget(index).source_delete.connect(
self.on_source_delete) # 绑定事件
self.stackedSource.widget(index).source_modified.connect(
self.on_modified)
self.glwindow.active_source = index
self.glwindow.active_building = -1
def add_sources():
print("Sources")
# Deletes any existing data in the users table
Source.query.delete()
for line in open("seed_files/sources.txt"):
line = line.rstrip()
content_type, content_source = line.split("|")
content = process_source(content_type, content_source)
source = Source(content_type=content_type,
content_source=content_source,
content=content)
db.session.add(source)
db.session.commit()
def load_sources():
"""Load real sources into database."""
print "Sources"
Source.query.delete()
src_dict = {
'mtup': 'Meetup.com',
'evtb': 'Eventbrite.com',
'sfpk': 'SF Parks and Recreation'
}
for k, v in src_dict.items():
src_id = k
name = v
source = Source(src_id=src_id, name=name)
db.session.add(source)
db.session.commit()
def main():
filename = config("savefilename")
lr = 0.0001
this_config = dict(csv_file=config("csv_file"),
img_path=config("image_path"),
learning_rate=lr,
num_classes=4,
batchsize=64)
wandb.init(project="prob_fix", name=filename, config=this_config)
tr_loader, va_loader, te_loader, _ = get_train_val_test_loaders(
task="default", batch_size=config("net.batch_size"))
print('successfully loading!')
model = Source()
criterion = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=lr)
print("Number of float-valued parameters:", count_parameters(model))
model, start_epoch, stats = restore_checkpoint(model,
config("cnn.checkpoint"))
axes = utils.make_training_plot()
prolist = []
evaluate_epoch(axes,
tr_loader,
va_loader,
te_loader,
model,
criterion,
start_epoch,
stats,
prolist,
multiclass=True)
# initial val loss for early stopping
prev_val_loss = stats[0][1]
# TODO: define patience for early stopping
patience = 5
curr_patience = 0
#
# Loop over the entire dataset multiple times
# for epoch in range(start_epoch, config('cnn.num_epochs')):
epoch = start_epoch
lowest_val_loss = 1
train_auroc = 0
test_auroc = 0
lowest_round = epoch
while curr_patience < patience:
# Train model
train_epoch(tr_loader, model, criterion, optimizer)
# Evaluate model
evaluate_epoch(axes,
tr_loader,
va_loader,
te_loader,
model,
criterion,
epoch + 1,
stats,
prolist,
multiclass=True)
# Save model parameters
save_checkpoint(model, epoch + 1, config("net.checkpoint"), stats)
# update early stopping parameters
curr_patience, prev_val_loss = early_stopping(stats, curr_patience,
prev_val_loss)
epoch += 1
if (prev_val_loss < lowest_val_loss):
lowest_val_loss = prev_val_loss
lowest_round = epoch
pickle.dump(prolist, open("base_pro.pck", "wb"))
print("Finished Training")
# Save figure and keep plot open
print("the lowest round: ", lowest_round)
# utils.save_cnn_training_plot()
# utils.save_cnn_other()
utils.hold_training_plot()
def import_to_db(self, event):
flag_for_review = False
venue_name = self.venue_name_fix(event['venue']['name'])
try:
v = Venue.byNameI(venue_name)
except SQLObjectNotFound:
v = Venue(name=venue_name, added_by=identity.current.user)
flag_for_review = True
if event['venue'].has_key('phone'):
phone = event['venue']['phone']
if not len(phone) >= 8:
phone = "503-" + phone
p = validators.PhoneNumber()
try:
event['venue']['phone'] = p.to_python(phone)
except:
event['venue']['phone'] = None
self._set_optional_fields(v, event['venue'], ("address", "phone", "zip_code",
"url", "description"))
event_name = self.event_name_fix(event['name'])
event_date = event["date"]
event_time = event.get("time")
if event_time:
event_time = event_time.lower()
# check same venue, date, time
db_events = Event.selectBy(date=event_date,
time=event_time, venue=v)
if db_events.count():
e = db_events[0]
new_event = False
else:
# no time? still could be skippable, if event name is the same
db_events = Event.selectBy(date=event_date,
name=event_name, venue=v)
if db_events.count():
e = db_events[0]
new_event = False
else:
e = Event(venue=v, name=event_name,
date=event_date, time=event_time,
added_by=identity.current.user)
new_event = True
try:
s = Source.byName(event["source"])
except SQLObjectNotFound:
s = Source(name=event["source"])
flag_for_review = True
e.addSource(s)
self._set_optional_fields(e, event, ("cost", "ages", "url",
"description", "ticket_url"))
for artist in self.artists_clean(event['artists']):
try:
a = Artist.byNameI(artist)
# artist was a similar, but now it's listed as having an event
# so approve it
if not a.approved:
a.approved = datetime.now()
except SQLObjectNotFound:
a = Artist(name=artist, added_by=identity.current.user)
flag_for_review = True
if not e.id in [existing.id for existing in a.events]:
a.addEvent(e)
# flag all events from certain unreliable sources
if event["source"] in ("ticketswest", "upcoming", "lastfm"):
flag_for_review = True
if new_event and not flag_for_review:
e.approved = datetime.now()
elif flag_for_review:
e.approved = None
return (new_event, flag_for_review)
def fill_basics():
with app.app_context():
Police1 = Police(police_dept_id=1, name="User_Input")
Police2 = Police(police_dept_id=2,
name="San Franciso Police Department",
city="San Francisco",
state="CA")
Police3 = Police(police_dept_id=3,
name="Oakland Police Department",
city="Oakland",
state="CA")
Police4 = Police(police_dept_id=4,
name="Alameda County Sheriff's Department",
city="Oakland",
state="CA")
Police5 = Police(police_dept_id=5,
name="Santa Clara County Sheriff's Office",
city="San Jose",
state="CA")
Police6 = Police(police_dept_id=6,
name="Fremont Police Department",
city="Fremont",
state="CA")
Police7 = Police(police_dept_id=7,
name="San Leandro Police Department",
city="San Leandro",
state="CA")
Police8 = Police(police_dept_id=8,
name="San Pablo Police Department",
city="San Pablo",
state="CA")
Police9 = Police(police_dept_id=9,
name="New York Police Department",
city="New York",
state="NY")
user = User(password="******",
username="******",
fname="Test123",
lname="Test123",
email="*****@*****.**",
description="Former Escort",
created_at=datetime.now(),
edited_at=datetime.now())
db.session.add_all([
Police1, Police2, Police3, Police4, Police5, Police6, Police7,
Police8, Police9, user
])
db.session.commit()
source2 = Source(
source_id=3,
s_name="DataSF",
police_dept_id=2,
s_description="San Franciso Police API",
url="https://data.sfgov.org/resource/cuks-n6tp.json?$limit=50000",
s_type="gov api")
source3 = Source(source_id=2,
s_name="DataSF",
police_dept_id=2,
s_description="San Franciso Police API",
url="https://data.sfgov.org/resource/PdId.json",
s_type="gov api")
source = Source(source_id=1, s_name="User_report")
source4 = Source(
source_id=4,
s_name="oaklandnet_90_days",
police_dept_id=3,
s_description="Oakland Police API Last 90 Days",
url="ftp://crimewatchdata.oaklandnet.com/crimePublicData.csv",
s_type="gov api")
source5 = Source(
source_id=5,
s_name="socrata",
police_dept_id=4,
s_description="Alameda County Sheriff's Department API",
url=
"https://moto.data.socrata.com/resource/bvi2-5rde.json?$where=incident_description%20like%20%27%25PROST%25%27",
s_type="gov api")
source6 = Source(
source_id=6,
s_name="socrata",
police_dept_id=5,
s_description="Santa Clara County Sheriff's Office API",
url=
"https://moto.data.socrata.com/resource/wrmr-tdyp.json?$where=incident_description%20like%20%27%25PROST%25%27",
s_type="gov api")
source7 = Source(
source_id=7,
s_name="socrata",
police_dept_id=6,
s_description="FPD API",
url=
"https://moto.data.socrata.com/resource/nnzs-rxi5.json?$where=incident_description%20like%20%27%25PROST%25%27",
s_type="gov api")
source8 = Source(
source_id=8,
s_name="socrata",
police_dept_id=7,
s_description="SLPD API",
url=
"https://moto.data.socrata.com/resource/6nbc-apvm.json?$where=incident_description%20like%20%27%25PROST%25%27",
s_type="gov api")
source9 = Source(
source_id=9,
s_name="socrata",
police_dept_id=8,
s_description="SPPD API",
url=
"https://moto.data.socrata.com/resource/tsdt-igxn.json?$where=incident_description%20like%20%27%25PROST%25%27",
s_type="gov api")
source10 = Source(
source_id=10,
s_name="socrata",
police_dept_id=9,
s_description="NYPD Socrata API Recent Data",
url=
"https://data.cityofnewyork.us/resource/7x9x-zpz6.json?$where=pd_desc%20like%20%27%25PROST%25%27",
s_type="gov api")
source11 = Source(
source_id=11,
s_name="socrata",
police_dept_id=9,
s_description="NYPD Socrata API Historic Data",
url=
"https://data.cityofnewyork.us/resource/9s4h-37hy.json?$where=pd_desc%20like%20%27%25PROST%25%27&$limit=50000",
s_type="gov api")
db.session.add_all([
source2, source3, source4, source5, source6, source7, source8,
source9, source10, source11
])
db.session.commit()
def parse_source(row, name=None, type_=None, url=None, provenance=None):
"""Parse a row into a Source object.
Parameters
----------
row : dict-like
Keys are column headings and values are the row values of a metadata
file.
name : str, optional
Name of data source.
type_ : str, optional
Type of data source.
url : str, optional
URL of data source.
provenance : Provenance, optional
Provenance object to be linked to the data source.
Raises
------
Exception : if there are no columns ending in 'study_id'.
Exception : if no source name could be found for the given row and no name
argument was given.
Returns
-------
Source
"""
source = Source()
# Initialize default values for attributes
source_name = str(name) if name else None
source_type = str(type_) if type_ else None
source_url = str(url) if url else None
# Find source_name in 'study_id' column
for key in row.keys():
match = re.match(r'(.*)study_id$', key)
if match:
if match.group(1):
source_name = match.group(1).rstrip('_')
break
else:
raise Exception('The metadata file must contain exactly one column '
'ending in "study_id".')
# Replace source_name with 'source' column value if it exists
new_source_name = row.get('source', '')
if not re_missing.match(new_source_name):
source_name = new_source_name
# If source_name could not be initialized
if not source_name:
raise Exception('No source name could be found in the "study_id" '
'column heading, or as a valid value in the "source" '
'column of the metadata_file, and no source name '
'argument was provided.')
# Replace source_type with 'source_type' column value if it exists
new_source_type = row.get('source_type', '')
if not re_missing.match(new_source_type):
source_type = new_source_type
if not source_type:
raise Exception('No source type could be found in the "source_type" '
'column of the metadata_file, and no source type_ '
'argument was provided.')
# Replace source_url with 'source_url' column value if it exists
new_source_url = row.get('source_url', '')
if not re_missing.match(new_source_url):
source_url = new_source_url
if not source_url:
raise Exception('No source url could be found in the "source_url" '
'column of the metadata_file, and no source url'
'argument was provided.')
# Assign to source attributes
source.name = source_name
source.type_ = source_type
source.url = source_url
if provenance:
source.provenances.append(provenance)
return source
self.buildings[active].move('forward')
self.building_z_changed.emit(self.buildings[active].position[2])
elif key == Qt.Key_Down:
self.buildings[active].move('back')
self.building_z_changed.emit(self.buildings[active].position[2])
elif key == Qt.Key_Left:
self.buildings[active].move('left')
self.building_x_changed.emit(self.buildings[active].position[0])
elif key == Qt.Key_Right:
self.buildings[active].move('right')
self.building_x_changed.emit(self.buildings[active].position[0])
if __name__ == '__main__':
glutInit()
displayMode = GLUT_DOUBLE | GLUT_ALPHA | GLUT_DEPTH
glutInitDisplayMode(displayMode)
app = QApplication(sys.argv)
building = Building(floors=5, rooms=5, floor_thickness=1, room_length=12, room_width=6, room_height=5, wall_thickness=0.2)
glwindow = GLWindow()
source = Source()
glwindow.sources.append(source)
# source2 = Source(pos_x=10)
# glwindow.sources.append(source2)
glwindow.setGeometry(200, 200, 960, 680)
glwindow.show()
sys.exit(app.exec_())
class SampleParserTest(unittest.TestCase):
sample_test_file = './data/test_data/samp_metadata/sample1.txt'
row = OrderedDict([
('sample_name', '317.F10'), ('age', '22'), ('age_unit', 'years'),
('altitude', '0'), ('anatomical_body_site', 'FMA:Palm'),
('anonymized_name', 'F10'), ('body_habitat', 'UBERON:skin'),
('body_product', 'UBERON:sebum'),
('body_site', 'UBERON:zone of skin of hand'),
('collection_date', '11/12/2006'),
('country', 'GAZ:United States of America'), ('depth', '0'),
('description', 'human skin metagenome'), ('dna_extracted', 'true'),
('dominant_hand', ''), ('elevation', '1591.99'),
('env_biome', 'ENVO:human-associated habitat'),
('env_feature', 'ENVO:human-associated habitat'),
('host_common_name', 'human'), ('host_subject_id', 'F1'),
('host_taxid', '9606'), ('latitude', '40'), ('longitude', '-105'),
('palm_size', ''), ('physical_specimen_remaining', 'false'),
('public', 'true'), ('qiita_study_id', '317'),
('sample_type', 'XXQIITAXX'), ('sex', 'female'),
('time_since_last_wash', '0'),
('title',
'The influence of sex handedness and washing on the diversity of hand surface bacteriaS1_V160'
)
])
dayfirst_dict = {'collection_date': False}
# TODO Update details of source (when necessary)
source1 = Source(name='qiita',
type_='Database (Public)',
url='https://qiita.ucsd.edu/study/description/0')
experiment1 = Experiment(source=source1, orig_study_id='317')
subject1 = Subject(
source=source1,
orig_study_id='317',
orig_subject_id='F1',
sex='female',
country='United States of America',
race=None,
csection=None,
disease=None,
dob=None,
)
subject2 = Subject(
source=source1,
orig_study_id='317',
orig_subject_id='F2',
sex='female',
country='United States of America',
race=None,
csection=None,
disease=None,
dob=None,
)
sampling_site = SamplingSite(
uberon_habitat_term='UBERON:skin',
uberon_product_term='UBERON:sebum',
uberon_site_term='UBERON:zone of skin of hand',
env_biom_term='ENVO:human-associated habitat',
env_feature_term='ENVO:human-associated habitat')
sampling_time = Time(timestamp=datetime.datetime(2006, 11, 12),
uncertainty=None,
date=datetime.date(2006, 11, 12),
time=None,
year=2006,
month=11,
day=12,
hour=None,
minute=None,
second=None,
season='autumn')
sample1 = Sample(source=source1,
orig_study_id='317',
orig_subject_id='F1',
orig_sample_id='317.F10',
age_units=ureg.years,
age=22.0,
latitude=40.0,
longitude=-105.0,
elevation=1591.99,
height_units=ureg.metres,
height=None,
weight_units=ureg.kilograms,
weight=None,
bmi=None,
sample_date=datetime.date(2006, 11, 12),
sample_time=None,
sampling_site=sampling_site,
sampling_time=sampling_time)
sample2 = Sample(source=source1,
orig_study_id='317',
orig_subject_id='F1',
orig_sample_id='317.F12',
age_units=ureg.years,
age=22.0,
latitude=40.0,
longitude=-105.0,
elevation=1591.99,
height_units=ureg.metres,
height=None,
weight_units=ureg.kilograms,
weight=None,
bmi=None,
sample_date=datetime.date(2006, 11, 12),
sample_time=None,
sampling_site=sampling_site,
sampling_time=sampling_time)
sample3 = Sample(source=source1,
orig_study_id='317',
orig_subject_id='F2',
orig_sample_id='317.F20',
age_units=ureg.years,
age=None,
latitude=40.0,
longitude=-105.0,
elevation=1591.99,
height_units=ureg.metres,
height=None,
weight_units=ureg.kilograms,
weight=None,
bmi=None,
sample_date=datetime.date(2006, 11, 12),
sample_time=None,
sampling_site=sampling_site,
sampling_time=sampling_time)
# Not necessary to establish these relationships for purpose of
# test_parse_objects:
sample1._subject = subject1
sample2._subject = subject1
sample3._subject = subject2
subject1._samples = {sample1, sample2}
subject2._samples = {sample3}
experiment1._subjects = {subject1, subject2}
experiment1._samples = {sample1, sample2, sample3}
def test_parse_objects(self):
experiment_ids = parse_objects(self.sample_test_file)
self.assertIn('317', experiment_ids)
experiment = experiment_ids['317']
self.assertEqual(self.experiment1, experiment)
self.assertIn(self.subject1, experiment.subjects)
self.assertIn(self.subject2, experiment.subjects)
self.assertIn(self.sample1, experiment.samples)
self.assertIn(self.sample2, experiment.samples)
self.assertIn(self.sample3, experiment.samples)
# TODO: We will have to test without the source keyword at some point.
def test_parse_sample(self):
self.maxDiff = None
blacklist_attrs = [
'_sa_instance_state', 'source', 'counts', '_experiments',
'_subject', '_preparations'
]
sample = parse_sample(self.row,
self.dayfirst_dict,
source=self.source1)
sample_attrs = set((key, value)
for key, value in sample.__dict__.items()
if key not in blacklist_attrs)
expected_attrs = set((key, value)
for key, value in self.sample1.__dict__.items()
if key not in blacklist_attrs)
self.assertEqual(sample_attrs, expected_attrs)
self.assertEqual(sample.source, self.source1)
self.assertEqual(sample.counts, self.sample1.counts)
# When sample is parsed, it is not yet associated with subject/experiments
self.assertEqual(sample._subject, None)
self.assertEqual(sample._experiments, set())
self.assertEqual(sample._preparations, set())
def test_parse_subject(self):
self.maxDiff = None
blacklist_attrs = [
'_sa_instance_state', 'source', 'counts', 'perturbation_facts',
'_experiments', '_samples', '_perturbations'
]
subject = parse_subject(self.row, source=self.source1)
subject_attrs = set((key, value)
for key, value in subject.__dict__.items()
if key not in blacklist_attrs)
expected_attrs = set((key, value)
for key, value in self.subject1.__dict__.items()
if key not in blacklist_attrs)
self.assertEqual(subject_attrs, expected_attrs)
self.assertEqual(subject.source, self.source1)
self.assertEqual(subject.counts, self.subject1.counts)
self.assertEqual(subject.perturbation_facts,
self.subject1.perturbation_facts)
# When subject is parsed, it is not yet associated with samples/experiments
self.assertEqual(subject._experiments, set())
self.assertEqual(subject._samples, set())
self.assertEqual(subject._perturbations, set())
def test_parse_processing(self):
self.maxDiff = None
processing1 = Processing(parent=None,
parameter_values='{}',
orig_prep_id='577',
orig_proc_id='2593')
processing2 = Processing(parent=processing1,
parameter_values='{'
'"barcode_type":"golay_12",'
'"command":"Split libraries (QIIMEq2 1.9.1)",'
'"disable_bc_correction":"False",'
'"disable_primers":"False",'
'"generated on":"2016-01-14 17:01",'
'"input_data":"2593",'
'"max_ambig":"6",'
'"max_barcode_errors":"1.5",'
'"max_homopolymer":"6",'
'"max_primer_mismatch":"0",'
'"max_seq_len":"1000",'
'"min_qual_score":"25",'
'"min_seq_len":"200",'
'"qual_score_window":"0",'
'"reverse_primer_mismatches":"0",'
'"reverse_primers":"disable",'
'"trim_seq_length":"False",'
'"truncate_ambi_bases":"False"'
'}',
orig_prep_id='577',
orig_proc_id='310')
processing3 = Processing(
parent=processing2,
parameter_values='{'
'"command":"Pick closed-reference OTUs (QIIMEq2 1.9.1)",'
'"generated on":"2015-06-30 14:06",'
'"input_data":"310",'
'"reference-seq":"/databases/gg/13_8/rep_set/97_otus.fasta",'
'"reference-tax":"/databases/gg/13_8/taxonomy/97_otu_taxonomy.txt",'
'"similarity":"0.97",'
'"sortmerna_coverage":"0.97",'
'"sortmerna_e_value":"1",'
'"sortmerna_max_pos":"10000",'
'"threads":"1"'
'}',
orig_prep_id='577',
orig_proc_id='2594')
expected_processings = {
'2593': processing1,
'310': processing2,
'2594': processing3
}
processings = parse_processings('./data/test_data/proc1.json')
# TODO: Implement workflows and parents as mocks?
blacklist_attrs = ['_sa_instance_state', 'workflows', 'parent']
for proc_id, processing in processings.items():
self.assertIn(proc_id, expected_processings)
processing_attrs = set(
(key, value) for key, value in processing.__dict__.items()
if key not in blacklist_attrs)
expected_attrs = set(
(key, value) for key, value in
expected_processings[proc_id].__dict__.items()
if key not in blacklist_attrs)
self.assertEqual(processing_attrs, expected_attrs)
def __findBuildPath(path):
return Source.findBuildPath(path, modulePath)
def __findBuildPath(path):
return Source.findBuildPath(path, modulePath)
_ = gcam.forward(xi)
gcam.backward(ids=torch.tensor([[target_class]]).to(device))
regions = gcam.generate(target_layer=target_layer)
activation = regions.detach()
save_gradcam(
np.squeeze(activation),
utils.denormalize_image(np.squeeze(xi.numpy()).transpose(1, 2, 0)),
axarr,
i,
)
if __name__ == "__main__":
# Attempts to restore from checkpoint
print("Loading cnn...")
model = Source()
model, start_epoch, _ = restore_checkpoint(model,
config("net.checkpoint"),
force=True)
tr_loader, va_loader, te_loader, _ = get_train_val_test_loaders(
task="target",
batch_size=config("net.batch_size"),
)
for i in range(40):
plt.clf()
f, axarr = plt.subplots(1, 2)
visualize_input(i, axarr)
visualize_layer1_activations(i, axarr)
plt.close()
def get(self):
vars = {
"featured_sources": Source.query().filter(Source.featured_priority >= 0).order(-Source.featured_priority).fetch()
}
self.response.write(template('sources.html', vars))