def init(section):
print color.success_blue('# ') + 'Fetching stories from Backlog and Queue...'
progress.update(0, 2)
backlog_stories = filter_not_accepted(section.backlog.fetch_stories())
backlog_stories_sorted = effort_sorted(backlog_stories)
progress.update(1, 2)
queue_backlog = filter_backlog(section.queue.fetch_stories())
progress.finish()
print color.success_blue('# ') + 'Writing original story order to config file...'
section.order = map(lambda story: (story.id, story.current_state), backlog_stories)
section.write_order()
progress.finish()
def move_to_estimator(story, origin):
story.description = getattr(story, 'description', '') + '\n' + origin_key + origin
story.sync_to_project(section.estimator, safe=True)
for story in backlog_stories_sorted:
if story.story_type == 'release' and story.current_state == 'unscheduled':
story.current_state = 'unstarted'
continue
if story.story_type != 'release' and story.current_state in ['unscheduled', 'unstarted']:
story.current_state = 'started'
if not hasattr(story, 'estimate') and story.is_estimatable():
print color.warn('Warning: ') + 'Story %s has no estimate. Adding an estimate of one; this is needed to start the story in Estimator.' % story.name
story.estimate = 1
print color.success_blue('# ') + 'Moving stories from Backlog to Estimator...'
progress.iter(backlog_stories_sorted, move_to_estimator, 'backlog')
for story in queue_backlog: story.current_state = 'unscheduled'
print color.success_blue('# ') + 'Moving stories from Queue to Estimator...'
progress.iter(queue_backlog[::-1], move_to_estimator, 'queue')
def run(netdef,tosave,modify,procs,thisProc,stims,param,repeats,sim_time,SaveSpikes,SaveVoltage,SaveConductance,SaveCurrent):
net = netdef()
if SaveVoltage:
net.recordVoltage()
repeats = int(repeats)
# Randomseed was 200 for most figures
# Changed to 200 for rat
# Changed to 200 for anurans
s = Simulation(net, randomseed=202,delay=25)
s.verbose = False
s.sim_time = sim_time
s.dt = 0.050
total = len(stims)*len(param)*repeats
spp = ceil(float(total)/procs)
start = thisProc*spp
end = (thisProc+1)*spp
count = 0
for a in param:
s.set_amplitude(net.sim_amp)
for d in stims*repeats:
if count >= start and count < end:
net = modify(net,a,d)
progress.update(count-start,spp,thisProc)
s.stim_dur = d
s.run()
key = [a,d]
net.savecells(tosave, key, spikes=SaveSpikes,voltage=SaveVoltage,conductance=SaveConductance,current=SaveCurrent)
count += 1
progress.update(spp,spp,thisProc)
r = [thisProc,net.savedparams,net.savedcells]
return r
def do_search(api,db,keyword_query,geocode,from_id,to_id,next_id):
#r = api.request('statuses/filter', {'locations': '112.5,-37.5,154.1,-12.8'})
next_id=-1
cur_id=-1
if from_id==-1:
from_id=None
if to_id==-1:
to_id=0
count=0
pager = TwitterPager(api, 'search/tweets', {'q': keyword_query, 'geocode': geocode, 'count': '100', 'max_id': str(from_id), 'since_id' : str(to_id)})
while True:
try:
for item in pager.get_iterator():
#print(item)
if 'text' in item:
#try:
if True:
#print item["id"]
cur_id=int(item["id"])
#if next_id != -1, we run in re-start mode, don't reset next_id
#else we need to update next_id when the first item arrives in this iteration
#and next iteration's to_id will be set to next_id of this iteration
if next_id==-1:
next_id=cur_id
if cur_id<=to_id:
break
info=get_dict_object_from_tweet(item)
if not info:
print "Error parsing the tweet, ignore it"
continue
#put the data in the db
db.put(info)
count+=1
if count % 1000 == 0:
print count
#print item["id"],"ok"
#print(info["post_text"])
#persist the progress to ensure we can resume the harvester from here
progress.update(cur_id,to_id,next_id)
elif 'message' in item:
# something needs to be fixed before re-connecting
raise Exception(item['message'])
return count
except TwitterAPI.TwitterError.TwitterRequestError,e:
if e.status_code==429:
print ("Too Many Requests, now sleeping...")
sleep(60)
else:
raise e
def run_session(section):
if section.has_estimator_id():
print color.success_blue('# ') + 'Loaded existing Estimator session'
else:
print color.success_blue('# ') + 'No existing Estimator session found, starting new session...'
print color.success_blue('# ') + 'Making backup of Queue and Backlog...'
backup.backup_project(section.queue)
progress.update(1, 2)
backup.backup_project(section.backlog)
progress.finish()
try:
section.estimator = create_estimator(section)
section.write_estimator_id()
try:
print color.success_blue('# ') + 'Sharing Estimator with all members in account...'
share_estimator_with_account(section)
except Exception:
pass
except RuntimeError:
est_id = raw_input(color.err('ERROR: ') + 'Estimator project already exists, but was not found in the config file. Please enter id of the Estimator project: ')
section.estimator = piv.Project.load(int(est_id))
section.write_estimator_id()
section.imported = False
section.write_imported()
if not section.has_imported():
stories = section.estimator.fetch_stories()
if len(stories) != 0:
print color.success_blue('# ') + 'Estimator project has not imported all the stories from Queue and Backlog; continuing imports...'
init(section)
section.imported = True
section.write_imported()
print color.success_blue('# ') + 'Estimator project at: https://www.pivotaltracker.com/n/projects/' + str(section.estimator.id)
query_user_input(section)
print color.success_blue('# ') + 'Making backup of Estimator...'
backup.backup_project(section.estimator)
progress.finish()
finalize(section)
def processCommits(process_commits):
global commits
processed_commits = set()
for commit_id in process_commits:
try:
gitobject = repository.fetch(commits[commit_id])
if gitobject.type == "commit": processed_commits.add(commit_id)
except gitutils.GitError:
pass
except KeyboardInterrupt:
sys.exit(1)
except:
raise
progress.update()
return processed_commits
def processCommits(process_commits):
global commits
processed_commits = set()
for commit_id in process_commits:
try:
gitobject = repository.fetch(commits[commit_id])
if gitobject.type == "commit": processed_commits.add(commit_id)
except gitutils.GitError:
pass
except KeyboardInterrupt:
sys.exit(1)
except:
raise
progress.update()
return processed_commits
commits = {}
pending_commits = set()
cursor.execute("SELECT COUNT(*) FROM commits")
print
progress.start(cursor.fetchone()[0], prefix="Fetching commits ...")
cursor.execute("SELECT id, sha1 FROM commits")
for commit_id, commit_sha1 in cursor:
commits[commit_id] = commit_sha1
pending_commits.add(commit_id)
progress.update()
progress.end(" %d commits." % len(commits))
print
cursor.execute("SELECT MAX(CHARACTER_LENGTH(name)) FROM repositories")
repository_name_length = cursor.fetchone()[0]
cursor.execute("SELECT id FROM repositories ORDER BY id ASC")
repositories = [repository_id for (repository_id,) in cursor]
def processCommits(process_commits):
global commits
if len(sha1) == 40 and ref.startswith("refs/heads/"):
refs[ref[11:]] = sha1
except ValueError:
pass
except IOError as error:
if error.errno == errno.ENOENT: pass
else: raise
progress.start(len(branches), "Repository: %s" % repository.name)
heads_path = os.path.join(repository.path, "refs", "heads")
branches_in_db = set()
for branch_id, branch_name, branch_type, branch_base_id, branch_sha1 in branches:
progress.update()
branches_in_db.add(branch_name)
try:
try: repository_sha1 = open(os.path.join(heads_path, branch_name)).read().strip()
except: repository_sha1 = refs.get(branch_name)
if repository_sha1 != branch_sha1:
progress.write("NOTE[%s]: %s differs (db:%s != repo:%s)" % (repository.name, branch_name, branch_sha1[:8], repository_sha1[:8]))
if branch_type == "review":
head = getReviewHead(repository, getReview(branch_id))
if not head:
progress.write(" invalid review meta-data: r/%d" % getReview(branch_id))
repeats = 15
param = [(i*2)+10 for i in range(11)]
total = len(stims)*len(param)*repeats
print "Running %d simulations..." % total
count = 0
for a in param:
for net in networks:
networks["AC"].cells["IC"]["cells"][0].sec["soma"].L = a
networks["AC"].cells["IC"]["cells"][0].sec["soma"].diam = a
networks["C"].cells["IC"]["cells"][0].sec["soma"].L = a
networks["C"].cells["IC"]["cells"][0].sec["soma"].diam = a
for d in stims*repeats:
progress.update(count, total)
count += 1
for net in networks:
s[net].stim_dur = d
s[net].run()
key = [a,d]
networks[net].savecells([["IC","soma"],["IC","dendE"]], key, spikes=True,
conductance=False,
current=False,
voltage=True)
if True:
ns.plot_mean_spikes(networks["C"], "IC-soma", "c_soma_size.dat")
ns.plot_mean_spikes(networks["AC"], "IC-soma", "ac_soma_size.dat")
ns.show() # Comment out to just save the results to file
def recommend(user_user_similarities, user_ids):
"""Recommend with the test set
Recommend with the test set, and return the user_news_rating_predictions array.
Args:
user_user_similarities(Type: numpy.ndarray): The similarities between user and user(user_user_similarities[i, j] represents the similarity between user "i" and user "j". Similarity of oneself is "1".)
user_ids(Type: numpy.ndarray(vector)): user's ids(from small to large) associated with the array
Returns:
user_news_rating_predictions(Type: numpy.ndarray): The rating prediction of each user to each news(rating_prediction[i, j] represents the rating prediction of user "i" to news "j".)
user_ids(Type: numpy.ndarray(vector)): user's ids(from small to large) associated with the array
news_ids(Type: numpy.ndarray(vector)): news's ids(from small to large) associated with the array
"""
# get user-news array
user_news_array_of_train = numpy.load("user_news_array_of_train.npy")
user_ids_of_train = numpy.load("user_ids_of_train.npy")
news_ids_of_train = numpy.load("news_ids_of_train.npy")
user_news_array_of_test = numpy.load("user_news_array_of_test.npy")
user_ids_of_test = numpy.load("user_ids_of_test.npy")
news_ids_of_test = numpy.load("news_ids_of_test.npy")
user_num_of_train = len(user_ids_of_train)
news_num_of_train = len(news_ids_of_train)
user_num_of_test = len(user_ids_of_test)
news_num_of_test = len(news_ids_of_test)
# find k nearest neighbors of users
print("Find k nearest neighbors of users started.")
time_start = time.time()
neighbor_size = min(20, user_num_of_train-1) # TODO refine this parameter
user_neighbors_indexes = numpy.zeros((user_num_of_test, neighbor_size), numpy.int16)
user_ids_of_train_dict = {user_id: index for (index, user_id) in enumerate(user_ids_of_train)}
user_index_from_test_to_train_dict = {user_index_in_test: user_ids_of_train_dict[user_id_of_test] for (user_index_in_test, user_id_of_test) in enumerate(user_ids_of_test)} # dictionary of user index from test to train
for i in range(user_num_of_test):
sorted_indexes = numpy.argsort(-user_user_similarities[user_index_from_test_to_train_dict[i], :])
user_neighbors_indexes[i, :] = sorted_indexes[1:neighbor_size+1] # choose the first k in the sorted list, remove the first one which is oneself
if i % 100 == 0:
# print("%.1f%%" % (i / user_num_of_test * 100))
progress.update(i / user_num_of_test)
progress.update(1)
time_end = time.time()
print("Find k nearest neighbors of users ended. %f s cost." % (time_end - time_start))
# predict ratings
print("Predict ratings started.")
time_start = time.time()
news_ids_of_test_dict = {news_id: index for (index, news_id) in enumerate(news_ids_of_test)}
news_index_from_train_to_test_dict = {news_index_in_train: news_ids_of_test_dict[news_id_of_train] for (news_index_in_train, news_id_of_train) in enumerate(news_ids_of_train)} # dictionary of news index from train to test
user_news_array_of_train_expanded = numpy.zeros((user_num_of_train, news_num_of_test), numpy.int8) # int is faster than bool_
user_news_array_of_train_expanded[:, [news_index_from_train_to_test_dict[i] for i in range(news_num_of_train)]] = user_news_array_of_train # expand the column of user_news_array_of_train to the same size as user_news_array_of_test
user_news_rating_predictions = numpy.zeros((user_num_of_test, news_num_of_test), numpy.float16)
eps = numpy.finfo(float).eps
for i in range(user_num_of_test):
this_user_index_in_test = user_index_from_test_to_train_dict[i]
similarities_sum = 0.0
for user_neighbors_index in user_neighbors_indexes[i]:
user_news_rating_predictions[i] += user_news_array_of_train_expanded[user_neighbors_index] * user_user_similarities[user_neighbors_index, this_user_index_in_test]
similarities_sum += user_user_similarities[user_neighbors_index, this_user_index_in_test]
user_news_rating_predictions[i] /= (similarities_sum + eps)
if i % 100 == 0:
# print("%.1f%%" % (i / user_num_of_test * 100))
progress.update(i / user_num_of_test)
progress.update(1)
user_news_rating_predictions[user_news_array_of_train_expanded[[user_index_from_test_to_train_dict[i] for i in range(user_num_of_test)], :] == 1] = 0 # remove news one user has clicked
time_end = time.time()
print("Predict ratings ended. %f s cost." % (time_end - time_start))
print("[NMF-User-based Collaborative Filtering] Recommend finished!")
return user_news_rating_predictions, user_ids_of_test, news_ids_of_test
#selector = ROOT.TTreeFormula("selector", str(tcut), chain)
for i, entry in enumerate(chain):
filename = entry.GetFile().GetName()
if filename != oldfilename:
#selector.UpdateFormulaLeaves()
#if selector.EvalInstance() == False: continue
oldfilename = filename
if "LT" in filename:
h2key = pattern.match(filename).group(1)
if h2key not in lt_h2:
lt_h2[h2key] = ROOT.TH2F(args.name + h2key,
args.name + ' ' + h2key, ncat, 0,
ncat, ncat, 0, ncat)
else:
h2key = None
progress.update(100. * i / chain.GetEntries())
if entry.smearerCat[0] >= 0:
x = cat1[entry.smearerCat[0]]
y = cat2[entry.smearerCat[0]]
h2.Fill(x, y)
if h2key in lt_h2:
lt_h2[h2key].Fill(x, y)
progress.end()
h2.Draw("colz")
c.SetLogz()
c.SaveAs("plots/smearerCat_" + h2.GetName() + "_" + region_name + ".png")
for key in lt_h2:
lt_h2[key].Draw("colz")
c.SaveAs("plots/smearerCat_" + lt_h2[key].GetName() + "_" + region_name +
".png")
