def getAlreadyExaminedUsers():
allUsersExamined = []
i = 0
for og_tweet_collection in db.collection_names():
if og_tweet_collection != "users_info":
try:
tweet = api.get_status(og_tweet_collection)
userID = str(tweet.user.id)
if userID not in allUsersExamined:
allUsersExamined.append(userID)
except:
collection = db[og_tweet_collection]
cursor = collection.find({})
if cursor.count() == 0:
print("the folowing tweet id has a problem",
og_tweet_collection)
i = i + 1
else:
retweet = cursor.next()
jsonDumpsTweet = bsondumps(retweet)
jsonDumpsTweet = loads(jsonDumpsTweet)
userID = str(
jsonDumpsTweet["retweeted_status"]["user"]["id"])
if userID not in allUsersExamined:
allUsersExamined.append(userID)
pass
print "In total,", i, "tweets where not available now."
print "There have been examined", len(allUsersExamined), " users already."
return allUsersExamined
def create_user(self, data, ):
if 'email' in data:
alias = data.get('name','')
email = data.get('email','')
if 'plan_name' in data:
plan_name = data['plan_name']
u = User.create_user(alias=alias, email=email)
else:
u = {'error': 'Email has not been supplied'}
msg = self.tpl
if 'error' in u:
msg['status'] = 'error'
msg['errormessage'] = u['error']
else:
u = User.get_user(profile_id=u['profile_id'])
msg['status'] = 'ok'
msg['payload'] = {'name': u['alias']}
self._send_confirmation_email(secret_key=u.get('secret_key',''),alias=alias,email=email)
return HttpResponse( bsondumps(msg), content_type='application/json' )
def get_new():
global IDEAS
if not IDEAS:
IDEAS = [x for x in db.hackathon_ideas.find({})]
index = random.randint(0, len(IDEAS)-1)
chosenIdea = IDEAS.pop(index)
RECENT.append(chosenIdea)
return bsondumps(chosenIdea)
def check_email(self, params, *args, **kwargs): email = params['email'] res = User.is_email_occupied(email) msg = self.tpl msg['payload']['email_occupied'] = res msg['status'] = 'ok' return HttpResponse( bsondumps(msg), content_type='application/json' )
def get_user_plan(self, params, *args, **kwargs): uid = params['userid'] msg = self.tpl u = User.get_user(profile_id=uid) if 'error'in u: msg['status'] = 'error' msg['errormessage'] = u['error'] return HttpResponse( bsondumps(msg), content_type='application/json' ) mongo_user = u plan = User(u).get_user_plan() msg['status'] = 'ok' msg['payload']['user'] = mongo_user msg['payload']['plan'] = plan return HttpResponse( bsondumps(msg), content_type='application/json' )
def get_user_and_plan(self, access_data, *args, **kwargs):
access_token = access_data.get('access_token', '')
msg = self.tpl
if not access_token:
msg['status'] = 'error'
msg['errormessage'] = 'Empty access token'
return HttpResponse( bsondumps(msg), content_type='application/json' )
url = 'https://graph.facebook.com/me?access_token={access_token}&fields=id,email,gender,link,locale,name,timezone,updated_time,verified'
url = url.format(access_token=access_token)
try:
response = urllib2.urlopen(url)
except urllib2.HTTPError:
fb_user={'error': {'message': 'Wrong api token'}}
except urllib2.URLError as e:
fb_user = {'error':{'message': 'Could not connect to Facebook'}}
else:
# All is fine
res = response.read()
fb_user = json.loads(res)
if 'error' in fb_user:
msg['status'] = 'error'
msg['errormessage'] = fb_user['error']['message']
return HttpResponse( bsondumps(msg), content_type='application/json' )
u = User.get_user(facebook_user=fb_user)
if 'error' in u:
msg['status'] = 'error'
msg['errormessage'] = u['error']
return HttpResponse( bsondumps(msg), content_type='application/json' )
mongo_user = u
plan = User(mongo_user).get_user_plan()
msg['status'] = 'ok'
msg['payload']['user'] = mongo_user
msg['payload']['plan'] = plan
return HttpResponse( bsondumps(msg), content_type='application/json' )
def get_user_plan(self, params, *args, **kwargs):
secret_key = params.get('secret_key','')
u = User.get_user(secret_key = secret_key)
msg = self.tpl
time.sleep(0.1)
if 'error' in u:
msg['status'] = 'error'
msg['errormessage'] = u['error']
return HttpResponse( bsondumps(msg), content_type='application/json' )
mongo_user = u
plan = User(u).get_user_plan()
msg['status'] = 'ok'
msg['payload']['user'] = mongo_user
msg['payload']['plan'] = plan
return HttpResponse( bsondumps(msg), content_type='application/json' )
def update_user(self, params, *args, **kwargs): uid = params['profile_id'] msg = self.tpl u = User.get_user(profile_id=uid) if 'error' in u: msg['status'] = 'error' msg['errormessage'] = u['error'] return HttpResponse( bsondumps(msg), content_type='application/json' ) res = User(u).delete() if res: msg['status'] = 'ok' msg['payload']['message'] = "user deleted successfully" else: msg['status'] = 'error' msg['errormessage'] = 'unable to delete the user' return HttpResponse( bsondumps(msg), content_type='application/json' )
def update_user(self, params, *args, **kwargs): uid = params['userid'] data = kwargs['data'] msg = self.tpl u = User.get_user(profile_id=uid) if 'error' in u: msg['status'] = 'error' msg['errormessage'] = u['error'] return HttpResponse( bsondumps(msg), content_type='application/json' ) res = User(u).update_info(data=data) if 'error' not in res: msg['status'] = 'ok' msg['payload']['user'] = u else: msg['status'] = 'error' msg['errormessage'] = res['error'] return HttpResponse( bsondumps(msg), content_type='application/json' )
def make_user_dict(this_db, dictionary):
user_dict = {}
for collection in dictionary:
try:
tweet = api.get_status(collection)
userID = str(tweet.user.id)
except:
col = this_db[collection]
cursor = col.find({})
if cursor.count() != 0:
retweet = cursor.next()
jsonDumpsTweet = bsondumps(retweet)
jsonDumpsTweet = loads(jsonDumpsTweet)
userID = str(jsonDumpsTweet["retweeted_status"]["user"]["id"])
if userID not in user_dict:
user_dict[userID] = 1
else:
value = user_dict.get(userID)
user_dict[userID] = int(value) + 1
return user_dict
def like_idea(oid):
document = db.hackathon_ideas.find_one({"_id": ObjectId(oid)})
document["num_likes"] = document["num_likes"] + 1
return bsondumps(db.hackathon_ideas.save(document))
def get_results_endpoint():
page_size = request.args.get('page_size')
page_num = request.args.get('page_num')
query_str = request.args.get('query_str')
print(page_size)
return bsondumps(get_results(page_size=page_size, page_num=page_num, query_str=query_str))
def post(self):
json_args = json_decode(self.request.body)
if not ("git_hash" in json_args["_id"] and
"build_id" in json_args["_id"]):
self.write_error(422)
return
# Generate line count results
git_hash = json_args["_id"]["git_hash"]
build_id = json_args["_id"]["build_id"]
self.write(git_hash + ", " + build_id)
pipeline = [{"$match":{"build_id": build_id, "git_hash": git_hash,
"file": re.compile("^src\/mongo")}},
{"$project":{"file":1, "lc":1}}, {"$unwind":"$lc"},
{"$group":{"_id":"$file", "count":{"$sum":1},
"noexec":{"$sum":{"$cond":[{"$eq":["$lc.ec",0]},1,0]}}} }]
cursor = yield self.application.collection.aggregate(pipeline, cursor={})
total = 0
noexec_total = 0
while (yield cursor.fetch_next):
bsonobj = cursor.next_object()
obj = bsondumps(bsonobj)
total += bsonobj["count"]
noexec_total += bsonobj["noexec"]
json_args["line_count"] = total
json_args["line_cov_count"] = total-noexec_total
json_args["line_cov_percentage"] = round(float(total-noexec_total)/total * 100, 2)
# Generate function results
pipeline = [{"$project": {"file":1,"functions":1}}, {"$unwind":"$functions"},
{"$group": { "_id":"$functions.nm",
"count" : { "$sum" : "$functions.ec"}}}]
cursor = yield self.application.collection.aggregate(pipeline, cursor={})
noexec_total = 0
total = 0
while (yield cursor.fetch_next):
bsonobj = cursor.next_object()
total += 1
if bsonobj["count"] == 0:
noexec_total += 1
json_args["func_count"] = total
json_args["func_cov_count"] = total-noexec_total
json_args["func_cov_percentage"] = round(float(total-noexec_total)/total * 100, 2)
# Retrieve test name list
match = {"$match": {"build_id": build_id, "git_hash": git_hash}}
testname_pipeline = copy.copy(pipelines.testname_pipeline)
testname_pipeline.insert(0, match)
cursor = yield self.application.collection.aggregate(testname_pipeline, cursor={})
while (yield cursor.fetch_next):
bsonobj = cursor.next_object()
json_args["test_names"] = bsonobj["test_names"]
json_args["date"] = datetime.datetime.strptime(json_args["date"], "%Y-%m-%dT%H:%M:%S.%f")
# Insert meta-information
query = {"_id.build_id": build_id, "_id.git_hash": git_hash}
try:
result = yield self.application.meta_collection.update(query, json_args, upsert=True)
except tornado.httpclient.HTTPError as e:
self.write_error(422)
return
# Generate coverage data by directory
line_pipeline = copy.copy(pipelines.line_pipeline)
function_pipeline = copy.copy(pipelines.function_pipeline)
match = {"$match": {"build_id": json_args["_id"]["build_id"],
"git_hash": json_args["_id"]["git_hash"],
"file": re.compile("^src\/mongo")}}
line_pipeline.insert(0, match)
function_pipeline.insert(0, match)
cursor = yield self.application.collection.aggregate(line_pipeline, cursor={})
while (yield cursor.fetch_next):
bsonobj = cursor.next_object()
# Generate line coverage percentage
line_count = bsonobj["line_count"]
line_cov_count = bsonobj["line_cov_count"]
bsonobj["line_cov_percentage"] = round(float(line_cov_count)/line_count * 100, 2)
query = {"_id.build_id": build_id, "_id.git_hash": git_hash,
"_id.dir": bsonobj["_id"]["dir"]}
result = yield self.application.cov_collection.update(query, bsonobj, upsert=True)
cursor = yield self.application.collection.aggregate(function_pipeline, cursor={})
while (yield cursor.fetch_next):
bsonobj = cursor.next_object()
# Generate function coverage percentage
func_count = bsonobj["func_count"]
func_cov_count = bsonobj["func_cov_count"]
func_cov_percentage = round(float(func_cov_count)/func_count * 100, 2)
query = {"_id": bsonobj["_id"]}
modification = {"$set": {"func_count": bsonobj["func_count"],
"func_cov_count": bsonobj["func_cov_count"],
"func_cov_percentage": func_cov_percentage}}
result = yield self.application.cov_collection.update(query, modification)
def post(self):
json_args = json_decode(self.request.body)
if not ("git_hash" in json_args["_id"]
and "build_id" in json_args["_id"]):
self.write_error(422)
return
# Generate line count results
git_hash = json_args["_id"]["git_hash"]
build_id = json_args["_id"]["build_id"]
self.write(git_hash + ", " + build_id)
pipeline = [{
"$match": {
"build_id": build_id,
"git_hash": git_hash,
"file": re.compile("^src\/mongo")
}
}, {
"$project": {
"file": 1,
"lc": 1
}
}, {
"$unwind": "$lc"
}, {
"$group": {
"_id": "$file",
"count": {
"$sum": 1
},
"noexec": {
"$sum": {
"$cond": [{
"$eq": ["$lc.ec", 0]
}, 1, 0]
}
}
}
}]
cursor = yield self.application.collection.aggregate(pipeline,
cursor={})
total = 0
noexec_total = 0
while (yield cursor.fetch_next):
bsonobj = cursor.next_object()
obj = bsondumps(bsonobj)
total += bsonobj["count"]
noexec_total += bsonobj["noexec"]
json_args["line_count"] = total
json_args["line_cov_count"] = total - noexec_total
json_args["line_cov_percentage"] = round(
float(total - noexec_total) / total * 100, 2)
# Generate function results
pipeline = [{
"$project": {
"file": 1,
"functions": 1
}
}, {
"$unwind": "$functions"
}, {
"$group": {
"_id": "$functions.nm",
"count": {
"$sum": "$functions.ec"
}
}
}]
cursor = yield self.application.collection.aggregate(pipeline,
cursor={})
noexec_total = 0
total = 0
while (yield cursor.fetch_next):
bsonobj = cursor.next_object()
total += 1
if bsonobj["count"] == 0:
noexec_total += 1
json_args["func_count"] = total
json_args["func_cov_count"] = total - noexec_total
json_args["func_cov_percentage"] = round(
float(total - noexec_total) / total * 100, 2)
# Retrieve test name list
match = {"$match": {"build_id": build_id, "git_hash": git_hash}}
testname_pipeline = copy.copy(pipelines.testname_pipeline)
testname_pipeline.insert(0, match)
cursor = yield self.application.collection.aggregate(testname_pipeline,
cursor={})
while (yield cursor.fetch_next):
bsonobj = cursor.next_object()
json_args["test_names"] = bsonobj["test_names"]
json_args["date"] = datetime.datetime.strptime(json_args["date"],
"%Y-%m-%dT%H:%M:%S.%f")
# Insert meta-information
query = {"_id.build_id": build_id, "_id.git_hash": git_hash}
try:
result = yield self.application.meta_collection.update(query,
json_args,
upsert=True)
except tornado.httpclient.HTTPError as e:
self.write_error(422)
return
# Generate coverage data by directory
line_pipeline = copy.copy(pipelines.line_pipeline)
function_pipeline = copy.copy(pipelines.function_pipeline)
match = {
"$match": {
"build_id": json_args["_id"]["build_id"],
"git_hash": json_args["_id"]["git_hash"],
"file": re.compile("^src\/mongo")
}
}
line_pipeline.insert(0, match)
function_pipeline.insert(0, match)
cursor = yield self.application.collection.aggregate(line_pipeline,
cursor={})
while (yield cursor.fetch_next):
bsonobj = cursor.next_object()
# Generate line coverage percentage
line_count = bsonobj["line_count"]
line_cov_count = bsonobj["line_cov_count"]
bsonobj["line_cov_percentage"] = round(
float(line_cov_count) / line_count * 100, 2)
query = {
"_id.build_id": build_id,
"_id.git_hash": git_hash,
"_id.dir": bsonobj["_id"]["dir"]
}
result = yield self.application.cov_collection.update(query,
bsonobj,
upsert=True)
cursor = yield self.application.collection.aggregate(function_pipeline,
cursor={})
while (yield cursor.fetch_next):
bsonobj = cursor.next_object()
# Generate function coverage percentage
func_count = bsonobj["func_count"]
func_cov_count = bsonobj["func_cov_count"]
func_cov_percentage = round(
float(func_cov_count) / func_count * 100, 2)
query = {"_id": bsonobj["_id"]}
modification = {
"$set": {
"func_count": bsonobj["func_count"],
"func_cov_count": bsonobj["func_cov_count"],
"func_cov_percentage": func_cov_percentage
}
}
result = yield self.application.cov_collection.update(
query, modification)
def first_test(excludeZeroCollections, filename, this_db):
data = np.loadtxt(filename, dtype=str)
all_users = data[:, :]
numOfTweets = []
i = 0
for user in all_users:
numOfTweets.append(0)
i = i + 1
counter = 0
tweetchains = 0
deletedTweet = 0
notdeleted = 0
zeroRetweeets = 0
for og_tweet_collection in this_db.collection_names():
if og_tweet_collection != "users_info":
try:
collection = this_db[og_tweet_collection]
cursor = collection.find({})
numOfTweetsInTHISCollection = cursor.count()
tweet = api.get_status(og_tweet_collection)
userID = str(tweet.user.id)
i = 0
for user in all_users:
if userID == user[1]:
if excludeZeroCollections:
if numOfTweetsInTHISCollection != 0:
numOfTweets[i] = int(numOfTweets[i]) + 1
else:
zeroRetweeets = zeroRetweeets + 1
else:
numOfTweets[i] = int(numOfTweets[i]) + 1
i = i + 1
notdeleted = notdeleted + 1
except:
collection = this_db[og_tweet_collection]
cursor = collection.find({})
if cursor.count() == 0:
print("the folowing tweet id has a problem",
og_tweet_collection)
counter = counter + 1
zeroRetweeets = zeroRetweeets + 1
else:
retweet = cursor.next()
jsonDumpsTweet = bsondumps(retweet)
jsonDumpsTweet = loads(jsonDumpsTweet)
userID = str(
jsonDumpsTweet["retweeted_status"]["user"]["id"])
i = 0
for user in all_users:
if userID == user[1]:
numOfTweets[i] = int(numOfTweets[i]) + 1
i = i + 1
deletedTweet = deletedTweet + 1
pass
tweetchains = tweetchains + 1
print "_________________FINAL RESULTS_________________________________________"
i = 0
total = 0
for user in all_users:
print user, numOfTweets[i]
total = total + numOfTweets[i]
i = i + 1
print counter, "collections had a problem"
print "Tweet-chains num:", tweetchains
print "Deleted tweets num:", deletedTweet
print "notdeleted=", notdeleted
if excludeZeroCollections:
print "Tweet-chains with zero retweets:", zeroRetweeets
print "total=", total
