def find_user_by_id(user_id): ''' (str) --> dict Returns a user's info given their id ''' try: db = database() if type(user_id) != str: raise TypeError data = db.users.find_one({"_id": user_id}) if data: return data else: raise ValueError except TypeError: raise TypeError( bcolors.FAIL + "Invalid Input. Enter a valid user id as a string to use this function" + bcolors.ENDC) return False except ValueError: raise ValueError(bcolors.FAIL + "Invalid User ID" + bcolors.ENDC) return False
def find_user_by_email(email): ''' (str) --> dict Returns a user's info given their email ''' try: db = database() data = [] if type(email) != str: raise TypeError data = db.users.find_one({"emails.address": email}) if data: return data else: raise ValueError except TypeError: raise TypeError( bcolors.FAIL + "Invalid Input. Enter a valid email as a string to use this function" + bcolors.ENDC) return False except ValueError: raise ValueError(bcolors.FAIL + "No user found with that email" + bcolors.ENDC) return False
def find_user_by_name(name): ''' (str) --> list Returns a list of users info given a name. ''' try: start_time = time.time() db = database() data = [] if type(name) != str: raise TypeError name = name.lower() for i in db.users.find(): data.append(i) #seperates string into array of words temp_name = re.findall(r'\w+', name) user_list = [] #changes search parameters based on number of words in search if len(temp_name) == 1: #searches for name in the first and last names of people in the profile for i in range(len(data)): temp_profile = data[i].get("profile") if name in temp_profile.get("firstName").lower() or name in temp_profile.get("lastName").lower() : user_list = user_list + [data[i]] elif len(temp_name) == 2: #searches for the full name with exact match temp_firstName = temp_name[0] temp_lastName = temp_name[1] for i in range(len(data)): temp_profile = data[i].get("profile") if temp_firstName in temp_profile.get("firstName").lower() and temp_lastName in temp_profile.get("lastName").lower(): user_list = user_list + [data[i]] elif len(temp_name) > 2: #if more than three words are in name search, checks each word against first and last name for i in range(len(data)): for c in range(len(temp_name)): temp_profile = data[i].get("profile") if temp_name[c] in temp_profile.get("firstName").lower() or temp_name[c] in temp_profile.get("lastName").lower(): user_list = user_list + [data[i]] if len(user_list) == 0: raise ValueError for i in range(len(user_list)): print user_list[i] print "" return user_list except TypeError: raise TypeError(bcolors.FAIL + "Invalid Input. Enter a valid name as a string to use this function" + bcolors.ENDC) return False except ValueError: raise ValueError(bcolors.FAIL + "Name not found in database" + bcolors.ENDC)
def parse_user_site(user_id): ''' (str) --> bool Parses a user's site given a user id. ''' try: start_time = time.time() db = database() if type(user_id) != str and type(user_id) != unicode: raise TypeError user_id = str(user_id) user = db.users.find_one({"_id": user_id}) if user: url_temp = user.get("profile").get("url") else: raise ValueError print bcolors.OKGREEN + ("parsing through: " + url_temp) + bcolors.ENDC user_id = user.get("_id") title = get_title(url_temp) last_parsed = time.strftime("%c") db.users.update({"profile.url": url_temp}, { "$set": { "profile.title": title, "profile.lastParsed": last_parsed } }, False, True) # there will be 2 types of tags, # from website, and from pdf tags_temp = get_html(url_temp) # this is all words from the html tagsPDF_temp = get_pdf(url_temp) # this is all words from the pdf keyword_list = [] for keyword in tags_temp: try: print bcolors.OKGREEN + ( "Incremented: %s to %d" % (keyword, increment_word(keyword))) + bcolors.ENDC keyword_list.append(keyword) except Exception, e: print bcolors.FAIL + "Invalid Entry" + bcolors.ENDC print e for keyword in tagsPDF_temp: try: if keyword not in keyword_list: print bcolors.OKGREEN + ( "Incremented: %s to %d" % (keyword, increment_word(keyword))) + bcolors.ENDC except Exception, e: print bcolors.FAIL + "Invalid Entry" + bcolors.ENDC print e
def count_total_words(): """ () --> integer Counts the total number of words in the database that have been counted more than once, and returns an integer value. """ db = database() counter = 0 for i in db.word_count.find({"total" : {'$gt' : 1} }): counter += i.get("total") return counter
def count_distinct_words(): """ () --> integer Counts the number of distinct words in the database which have appeared more than once, and returns an integer value. """ db = database() counter = 0 for i in db.word_count.find({"total" : {'$gt' : 1} }): counter += 1 return counter
def calculate_keywords(): """ () --> list Returns a list of all the keywords in the database which are less than 0.8414 standard deviations above the mean (bottom 80%) """ db = database() data = [] for i in db.word_count.find(): data.append(i) sorted_list = [] avg_val = _average_count() std_dev = _std_count() num_list = [] # creates a list with all the total values for i in range(len(data)): num_list = num_list + [data[i].get("total")] #print num_list #creates a list that lists the number of standard deviations from the mean each index is std_list = [] for i in num_list: std_list.append((avg_val-i)/std_dev) #filters out any values greater than 0.8416 standard deviations above the mean from num_list for i in range(len(std_list)): if std_list[i] < -0.8416: #assigns 0 value to values out of range num_list[i] = 0 sorted_list = [] highest_val = max(num_list) word_list = [] #adds the highest name/value pair from num_list to sorted list then removes it from num_list while highest_val != 0: for i in range(len(data)): if data[i].get("total") == highest_val and data[i].get("word") not in word_list and num_list[i] != 0: sorted_list.append([data[i].get("word"),data[i].get("total")]) word_list.append(data[i].get("word")) num_list[i] = 0 highest_val = max(num_list) print sorted_list return sorted_list
def get_all_urls(): """ () --> list Returns a list of all the urls users have submitted. """ db = database() url_list = [] data = [] for i in db.users.find(): if i.get("profile").get("url"): url_list.append(i.get("profile").get("url")) return url_list
def get_all_urls(): """ () --> list Returns a list of all the urls users have submitted. """ db = database() url_list = [] data = [] for i in db.users.find(): if i.get("profile").get("url"): url_list.append(i.get("profile").get("url")) return url_list
def parse_user_site(user_id): ''' (str) --> bool Parses a user's site given a user id. ''' try: start_time = time.time() db = database() if type(user_id) != str and type(user_id) != unicode: raise TypeError user_id = str(user_id) user = db.users.find_one({"_id" : user_id}) if user: url_temp = user.get("profile").get("url") else: raise ValueError print bcolors.OKGREEN + ("parsing through: " + url_temp) + bcolors.ENDC user_id = user.get("_id") title = get_title(url_temp) last_parsed = time.strftime("%c") db.users.update({"profile.url" : url_temp}, { "$set": {"profile.title" : title, "profile.lastParsed" : last_parsed}},False,True) # there will be 2 types of tags, # from website, and from pdf tags_temp = get_html(url_temp) # this is all words from the html tagsPDF_temp = get_pdf(url_temp) # this is all words from the pdf keyword_list = [] for keyword in tags_temp: try: print bcolors.OKGREEN + ("Incremented: %s to %d" % (keyword, increment_word(keyword))) + bcolors.ENDC keyword_list.append(keyword) except Exception, e: print bcolors.FAIL + "Invalid Entry" + bcolors.ENDC print e for keyword in tagsPDF_temp: try: if keyword not in keyword_list: print bcolors.OKGREEN + ("Incremented: %s to %d" % (keyword, increment_word(keyword))) + bcolors.ENDC except Exception, e: print bcolors.FAIL + "Invalid Entry" + bcolors.ENDC print e
def _std_count(): """ () --> integer Calculates the standard deviation of the number of repititions a words has in the database and returns an integer value. """ db = database() data = [] count_list = [] for i in db.word_count.find(): if i.get("total"): count_list.append(i.get("total")) print "Standard Deviation: " + str(numpy.std(count_list)) return numpy.std(count_list)
def _insert_word(word): """ (string) --> boolean Inserts a word into the database with a count of zero, and returns true, unless it already exists, in which case it returns false. """ db = database() data = db.word_count.find_one({"word" : word}) if data: raise LookupError("Word: %s already exists in database" % word) else: db.word_count.insert({"word": word, "total": 0}) return True
def delete_all_keywords(): ''' () --> bool Empties the keyword database for all users. ''' try: db = database() #deletes existing data db.keywords_coll.drop() db.word_count.drop() print "All Entries Deleted" return True except Exception as e: print e return False
def delete_all_keywords(): ''' () --> bool Empties the keyword database for all users. ''' try: db = database() #deletes existing data db.keywords_coll.drop() db.word_count.drop() print "All Entries Deleted" return True except Exception as e: print e return False
def parse_all_users(): ''' () --> bool Parse the sites of all users in database. ''' try: db = database() #parses through every users' site for i in db.users.find(): try: parse_user_site(i.get("_id")) except Exception as e: print e pass return True except Exception as e: print e return False
def parse_all_users(): ''' () --> bool Parse the sites of all users in database. ''' try: db = database() #parses through every users' site for i in db.users.find(): try: parse_user_site(i.get("_id")) except Exception as e: print e pass return True except Exception as e: print e return False
def increment_word(word): """ (string) --> integer Adds one to the total number of a word unless it doesn't exist in the database, in which case it adds it, with an initial count of 1, returning an integer of it's count. """ db = database() word = str(word) data = db.word_count.find_one({"word" : word}) if not data: _insert_word(word) data = db.word_count.find_one({"word" : word}) if data: count = data.get("total") + 1 db.word_count.update({ "word": word}, {"word": word, "total": count}) return count
def delete_user_keywords(user_id): ''' (str) --> bool Deletes all the keywords of a user, given an id. ''' try: db = database() if type(user_id) != str and type(user_id) != unicode: raise TypeError data = db.users.find_one({"_id": user_id}) if data: db.keywords_coll.delete_many({"user_id": user_id}) else: raise ValueError print "User Entries Deleted" return True except TypeError, e: print e raise TypeError(bcolors.FAIL + "Invalid ID type" + bcolors.ENDC) return False
def find_user_by_id(user_id): ''' (str) --> dict Returns a user's info given their id ''' try: db = database() if type(user_id) != str: raise TypeError data = db.users.find_one({"_id" : user_id}) if data: return data else: raise ValueError except TypeError: raise TypeError(bcolors.FAIL + "Invalid Input. Enter a valid user id as a string to use this function" + bcolors.ENDC) return False except ValueError: raise ValueError(bcolors.FAIL + "Invalid User ID" + bcolors.ENDC) return False
def delete_user_keywords(user_id): ''' (str) --> bool Deletes all the keywords of a user, given an id. ''' try: db = database() if type(user_id) != str and type(user_id) != unicode: raise TypeError data = db.users.find_one({"_id" : user_id}) if data: db.keywords_coll.delete_many({"user_id": user_id}) else: raise ValueError print "User Entries Deleted" return True except TypeError, e: print e raise TypeError(bcolors.FAIL + "Invalid ID type" + bcolors.ENDC) return False
def find_user_by_email(email): ''' (str) --> dict Returns a user's info given their email ''' try: db = database() data = [] if type(email) != str: raise TypeError data = db.users.find_one({"emails.address" : email}) if data: return data else: raise ValueError except TypeError: raise TypeError(bcolors.FAIL +"Invalid Input. Enter a valid email as a string to use this function" + bcolors.ENDC) return False except ValueError: raise ValueError(bcolors.FAIL + "No user found with that email" + bcolors.ENDC) return False
def _find_skills(bodyStr): """ (str) --> array Searches through the html and pdf for skills in a txt file. """ try: if type(bodyStr) != str and type(bodyStr) != unicode: raise TypeError except TypeError: raise TypeError(bcolors.FAIL + "bodyStr must be a string" + bcolors.ENDC) db = database() skill_words = [] # find words that are matching with our skills vocabulary data = [] lstSkills = [] for i in db.skill_coll.find(): data = data + [i] for j in range(len(data)): lstSkills = lstSkills + data[j].get("skill") counter = True # we will first check if the person has "skill" defined, # and if "skill" is not found, then try to collect all the vocabularies appearing in the body. if ("skill" in bodyStr): skill_tag = bodyStr[bodyStr.find("skill"):] # if "skill" was defined, then it will collect vocabularies after the definition for i in lstSkills: temp_i = i temp_i = temp_i.replace('\n', "") temp_i = temp_i.replace('\r', "") temp_i = str(temp_i) # fix the format as usual if temp_i.lower() in skill_tag: skill_words = skill_words + [temp_i] # this is special case, when person has C and/or R skills. # those skills (one worded) can be found by adding spaces # eg. "c" -> " c " # Did not add "GO" language skill due to huge problem. if " c " in skill_tag and counter == True: skill_words = skill_words + ["C"] counter = False if " r " in skill_tag and counter == True: skill_words = skill_words + ["R"] counter = False else: () else: # it will do the same, but for entire bodyStr, which will take longer time. for i in lstSkills: temp_i = i temp_i = temp_i.replace('\n', "") temp_i = temp_i.replace('\r', "") temp_i = str(temp_i) if temp_i.lower() in bodyStr: skill_words = skill_words + [temp_i] if " c " in bodyStr and counter == True: skill_words = skill_words + ["C"] counter = False if " r " in bodyStr and counter == True: skill_words = skill_words + ["R"] counter = False else: () return skill_words
def find_user_by_name(name): ''' (str) --> list Returns a list of users info given a name. ''' try: start_time = time.time() db = database() data = [] if type(name) != str: raise TypeError name = name.lower() for i in db.users.find(): data.append(i) #seperates string into array of words temp_name = re.findall(r'\w+', name) user_list = [] #changes search parameters based on number of words in search if len(temp_name) == 1: #searches for name in the first and last names of people in the profile for i in range(len(data)): temp_profile = data[i].get("profile") if name in temp_profile.get("firstName").lower( ) or name in temp_profile.get("lastName").lower(): user_list = user_list + [data[i]] elif len(temp_name) == 2: #searches for the full name with exact match temp_firstName = temp_name[0] temp_lastName = temp_name[1] for i in range(len(data)): temp_profile = data[i].get("profile") if temp_firstName in temp_profile.get("firstName").lower( ) and temp_lastName in temp_profile.get("lastName").lower(): user_list = user_list + [data[i]] elif len(temp_name) > 2: #if more than three words are in name search, checks each word against first and last name for i in range(len(data)): for c in range(len(temp_name)): temp_profile = data[i].get("profile") if temp_name[c] in temp_profile.get("firstName").lower( ) or temp_name[c] in temp_profile.get("lastName").lower(): user_list = user_list + [data[i]] if len(user_list) == 0: raise ValueError for i in range(len(user_list)): print user_list[i] print "" return user_list except TypeError: raise TypeError( bcolors.FAIL + "Invalid Input. Enter a valid name as a string to use this function" + bcolors.ENDC) return False except ValueError: raise ValueError(bcolors.FAIL + "Name not found in database" + bcolors.ENDC)