def get_closest_docs(uri): #user_doc = requests.get(uri).text r = requests.get(uri) if r.status_code == 200: user_doc = r.text print("URI content length", len(user_doc)) code, _ = separate_code_and_comments(user_doc, "user doc") normalized_code = normalize_text(code, remove_stop_words=False, only_letters=False, return_list=True) model.random.seed(0) user_vector = model.infer_vector(normalized_code) print("finding similar...") sys.stdout.flush() stored_urls = list() stored_vectors = list() for url in vectors: stored_urls.append(url) stored_vectors.append(vectors[url]) pair_sims = cosine_similarity(user_vector.reshape(1, -1), stored_vectors) indices = (-pair_sims[0]).argsort()[:5] return [(stored_urls[index], round(float(pair_sims[0][index]), 2)) for index in indices] else: print("URL returned status code", r.status_code) raise ValueError('URL error')
def main(script_folder,topics,vocab_pickle_filename,model_pickle_filename,max_script_count,use_binary,n_jobs): # Retrieve existing vocabulary if vocab_pickle_filename is not None: vocab = pickle.load(open(vocab_pickle_filename, "rb")) else: logger.warning("Pickle file containing bag of words vocabulary required") quit() code_scripts_list = list() counter = 0 # Retrieve files containing Python scripts # Altair's JSON format uses the 'content' label for the script code for py_file in sorted(os.listdir(script_folder)): if counter >= max_script_count: break fullpath = os.path.join(script_folder, py_file) with open(fullpath, "r") as py_file_contents: for line in py_file_contents: counter += 1 parsed_json = json.loads(line) code, comments = separate_code_and_comments(parsed_json['content'],py_file) if len(code) == 0: continue else: normalized_code = normalize_text(code, remove_stop_words=True, only_letters=False, return_list=False, remove_one_char_words=True) code_scripts_list.append(normalized_code) lda_model = build_lda_model(code_scripts_list,topics,vocab,use_binary,n_jobs) #logger.info("Saving LDA model in a pickle file at %s" % model_pickle_filename) pickle.dump(lda_model, open(model_pickle_filename, "wb")) logger.info("LDA model pickle file saved at %s" % model_pickle_filename)
def vectorize_code(code_vectors, model, code_urls, vector_file): # code_urls is a set, make it a list to allow skipping some entries code_urls_list = list(code_urls) # start at a point close to where the previous code_vectors file left off starting_point = len(code_vectors) for i in range(starting_point, len(code_urls_list)): url = code_urls_list[i] if i > starting_point and i % 5000 == 0: intermediate_save(code_vectors, vector_file) if requests.get(url).status_code == 200: try: code = requests.get(url).text parsed_code, _ = separate_code_and_comments(code, "code") normalized_code = normalize_text(parsed_code, remove_stop_words=False, only_letters=False, return_list=True) if len(normalized_code) > 1: model.random.seed(0) vector = model.infer_vector(normalized_code) code_vectors[url] = vector else: print("Parsing resulted in empty list for", url) continue except: print("Unexpected error:", sys.exc_info()[0]) continue else: print("Error code {} for url: {}".format( requests.get(url).status_code, url)) return code_vectors
def build_bow_script_vocabulary(script_folder, max_script_count=10000, max_vocab_size=5000, min_word_count=2): ''' Generates a dictionary of words to be used as the vocabulary in techniques that utilize bag of words. Args: script_folder (str): Folder location of corpus containing script files max_script_count (int): the maximum number of code scripts to process in the script_folder max_vocab_size (int): the maximum number of words to be used in the vocabulary (dimension of bag of words vector) min_word_count (int): a word will be included in vocabulary if it appears at least min_count times in the corpus Returns: words_ordered_by_count (list): a list of size equal or less than vocab_size that contains the most frequent normalized words in the corpus ''' word_count = defaultdict(int) counter = 0 # Read file contents, extract code, normalize contents and count resulting tokens # Altair's JSON format uses the 'content' label for the script code for py_file in sorted(os.listdir(script_folder)): if counter >= max_script_count: break fullpath = os.path.join(script_folder, py_file) with open(fullpath, "r") as py_file_contents: for line in py_file_contents: counter += 1 parsed_json = json.loads(line) code, comments = separate_code_and_comments( parsed_json['content'], py_file) normalized_script = normalize_text(code, remove_stop_words=True, only_letters=False, return_list=True, remove_one_char_words=True) for token in normalized_script: word_count[token] += 1 if counter >= max_script_count: break # Determine descending order for library based on count and restricted by min_count threshold words_ordered_by_count = [ i[0] for i in sorted( word_count.items(), key=lambda x: (x[1], x[0]), reverse=True) if i[1] > min_word_count ] # Trim the vocabulary to the requested vocab_size if len(words_ordered_by_count) >= max_vocab_size: words_ordered_by_count = words_ordered_by_count[:max_vocab_size] else: logger.warning("Only %d words were observed using max_script_count=%d, max_vocab_size=%d and min_word_count=%d" % \ (len(words_ordered_by_count),max_script_count, max_vocab_size,min_word_count)) return words_ordered_by_count
def main(script_folder, output_folder, min_script_len, max_total_files, max_per_pkl): doc2vec_tagged_documents = list() counter = 0 logger.info("retrieving files") just_started = True # Retrieve files containing Python scripts # Altair's JSON format uses the 'content' label for the script code for py_file in sorted(os.listdir(script_folder)): if counter >= max_total_files: break fullpath = os.path.join(script_folder, py_file) with open(fullpath, "r") as py_file_contents: for line in py_file_contents: if counter >= max_total_files: break if counter != 0 and counter % 50000 == 0: logger.info("processed %d files" % counter) if not just_started and counter % max_per_pkl == 0: logger.info( "Saving pickle file of tagged documents for size %d", max_per_pkl) pickle.dump( doc2vec_tagged_documents, open( os.path.join(output_folder, "training" + str(counter) + ".pkl"), "wb")) doc2vec_tagged_documents = list() just_started = True parsed_json = json.loads(line) code, _ = separate_code_and_comments(parsed_json['content'], py_file) if len(code) < min_script_len: continue else: tokenized_code = normalize_text(code, remove_stop_words=False, only_letters=False, return_list=True, remove_one_char_words=True) if len(tokenized_code) > 1: doc2vec_tagged_documents.append( doc2vec.TaggedDocument(tokenized_code, [counter])) counter += 1 just_started = False logger.info("Saving final pickle file of tagged documents for size %d", max_per_pkl) pickle.dump( doc2vec_tagged_documents, open(os.path.join(output_folder, "training" + str(counter) + ".pkl"), "wb"))
def main(script_folder, model_pickle_filename, training_algorithm, num_cores, epochs, vector_size, window, min_count, alpha, max_script_count, min_script_len, negative): doc2vec_tagged_documents = list() counter = 0 logger.info("retrieving files") # Retrieve files containing Python scripts # Altair's JSON format uses the 'content' label for the script code for py_file in sorted(os.listdir(script_folder)): if counter >= max_script_count: break if counter % 100000 == 0: logger.info("processed %d files" % counter) fullpath = os.path.join(script_folder, py_file) with open(fullpath, "r") as py_file_contents: for line in py_file_contents: parsed_json = json.loads(line) code, comments = separate_code_and_comments( parsed_json['content'], py_file) if len(code) < min_script_len: continue else: tokenized_code = normalize_text(code, remove_stop_words=False, only_letters=False, return_list=True, remove_one_char_words=True) doc2vec_tagged_documents.append( doc2vec.TaggedDocument(tokenized_code, [counter])) counter += 1 doc2vec_model = build_doc2vec_model(doc2vec_tagged_documents, training_algorithm, num_cores, epochs, vector_size, window, min_count, alpha, negative) # Per http://radimrehurek.com/gensim/models/doc2vec.html, delete_temporary_training_data reduces model size # If keep_doctags_vectors is set to false, most_similar, similarity, sims is no longer available # If keep_inference is set to false, infer_vector on a new document is no longer possible doc2vec_model.delete_temporary_training_data(keep_doctags_vectors=False, keep_inference=True) # Per http://radimrehurek.com/gensim/models/doc2vec.html, doc2vec has its own method for saving/loading models # doc2vec_model.save(model_pickle_filename) # doc2vec_model = doc2vec.Doc2Vec.load(model_pickle_filename) #logger.info("saving doc2vec model in a pickle file at %s" % model_pickle_filename) pickle.dump(doc2vec_model, open(model_pickle_filename, "wb")) logger.info("doc2vec model pickle file saved at %s" % model_pickle_filename)
def vectorize_code(current_script_fullpath, model, remove_comments): with open(current_script_fullpath, "r") as input: code = input.read() if remove_comments: parsed_code, _ = separate_code_and_comments(code, "code") else: parsed_code = code normalized_code = normalize_text(parsed_code, remove_stop_words=False, only_letters=False, return_list=True) if len(normalized_code) > 1: model.random.seed(0) return model.infer_vector(normalized_code) else: print("Warning - Parsing resulted in empty list for", current_script_fullpath) return None
def get_closest_docs(uri): user_doc = requests.get(uri).text code, _ = separate_code_and_comments(user_doc, "user doc") normalized_code = normalize_text(code, remove_stop_words=False, only_letters=False, return_list=True) model.random.seed(0) user_vector = model.infer_vector(normalized_code) print("finding similar...") stored_urls = list() stored_vectors = list() for url in vectors: stored_urls.append(url) stored_vectors.append(vectors[url]) pair_sims = cosine_similarity(user_vector.reshape(1, -1), stored_vectors) indices = (-pair_sims[0]).argsort()[:5] return [(stored_urls[index], round(float(pair_sims[0][index]), 2)) for index in indices]
def main(data_path, num_cores, top_n_param, vectorizer): global raw global features global q # Patch for error thrown by score_performance on declaration of top_n global top_n top_n = top_n_param raw = read_data(data_path) """ # Remove items where competition IDs are in: # PyCon2015 Tutorial (#4353) # Word2Vec NLP Tutorial (#3971) filter_comp_ids = ["4353", "3971"] idxs_to_remove = set() for idx, r in enumerate(raw): if r["CompetitionId"] in filter_comp_ids: idxs_to_remove.add(idx) raw = [r for idx, r in enumerate(raw) if idx not in idxs_to_remove] """ """ # Take a random sample from raw. import random raw = random.sample(raw, 2000) """ # Strip out comments and add to scripts if it has code; otherwise remove it from raw list scripts = list() for index, script in list(enumerate(raw)): code, _ = separate_code_and_comments(script["ScriptContent"], script["ScriptTitle"]) if len(code) > 0: scripts.append(code) else: raw.pop(index) #scripts = [script["ScriptContent"] for script in raw] # Choose vectorizer print("Vectorizing documents...") #vectorizer.vectorizer.fit(scripts) features = vectorizer.vectorize_multi(scripts) features_dense = features.todense() if issparse(features) else features p = Pool(num_cores, q_init, [q]) print("Calculating pairwise similarities + scores...") for _ in tqdm.tqdm(p.imap_unordered(score_performance, list(enumerate(features_dense))), total=len(features_dense)): pass score_top_1 = 0 score_top_n_any = 0 score_top_n_all = 0 while not q.empty(): top_1, top_n_any, top_n_all = q.get() score_top_1 += top_1 score_top_n_any += top_n_any score_top_n_all += top_n_all top_1_accuracy = score_top_1 / float(len(raw)) top_n_any_accuracy = score_top_n_any / float(len(raw)) top_n_all_accuracy = score_top_n_all / float(len(raw)) print("Top 1: %s" % top_1_accuracy) print("Top N (Any): %s" % top_n_any_accuracy) print("Top N (All): %s" % top_n_all_accuracy) print("(N = %s)" % top_n) return { "top_1_accuracy": top_1_accuracy, "top_n_any_accuracy": top_n_any_accuracy, "top_n_all_accuracy": top_n_all_accuracy, "top_n": top_n }
def main(args): # Open input and output files csv_file = open(args.input_file, 'r') json_file = open(args.output_file, 'w') # Check if user wants to customize csv field order if not args.field_order_file: field_names = ["ScriptProjectId","ScriptVersionId","AuthorUserId","UserDisplayName","CompetitionId","CompetitionName","ScriptTitle","ScriptContent"] else: with open(args.field_order_file, 'r') as _order_file: field_order_reader = csv.reader(order_file) for row in field_order_reader: field_names = row continue # Address csv error regarding fields that exceed default size limit # Adapted from Stack Overflow post by user1251007 maxInt = sys.maxsize decrement = True while decrement: # decrease the maxInt value by factor 10 # as long as the OverflowError occurs. decrement = False try: csv.field_size_limit(maxInt) except OverflowError: maxInt = int(maxInt/10) decrement = True # Read CSV and Write the JSON to output_file after doing some preprocessing reader = csv.DictReader(csv_file, field_names) files = 0 parsed_competitions = defaultdict(list) logger.info("Processing csv file...") for row in reader: files+=1 # Remove very short scripts based on command line arguments script_len = len(separate_code_and_comments(row['ScriptContent'],row['ScriptTitle'])[0]) if script_len<args.min_script_len: continue # Remove meta kaggle scripts labeled as python that are probably R if row['ScriptContent'].find("<-")!=-1 and row['ScriptContent'].find("library(")!=-1: continue # Remove Kaggle competition name from the script content to allow model testing on competitions if 'CompetitionName' in row and 'ScriptContent' in row: row['ScriptContent'].replace(row['CompetitionName']," ") row['ScriptContent'].replace(row['CompetitionName'].lower(), " ") parsed_competitions[row['CompetitionId']].append(row) submissions_deduped = list() logger.info("Removing duplicates...") # Iterate over competitions to remove duplicates and near duplicates for competition in parsed_competitions: counter = 0 submissions = parsed_competitions[competition] # Pair-wise SequenceMatcher comparison of ScriptContent for i in range(len(submissions)): for j in range(len(submissions)): if i!=j and SequenceMatcher(None, submissions[i]['ScriptContent'].lower(), \ submissions[j]['ScriptContent'].lower()).ratio() > args.duplicate_threshold: submissions[i]['ScriptContent'] = "" counter+=1 break remove_empties = [x for x in submissions if x['ScriptContent']!=""] logger.info("%d duplicates removed from %d submissions in competition %s" % (counter,len(submissions),competition)) # Ensure competition has at least ten entries for future comparison if len(remove_empties)>=10: for item in remove_empties: submissions_deduped.append(item) else: logger.warning("Competition %s has too few remaining submissions at threshold %f" % (competition,args.duplicate_threshold)) # Build a custom namedtuple to integrate into pyminifer argparse command line methods if args.minimize or args.obfuscate: options_tuple = namedtuple("options_tuple", ["tabs", "minimize", "obfuscate", "replacement_length"]) options = options_tuple(False, args.minimize, args.obfuscate, 1) errors = 0 written = 0 for row in submissions_deduped: # Minimize size of python script if set in args if args.minimize or args.obfuscate: try: tokens = token_utils.listified_tokenizer(row['ScriptContent']) source = minification.minify(tokens,options) tokens = token_utils.listified_tokenizer(source) # Obsfuscate python script if args.obfuscate: table = [{}] module = row['ScriptTitle'] name_generator = obfuscate.obfuscation_machine(identifier_length=int(options.replacement_length)) obfuscate.obfuscate(module, tokens, options, name_generator=name_generator, table=table) # Convert back to text result = '' result += token_utils.untokenize(tokens) row['ScriptContent'] = result except Exception as e: # logger.info("%s in %s; continuing" % (e.__class__.__name__,row['ScriptTitle'])) errors+=1 continue written+=1 json.dump(row, json_file) json_file.write('\n') logger.info("Total files reviewed: %d" % files) if args.minimize or args.obfuscate: logger.info("File that failed pyminifier minimization/obfuscation parsing: %d" % errors) logger.info("Files successfully parsed to json: %d" % written) csv_file.close() json_file.close()