def data_extract():
final_data = []
roles = parser(roles_filename)
sites = parser(job_sites_filename)
for site in sites.keys():
url = sites[site]["url"]
item = sites[site]["item"]
for role in roles:
items = extract_xml_data(url, role, item)
final_data.append(items)
return final_data
def runQuery():
graphdict = {}
client = pymongo.MongoClient(
"mongodb+srv://gurpreet:[email protected]/myFirstDatabase?retryWrites=true&w=majority"
)
db = client["mydatabase"]
doc = db["historic"]
dic = parser('./technologies.json')
lang = [[l, doc.find({
"sector": l.lower()
}).count()] for l in dic['languages']]
db = [[l, doc.find({
"sector": l.lower()
}).count()] for l in dic['Databases']]
web = [[l, doc.find({
"sector": l.lower()
}).count()] for l in dic['Web Technologies']]
devops = [[l, doc.find({
"sector": l.lower()
}).count()] for l in dic['DevOps Tools']]
contract = doc.find({"sector": "contract"}).count()
fulltime = doc.find({"sector": {'$all': ['full', 'time']}}).count()
graphdict['lang'] = lang
graphdict['db'] = db
graphdict['web'] = web
graphdict['devops'] = devops
graphdict['contract'] = contract
graphdict['fulltime'] = fulltime
return (graphdict)
def query():
client = pymongo.MongoClient("mongodb+srv://gurpreet:[email protected]/myFirstDatabase?retryWrites=true&w=majority")
db = client["mydatabase"]
doc = db["historic"]
dic = parser('../technologies.json')
lang = [ [l, doc.find({"sector": l.lower()}).count() ] for l in dic['languages']]
db = [ [l, doc.find({"sector": l.lower()}).count() ] for l in dic['Databases']]
web = [ [l, doc.find({"sector": l.lower()}).count() ] for l in dic['Web Technologies']]
devops = [ [l, doc.find({"sector": l.lower()}).count() ] for l in dic['DevOps Tools']]
def retrieve(self):
data_parser = parser()
data = data_parser.parse()
text_scraper = scraper()
final_text = text_scraper.text_scraper()
locations = []
for info in data:
if info["Value"].upper() in final_text:
if info["Value"].upper not in locations:
locations.append(info)
return locations
import sys
from utils import clear, generate_project, write, set_packages, get_file_loc
from json_parser import parser
from basic_templates import *
CLASSES = {
} # key = class name, value = list of tupples (each tupple = each attribute of class), n = 8
ENUMS = [] # list of tupples (enum_name, list of enum values), n = 2
SETTINGS = {} # key = setting, value = value
if __name__ == "__main__":
clear()
SETTINGS, ENUMS, CLASSES = parser(sys.argv[1])
lombok = SETTINGS["lombok"]
root_package = SETTINGS["rootPackage"]
root_package_path = root_package.replace(".", "/")
collection = SETTINGS["serviceCollection"].split(":")[0]
collection_impl = SETTINGS["serviceCollection"].split(":")[1]
pagination = SETTINGS["pagination"]
set_packages(root_package_path)
generate_project()
write(get_file_loc("mapper", "Mapper"),
mapper_interface_template(root_package, collection, pagination))
if (pagination):
write(get_file_loc("dto", "PageDTO"),
page_dto_template(root_package, lombok))
for enum in ENUMS:
print(" 1. TRAIN ")
print(" 2. CLASSIFY ")
print(" 0. EXIT ")
menu_choice = input(" Enter an input: ")
if menu_choice == '1':
print(" Enter the path for dataset (ex. format = ./dataset )")
dataset_path = input(" Enter an input: ")
print("\n # Processing...")
parser(dataset_path)
print("\n # output.csv created\n")
print(" # Training...")
x = dataset_path + '/output.csv'
func(x)
print("\n # Train finished\n")
# python3 train.py ./data/consumer_complaints.csv.zip ./parameters.json
elif menu_choice == '2':
print(" ------------ ------------")
print(" Choose classification method (default SVC)\n")
print(" 1. Multinomial Naive Bayes")
print(" 2. Linear Support Vector Clusters")
print(" 3. Support Vector Clusters with ratio \n")