def __init__(self, username, data, host, port, database): self.username = username self.host = host self.port = port self.database = database self.db = Database(self.host, self.port, database=database) self.data = data
class Spotify: def __init__(self, username): scope = 'user-library-read' self.en = Echonest() self.token = util.prompt_for_user_token(username, scope) self.db = Database() def saved_tracks(self, limit, offset): if self.token: sp = spotipy.Spotify(auth=self.token) results = sp.current_user_saved_tracks(limit, offset) if len(results['items']) == 0: print("No more saved tracks :(") return results['items'] else: print("Can't get token") return [] def parse_saved_tracks(self, start, num_of_songs): offset = start # option to not start at the beginning limit = 20 # echonest rate limit :/ while offset < num_of_songs: print("Request at offset " + str(offset)) for item in self.saved_tracks(limit, offset): # get the song data using Echonest data = self.en.track_attributes(item) if data: self.db.add_row(data) offset += limit time.sleep(60) # we only have 20 requests/minute
class Spotify: def __init__(self, username): scope = 'user-library-read' self.en = Echonest() self.token = util.prompt_for_user_token(username, scope) self.db = Database() def saved_tracks(self, limit, offset): if self.token: sp = spotipy.Spotify(auth=self.token) results = sp.current_user_saved_tracks(limit, offset) if len(results['items']) == 0: print("No more saved tracks :(") return results['items'] else: print("Can't get token") return [] def parse_saved_tracks(self, start, num_of_songs): offset = start # option to not start at the beginning limit = 20 # echonest rate limit :/ while offset < num_of_songs: print("Request at offset " + str(offset)) for item in self.saved_tracks(limit, offset): # get the song data using Echonest data = self.en.track_attributes(item) if data: self.db.add_row(data) offset += limit time.sleep(60) # we only have 20 requests/minute
class MediumParser(object): def __init__(self, username, data, host, port, database): self.username = username self.host = host self.port = port self.database = database self.db = Database(self.host, self.port, database=database) self.data = data def __repr__(self): return f"Parser Object for data: {self.data}" def grab_elements(self, website): # This extracts the Tag class from html page # Note: This needs an html page, NOT a url. soup = bs4.BeautifulSoup(website, 'html.parser') children = list(soup.children) element = [ children[num] for num, value in enumerate(children) if type(children[num]) is bs4.element.Tag ] return element def grab_text(self): """ Main Metod: grab_text is responsible for grabbing the text from all the articles and storing the information into an array. """ texts = [] for website in self.data: data = requests.get(website).content.decode('utf-8') elms = self.grab_elements(website=data) text = elms[0].find_all("p") text = [''.join(text[num].text) for num, val in enumerate(text)] texts.append("".join(text)) return texts def store_information(self): """ Main Method: store_information is responsible for storing the information into the database via the Database interface """ data = self.grab_text() db = self.db.select_database(self.database) for num, val in enumerate(data): query = {"article " + str(num + 1): val} if self.db.check_collection(collection=self.username, query=query) == False: db[self.username].insert_one(query).inserted_id print("Successfully added to database") else: print("Data already exists in database")
def load_file(file_name, file_data, file_type): object_name = ''.join(file_name.split('.')[:-1]) if db.check_history(type_to_table_name[file_type], object_name): raise DuplicateException('This file has already been loaded.') { 'ups': load_ups_file, 'givens': load_givens_file, 'freight': load_freight_report_file }.get(file_type)(object_name, file_data)
def __init__(self, username): scope = 'user-library-read' self.en = Echonest() self.token = util.prompt_for_user_token(username, scope) self.db = Database()
def run(host, port, db_path): server = socket(AF_INET, SOCK_STREAM) server.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1) # Database database = Database(db_path) # database.save() try: server.bind((host, int(port))) server.listen(5) # Greetings print_line( f'<nl><green>JSON Server</green><nl><cyan>http://{host}:{port}/</cyan><nl>' ) while True: headers = {} client, address = server.accept() # todo: Work around timeout # client.settimeout(60) # except socket.timeout td = datetime.now().strftime('%H:%M:%S') rd = client.recv(PACKET_SIZE).decode() # while True: # rd = client.recv(PACKET_SIZE) # if not rd: # break # Check received data if not rd: print_line('<yellow>EMPTY !</yellow><nl><nl>') client.close() continue pieces = rd.split(CRLF) method, url = pieces[0].split()[0:2] body = None status_code = 200 # Favicon - just fun if url == '/favicon.ico': with open('favicon.ico', 'rb') as fp: output = get_response( status_code, fp.readlines()[0], {'Content-type': 'image/vnd.microsoft.icon'}) client.sendall(output) client.close() # print_line('<green>/favicon.ico</green>') continue print_line( f'<nl><cyan>************ {td} **************</cyan><nl>{rd}') if method not in METHODS: status_code = 501 elif method not in ALLOWED_METHODS: status_code = 405 headers['Allow'] = ', '.join(ALLOWED_METHODS) else: parts = url.strip('/').split('/') pk = None if len(parts) > 1: pk = parts[-1] alias = '/'.join(parts[0:-1]) else: alias = parts[0] # Search by route_name and primary_key body = database.select(alias, pk) if method in ('POST', 'PUT'): print_line( f'<nl><yellow>{method} not implemented</yellow>') if not body: status_code = 404 output = get_response(status_code, body, headers) client.sendall(output) client.shutdown(SHUT_RDWR) client.close() except KeyboardInterrupt: print_line('<nl><gray>Shutting down...</gray><nl>') finally: server.close()
def setup_db(): global db db = Database(USER, PASSWORD, DB, HOST, PORT)
def __init__(self): self.db = Database()
def main(): directory = '' match = False inject = False db_name = 'certs.db' output = False try: opts, args = getopt.getopt(sys.argv[1:], 'hd:midb:-o', ['help', 'directory=', 'match', 'inject', 'database=', '--output']) except: usage() for opt, arg in opts: if opt in ('-h', '--help'): print('printing hep') usage() elif opt in ('-d', '--directory'): directory = arg if(not directory.endswith('/')): directory += '/' elif opt in ('-m', '--match'): match = True elif opt in ('-i', '--inject'): inject = True elif opt in ('-b', '--database'): db = arg elif opt in ('-o', '--output'): output = True outputfile = arg if(outputfile == ''): outputfile = 'matches.txt' if directory: print("Grabbing data...") grabber = Grabber(directory) classifier = Classifier(grabber) print("Classifying...") classifier.classify() certs, keys = classifier.get_data() db = Database(db_name) if inject: try: print("Creating the database") db.create_db() except: print("Database already exists.") print("Injecting data into the database...") db.insert_keys(keys) db.insert_certs(certs) if match: print("Matching data...") db.match_cert_key() if output: db.export_matches(outputfile)
from src.db import Database from src.query import Query from random import choice, randint, sample, seed # Student Id and 4 grades db = Database() db.open('~/ECS165') grades_table = db.get_table('Grades') query = Query(grades_table) # repopulate with random data records = {} seed(3562901) for i in range(0, 1000): key = 92106429 + i records[key] = [ key, randint(0, 20), randint(0, 20), randint(0, 20), randint(0, 20) ] keys = sorted(list(records.keys())) for _ in range(10): for key in keys: for j in range(1, grades_table.num_columns): value = randint(0, 20) records[key][j] = value keys = sorted(list(records.keys())) # for key in keys:
def load_freight_report_file(file_name, freight_report_file): db.execute_query( db.qb.bulk_insert('frieght_report_row', freight_report_file)) db.update_history('frieght_report', file_name)
def load_givens_file(file_name, givens_file): db.execute_query(db.qb.bulk_insert('givens', givens_file[0])) db.execute_query(db.qb.bulk_insert('givens_adjustments', givens_file[1])) db.update_history('givens_invoice', file_name)
def load_ups_file(file_name, ups_file): db.execute_query(db.qb.bulk_insert('ups_invoice_row', ups_file)) db.update_history('ups_invoice', file_name)
def __init__(self): self.db = Database() self.labels = LabelModel() self.notes = NoteModel()
from src.db import Database from src.query import Query from src.transaction import Transaction from src.transaction_worker import TransactionWorker import threading from random import choice, randint, sample, seed db = Database() db.open('~/ECS165') grades_table = db.create_table('Grades', 5, 0) keys = [] records = {} num_threads = 2 seed(8739878934) # Generate random records for i in range(0, 10000): key = 92106429 + i keys.append(key) records[key] = [key, 0, 0, 0, 0] q = Query(grades_table) q.insert(None, False, False, *records[key]) # create TransactionWorkers transaction_workers = [] for i in range(num_threads): transaction_workers.append(TransactionWorker([])) # generates 10k random transactions
def __init__(self, username): scope = 'user-library-read' self.en = Echonest() self.token = util.prompt_for_user_token(username, scope) self.db = Database()
from src.db import Database db = Database() print(db.open())
from src.db import Database from src.query import Query from time import process_time from random import choice, randrange # Student Id and 4 grades db = Database() grades_table = db.create_table('Grades', 0, 5) query = Query(grades_table) keys = [] # Measuring Insert Performance insert_time_0 = process_time() for i in range(0, 10000): query.insert(906659671 + i, 93, 0, 0, 0) keys.append(906659671 + i) insert_time_1 = process_time() print("Inserting 10k records took: \t\t\t", insert_time_1 - insert_time_0) # Measuring update Performance update_cols = [ [randrange(0, 100), None, None, None, None], [None, randrange(0, 100), None, None, None], [None, None, randrange(0, 100), None, None], [None, None, None, randrange(0, 100), None], [None, None, None, None, randrange(0, 100)], ] update_time_0 = process_time() for i in range(0, 10000):
from src.db import Database from src.reports import (generate_author_index, generate_author_table, generate_known_aliases, OrderType) if '__name__' == '__main__': db = Database.create('../db/resolutions.csv', '../db/aliases.csv') generate_author_index(db) generate_author_table(db, OrderType.AUTHOR) generate_author_table(db, OrderType.TOTAL) generate_author_table(db, OrderType.ACTIVE_TOTAL) generate_author_table(db, OrderType.ACTIVE_NON_REPEALS_TOTAL) generate_author_table(db, OrderType.ACTIVE_REPEALS_TOTAL) generate_author_table(db, OrderType.REPEALED_TOTAL) generate_known_aliases(db)