def __init__(self, username, data, host, port, database):
self.username = username
self.host = host
self.port = port
self.database = database
self.db = Database(self.host, self.port, database=database)
self.data = data
class Spotify:
def __init__(self, username):
scope = 'user-library-read'
self.en = Echonest()
self.token = util.prompt_for_user_token(username, scope)
self.db = Database()
def saved_tracks(self, limit, offset):
if self.token:
sp = spotipy.Spotify(auth=self.token)
results = sp.current_user_saved_tracks(limit, offset)
if len(results['items']) == 0:
print("No more saved tracks :(")
return results['items']
else:
print("Can't get token")
return []
def parse_saved_tracks(self, start, num_of_songs):
offset = start # option to not start at the beginning
limit = 20 # echonest rate limit :/
while offset < num_of_songs:
print("Request at offset " + str(offset))
for item in self.saved_tracks(limit, offset):
# get the song data using Echonest
data = self.en.track_attributes(item)
if data:
self.db.add_row(data)
offset += limit
time.sleep(60) # we only have 20 requests/minute
class Spotify:
def __init__(self, username):
scope = 'user-library-read'
self.en = Echonest()
self.token = util.prompt_for_user_token(username, scope)
self.db = Database()
def saved_tracks(self, limit, offset):
if self.token:
sp = spotipy.Spotify(auth=self.token)
results = sp.current_user_saved_tracks(limit, offset)
if len(results['items']) == 0:
print("No more saved tracks :(")
return results['items']
else:
print("Can't get token")
return []
def parse_saved_tracks(self, start, num_of_songs):
offset = start # option to not start at the beginning
limit = 20 # echonest rate limit :/
while offset < num_of_songs:
print("Request at offset " + str(offset))
for item in self.saved_tracks(limit, offset):
# get the song data using Echonest
data = self.en.track_attributes(item)
if data:
self.db.add_row(data)
offset += limit
time.sleep(60) # we only have 20 requests/minute
class MediumParser(object):
def __init__(self, username, data, host, port, database):
self.username = username
self.host = host
self.port = port
self.database = database
self.db = Database(self.host, self.port, database=database)
self.data = data
def __repr__(self):
return f"Parser Object for data: {self.data}"
def grab_elements(self, website):
# This extracts the Tag class from html page
# Note: This needs an html page, NOT a url.
soup = bs4.BeautifulSoup(website, 'html.parser')
children = list(soup.children)
element = [
children[num] for num, value in enumerate(children)
if type(children[num]) is bs4.element.Tag
]
return element
def grab_text(self):
"""
Main Metod: grab_text is responsible for grabbing the text from all the articles
and storing the information into an array.
"""
texts = []
for website in self.data:
data = requests.get(website).content.decode('utf-8')
elms = self.grab_elements(website=data)
text = elms[0].find_all("p")
text = [''.join(text[num].text) for num, val in enumerate(text)]
texts.append("".join(text))
return texts
def store_information(self):
"""
Main Method: store_information is responsible for storing the information into the database via
the Database interface
"""
data = self.grab_text()
db = self.db.select_database(self.database)
for num, val in enumerate(data):
query = {"article " + str(num + 1): val}
if self.db.check_collection(collection=self.username,
query=query) == False:
db[self.username].insert_one(query).inserted_id
print("Successfully added to database")
else:
print("Data already exists in database")
def load_file(file_name, file_data, file_type):
object_name = ''.join(file_name.split('.')[:-1])
if db.check_history(type_to_table_name[file_type], object_name):
raise DuplicateException('This file has already been loaded.')
{
'ups': load_ups_file,
'givens': load_givens_file,
'freight': load_freight_report_file
}.get(file_type)(object_name, file_data)
def __init__(self, username):
scope = 'user-library-read'
self.en = Echonest()
self.token = util.prompt_for_user_token(username, scope)
self.db = Database()
def run(host, port, db_path):
server = socket(AF_INET, SOCK_STREAM)
server.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
# Database
database = Database(db_path)
# database.save()
try:
server.bind((host, int(port)))
server.listen(5)
# Greetings
print_line(
f'<nl><green>JSON Server</green><nl><cyan>http://{host}:{port}/</cyan><nl>'
)
while True:
headers = {}
client, address = server.accept()
# todo: Work around timeout
# client.settimeout(60)
# except socket.timeout
td = datetime.now().strftime('%H:%M:%S')
rd = client.recv(PACKET_SIZE).decode()
# while True:
# rd = client.recv(PACKET_SIZE)
# if not rd:
# break
# Check received data
if not rd:
print_line('<yellow>EMPTY !</yellow><nl><nl>')
client.close()
continue
pieces = rd.split(CRLF)
method, url = pieces[0].split()[0:2]
body = None
status_code = 200
# Favicon - just fun
if url == '/favicon.ico':
with open('favicon.ico', 'rb') as fp:
output = get_response(
status_code,
fp.readlines()[0],
{'Content-type': 'image/vnd.microsoft.icon'})
client.sendall(output)
client.close()
# print_line('<green>/favicon.ico</green>')
continue
print_line(
f'<nl><cyan>************ {td} **************</cyan><nl>{rd}')
if method not in METHODS:
status_code = 501
elif method not in ALLOWED_METHODS:
status_code = 405
headers['Allow'] = ', '.join(ALLOWED_METHODS)
else:
parts = url.strip('/').split('/')
pk = None
if len(parts) > 1:
pk = parts[-1]
alias = '/'.join(parts[0:-1])
else:
alias = parts[0]
# Search by route_name and primary_key
body = database.select(alias, pk)
if method in ('POST', 'PUT'):
print_line(
f'<nl><yellow>{method} not implemented</yellow>')
if not body:
status_code = 404
output = get_response(status_code, body, headers)
client.sendall(output)
client.shutdown(SHUT_RDWR)
client.close()
except KeyboardInterrupt:
print_line('<nl><gray>Shutting down...</gray><nl>')
finally:
server.close()
def setup_db():
global db
db = Database(USER, PASSWORD, DB, HOST, PORT)
def __init__(self):
self.db = Database()
def main():
directory = ''
match = False
inject = False
db_name = 'certs.db'
output = False
try:
opts, args = getopt.getopt(sys.argv[1:], 'hd:midb:-o',
['help', 'directory=', 'match', 'inject',
'database=', '--output'])
except:
usage()
for opt, arg in opts:
if opt in ('-h', '--help'):
print('printing hep')
usage()
elif opt in ('-d', '--directory'):
directory = arg
if(not directory.endswith('/')):
directory += '/'
elif opt in ('-m', '--match'):
match = True
elif opt in ('-i', '--inject'):
inject = True
elif opt in ('-b', '--database'):
db = arg
elif opt in ('-o', '--output'):
output = True
outputfile = arg
if(outputfile == ''):
outputfile = 'matches.txt'
if directory:
print("Grabbing data...")
grabber = Grabber(directory)
classifier = Classifier(grabber)
print("Classifying...")
classifier.classify()
certs, keys = classifier.get_data()
db = Database(db_name)
if inject:
try:
print("Creating the database")
db.create_db()
except:
print("Database already exists.")
print("Injecting data into the database...")
db.insert_keys(keys)
db.insert_certs(certs)
if match:
print("Matching data...")
db.match_cert_key()
if output:
db.export_matches(outputfile)
from src.db import Database
from src.query import Query
from random import choice, randint, sample, seed
# Student Id and 4 grades
db = Database()
db.open('~/ECS165')
grades_table = db.get_table('Grades')
query = Query(grades_table)
# repopulate with random data
records = {}
seed(3562901)
for i in range(0, 1000):
key = 92106429 + i
records[key] = [
key,
randint(0, 20),
randint(0, 20),
randint(0, 20),
randint(0, 20)
]
keys = sorted(list(records.keys()))
for _ in range(10):
for key in keys:
for j in range(1, grades_table.num_columns):
value = randint(0, 20)
records[key][j] = value
keys = sorted(list(records.keys()))
# for key in keys:
def load_freight_report_file(file_name, freight_report_file):
db.execute_query(
db.qb.bulk_insert('frieght_report_row', freight_report_file))
db.update_history('frieght_report', file_name)
def load_givens_file(file_name, givens_file):
db.execute_query(db.qb.bulk_insert('givens', givens_file[0]))
db.execute_query(db.qb.bulk_insert('givens_adjustments', givens_file[1]))
db.update_history('givens_invoice', file_name)
def load_ups_file(file_name, ups_file):
db.execute_query(db.qb.bulk_insert('ups_invoice_row', ups_file))
db.update_history('ups_invoice', file_name)
def __init__(self):
self.db = Database()
self.labels = LabelModel()
self.notes = NoteModel()
from src.db import Database
from src.query import Query
from src.transaction import Transaction
from src.transaction_worker import TransactionWorker
import threading
from random import choice, randint, sample, seed
db = Database()
db.open('~/ECS165')
grades_table = db.create_table('Grades', 5, 0)
keys = []
records = {}
num_threads = 2
seed(8739878934)
# Generate random records
for i in range(0, 10000):
key = 92106429 + i
keys.append(key)
records[key] = [key, 0, 0, 0, 0]
q = Query(grades_table)
q.insert(None, False, False, *records[key])
# create TransactionWorkers
transaction_workers = []
for i in range(num_threads):
transaction_workers.append(TransactionWorker([]))
# generates 10k random transactions
def __init__(self, username):
scope = 'user-library-read'
self.en = Echonest()
self.token = util.prompt_for_user_token(username, scope)
self.db = Database()
from src.db import Database db = Database() print(db.open())
from src.db import Database
from src.query import Query
from time import process_time
from random import choice, randrange
# Student Id and 4 grades
db = Database()
grades_table = db.create_table('Grades', 0, 5)
query = Query(grades_table)
keys = []
# Measuring Insert Performance
insert_time_0 = process_time()
for i in range(0, 10000):
query.insert(906659671 + i, 93, 0, 0, 0)
keys.append(906659671 + i)
insert_time_1 = process_time()
print("Inserting 10k records took: \t\t\t", insert_time_1 - insert_time_0)
# Measuring update Performance
update_cols = [
[randrange(0, 100), None, None, None, None],
[None, randrange(0, 100), None, None, None],
[None, None, randrange(0, 100), None, None],
[None, None, None, randrange(0, 100), None],
[None, None, None, None, randrange(0, 100)],
]
update_time_0 = process_time()
for i in range(0, 10000):
from src.db import Database
from src.reports import (generate_author_index, generate_author_table,
generate_known_aliases, OrderType)
if '__name__' == '__main__':
db = Database.create('../db/resolutions.csv', '../db/aliases.csv')
generate_author_index(db)
generate_author_table(db, OrderType.AUTHOR)
generate_author_table(db, OrderType.TOTAL)
generate_author_table(db, OrderType.ACTIVE_TOTAL)
generate_author_table(db, OrderType.ACTIVE_NON_REPEALS_TOTAL)
generate_author_table(db, OrderType.ACTIVE_REPEALS_TOTAL)
generate_author_table(db, OrderType.REPEALED_TOTAL)
generate_known_aliases(db)