Beispiel #1
0
 def __init__(self, username, data, host, port, database):
     self.username = username
     self.host = host
     self.port = port
     self.database = database
     self.db = Database(self.host, self.port, database=database)
     self.data = data
Beispiel #2
0
class Spotify:
    def __init__(self, username):
        scope = 'user-library-read'
        self.en = Echonest()
        self.token = util.prompt_for_user_token(username, scope)
        self.db = Database()

    def saved_tracks(self, limit, offset):
        if self.token:
            sp = spotipy.Spotify(auth=self.token)
            results = sp.current_user_saved_tracks(limit, offset)
            if len(results['items']) == 0:
                print("No more saved tracks :(")
            return results['items']
        else:
            print("Can't get token")
            return []

    def parse_saved_tracks(self, start, num_of_songs):
        offset = start  # option to not start at the beginning
        limit = 20  # echonest rate limit :/
        while offset < num_of_songs:
            print("Request at offset " + str(offset))
            for item in self.saved_tracks(limit, offset):
                # get the song data using Echonest
                data = self.en.track_attributes(item)
                if data:
                    self.db.add_row(data)
            offset += limit
            time.sleep(60)  # we only have 20 requests/minute
Beispiel #3
0
class Spotify:
    def __init__(self, username):
        scope = 'user-library-read'
        self.en = Echonest()
        self.token = util.prompt_for_user_token(username, scope)
        self.db = Database()

    def saved_tracks(self, limit, offset):
        if self.token:
            sp = spotipy.Spotify(auth=self.token)
            results = sp.current_user_saved_tracks(limit, offset)
            if len(results['items']) == 0:
                print("No more saved tracks :(")
            return results['items']
        else:
            print("Can't get token")
            return []

    def parse_saved_tracks(self, start, num_of_songs):
        offset = start # option to not start at the beginning
        limit = 20 # echonest rate limit :/
        while offset < num_of_songs:
            print("Request at offset " + str(offset))
            for item in self.saved_tracks(limit, offset):
                # get the song data using Echonest
                data = self.en.track_attributes(item)
                if data:
                    self.db.add_row(data)
            offset += limit
            time.sleep(60) # we only have 20 requests/minute
Beispiel #4
0
class MediumParser(object):
    def __init__(self, username, data, host, port, database):
        self.username = username
        self.host = host
        self.port = port
        self.database = database
        self.db = Database(self.host, self.port, database=database)
        self.data = data

    def __repr__(self):
        return f"Parser Object for data: {self.data}"

    def grab_elements(self, website):
        # This extracts the Tag class from html page
        # Note: This needs an html page, NOT a url.

        soup = bs4.BeautifulSoup(website, 'html.parser')
        children = list(soup.children)
        element = [
            children[num] for num, value in enumerate(children)
            if type(children[num]) is bs4.element.Tag
        ]
        return element

    def grab_text(self):
        """
        Main Metod: grab_text is responsible for grabbing the text from all the articles
        and storing the information into an array.
        """

        texts = []
        for website in self.data:
            data = requests.get(website).content.decode('utf-8')
            elms = self.grab_elements(website=data)
            text = elms[0].find_all("p")
            text = [''.join(text[num].text) for num, val in enumerate(text)]
            texts.append("".join(text))
        return texts

    def store_information(self):
        """
        Main Method: store_information is responsible for storing the information into the database via
        the Database interface
        """

        data = self.grab_text()
        db = self.db.select_database(self.database)
        for num, val in enumerate(data):
            query = {"article " + str(num + 1): val}
            if self.db.check_collection(collection=self.username,
                                        query=query) == False:
                db[self.username].insert_one(query).inserted_id
                print("Successfully added to database")
            else:
                print("Data already exists in database")
Beispiel #5
0
def load_file(file_name, file_data, file_type):
    object_name = ''.join(file_name.split('.')[:-1])
    if db.check_history(type_to_table_name[file_type], object_name):
        raise DuplicateException('This file has already been loaded.')
    {
        'ups': load_ups_file,
        'givens': load_givens_file,
        'freight': load_freight_report_file
    }.get(file_type)(object_name, file_data)
Beispiel #6
0
 def __init__(self, username):
     scope = 'user-library-read'
     self.en = Echonest()
     self.token = util.prompt_for_user_token(username, scope)
     self.db = Database()
Beispiel #7
0
def run(host, port, db_path):

    server = socket(AF_INET, SOCK_STREAM)
    server.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)

    # Database
    database = Database(db_path)
    # database.save()

    try:
        server.bind((host, int(port)))
        server.listen(5)

        # Greetings
        print_line(
            f'<nl><green>JSON Server</green><nl><cyan>http://{host}:{port}/</cyan><nl>'
        )

        while True:
            headers = {}
            client, address = server.accept()

            # todo: Work around timeout
            # client.settimeout(60)
            # except socket.timeout

            td = datetime.now().strftime('%H:%M:%S')
            rd = client.recv(PACKET_SIZE).decode()

            # while True:
            #     rd = client.recv(PACKET_SIZE)
            #     if not rd:
            #         break

            # Check received data
            if not rd:
                print_line('<yellow>EMPTY !</yellow><nl><nl>')
                client.close()
                continue

            pieces = rd.split(CRLF)
            method, url = pieces[0].split()[0:2]
            body = None
            status_code = 200

            # Favicon - just fun
            if url == '/favicon.ico':
                with open('favicon.ico', 'rb') as fp:
                    output = get_response(
                        status_code,
                        fp.readlines()[0],
                        {'Content-type': 'image/vnd.microsoft.icon'})
                    client.sendall(output)
                    client.close()

                # print_line('<green>/favicon.ico</green>')
                continue

            print_line(
                f'<nl><cyan>************ {td} **************</cyan><nl>{rd}')

            if method not in METHODS:
                status_code = 501
            elif method not in ALLOWED_METHODS:
                status_code = 405
                headers['Allow'] = ', '.join(ALLOWED_METHODS)
            else:
                parts = url.strip('/').split('/')
                pk = None
                if len(parts) > 1:
                    pk = parts[-1]
                    alias = '/'.join(parts[0:-1])
                else:
                    alias = parts[0]

                # Search by route_name and primary_key
                body = database.select(alias, pk)

                if method in ('POST', 'PUT'):
                    print_line(
                        f'<nl><yellow>{method} not implemented</yellow>')

                if not body:
                    status_code = 404

            output = get_response(status_code, body, headers)

            client.sendall(output)
            client.shutdown(SHUT_RDWR)
            client.close()

    except KeyboardInterrupt:
        print_line('<nl><gray>Shutting down...</gray><nl>')

    finally:
        server.close()
Beispiel #8
0
def setup_db():
    global db
    db = Database(USER, PASSWORD, DB, HOST, PORT)
 def __init__(self):
     self.db = Database()
Beispiel #10
0
def main():
    directory = ''
    match = False
    inject = False
    db_name = 'certs.db'
    output = False
    try:
        opts, args = getopt.getopt(sys.argv[1:], 'hd:midb:-o',
                                   ['help', 'directory=', 'match', 'inject',
                                    'database=', '--output'])
    except:
        usage()
    for opt, arg in opts:
        if opt in ('-h', '--help'):
            print('printing hep')
            usage()
        elif opt in ('-d', '--directory'):
            directory = arg
            if(not directory.endswith('/')):
                directory += '/'
        elif opt in ('-m', '--match'):
            match = True
        elif opt in ('-i', '--inject'):
            inject = True
        elif opt in ('-b', '--database'):
            db = arg
        elif opt in ('-o', '--output'):
            output = True
            outputfile = arg
            if(outputfile == ''):
                outputfile = 'matches.txt'
    if directory:
        print("Grabbing data...")
        grabber = Grabber(directory)
        classifier = Classifier(grabber)
        print("Classifying...")
        classifier.classify()
        certs, keys = classifier.get_data()
    db = Database(db_name)
    if inject:
        try:
            print("Creating the database")
            db.create_db()
        except:
            print("Database already exists.")
        print("Injecting data into the database...")
        db.insert_keys(keys)
        db.insert_certs(certs)
    if match:
        print("Matching data...")
        db.match_cert_key()
    if output:
        db.export_matches(outputfile)
Beispiel #11
0
from src.db import Database
from src.query import Query

from random import choice, randint, sample, seed

# Student Id and 4 grades
db = Database()
db.open('~/ECS165')
grades_table = db.get_table('Grades')
query = Query(grades_table)

# repopulate with random data
records = {}
seed(3562901)
for i in range(0, 1000):
    key = 92106429 + i
    records[key] = [
        key,
        randint(0, 20),
        randint(0, 20),
        randint(0, 20),
        randint(0, 20)
    ]
keys = sorted(list(records.keys()))
for _ in range(10):
    for key in keys:
        for j in range(1, grades_table.num_columns):
            value = randint(0, 20)
            records[key][j] = value
keys = sorted(list(records.keys()))
# for key in keys:
Beispiel #12
0
def load_freight_report_file(file_name, freight_report_file):
    db.execute_query(
        db.qb.bulk_insert('frieght_report_row', freight_report_file))
    db.update_history('frieght_report', file_name)
Beispiel #13
0
def load_givens_file(file_name, givens_file):
    db.execute_query(db.qb.bulk_insert('givens', givens_file[0]))
    db.execute_query(db.qb.bulk_insert('givens_adjustments', givens_file[1]))
    db.update_history('givens_invoice', file_name)
Beispiel #14
0
def load_ups_file(file_name, ups_file):
    db.execute_query(db.qb.bulk_insert('ups_invoice_row', ups_file))
    db.update_history('ups_invoice', file_name)
Beispiel #15
0
 def __init__(self):
     self.db = Database()
     self.labels = LabelModel()
     self.notes = NoteModel()
from src.db import Database
from src.query import Query
from src.transaction import Transaction
from src.transaction_worker import TransactionWorker

import threading
from random import choice, randint, sample, seed

db = Database()
db.open('~/ECS165')
grades_table = db.create_table('Grades', 5, 0)

keys = []
records = {}
num_threads = 2
seed(8739878934)

# Generate random records
for i in range(0, 10000):
    key = 92106429 + i
    keys.append(key)
    records[key] = [key, 0, 0, 0, 0]
    q = Query(grades_table)
    q.insert(None, False, False, *records[key])

# create TransactionWorkers
transaction_workers = []
for i in range(num_threads):
    transaction_workers.append(TransactionWorker([]))

# generates 10k random transactions
Beispiel #17
0
 def __init__(self, username):
     scope = 'user-library-read'
     self.en = Echonest()
     self.token = util.prompt_for_user_token(username, scope)
     self.db = Database()
Beispiel #18
0
from src.db import Database
db = Database()
print(db.open())
Beispiel #19
0
from src.db import Database
from src.query import Query
from time import process_time
from random import choice, randrange

# Student Id and 4 grades
db = Database()
grades_table = db.create_table('Grades', 0, 5)
query = Query(grades_table)
keys = []

# Measuring Insert Performance
insert_time_0 = process_time()
for i in range(0, 10000):
    query.insert(906659671 + i, 93, 0, 0, 0)
    keys.append(906659671 + i)
insert_time_1 = process_time()

print("Inserting 10k records took:  \t\t\t", insert_time_1 - insert_time_0)

# Measuring update Performance
update_cols = [
    [randrange(0, 100), None, None, None, None],
    [None, randrange(0, 100), None, None, None],
    [None, None, randrange(0, 100), None, None],
    [None, None, None, randrange(0, 100), None],
    [None, None, None, None, randrange(0, 100)],
]

update_time_0 = process_time()
for i in range(0, 10000):
from src.db import Database
from src.reports import (generate_author_index, generate_author_table,
                         generate_known_aliases, OrderType)

if '__name__' == '__main__':
    db = Database.create('../db/resolutions.csv', '../db/aliases.csv')
    generate_author_index(db)
    generate_author_table(db, OrderType.AUTHOR)
    generate_author_table(db, OrderType.TOTAL)
    generate_author_table(db, OrderType.ACTIVE_TOTAL)
    generate_author_table(db, OrderType.ACTIVE_NON_REPEALS_TOTAL)
    generate_author_table(db, OrderType.ACTIVE_REPEALS_TOTAL)
    generate_author_table(db, OrderType.REPEALED_TOTAL)
    generate_known_aliases(db)