def __init__(self, multiproc_dict):
self._multiproc_dict: dict = multiproc_dict
self.cli_args = InsightUtilities.InsightArgumentParser.get_cli_args()
self.set_crash_recovery(self.cli_args.crash_recovery, None)
self.config_file = configparser.ConfigParser()
self.config_file.read(self.get_config_file(self.cli_args.config))
self._header_dict = {}
self.welcome()
self._import_everything_flag = False
self._import_check()
self._db_manager = database.setup_database(self)
self._sc_session: scoped_session = self._db_manager.get_scoped_session(
)
self.static_data_import = static_data.static_data_import(
self, self._import_everything_flag)
self.routes = RouteMapper.RouteMapper(self)
self.routes.setup_load()
self.sso = EVEsso.EVEsso(self)
self.channel_manager = cm.Channel_manager(self)
self.zk_obj = zk.zk(self)
self._admin_module = InsightAdmins.InsightAdmins()
self.motd = self._read_motd()
self.set_crash_recovery(
self.cli_args.crash_recovery,
self._admin_module.get_default_admin()) # set id
def get_sessions():
sessions = []
conn = setup_database()
searches = get_searches(
conn,
'data/week4/data_for_one_week_bigquery/cleaned/bigquery_results_20180403_20180411_filtered.csv'
)
for search in searches.itertuples():
query = search.search_term_lowercase
session = TaskCentricSearchSession(query, query)
for url in search.passed_over_urls:
if url in search.clicked_urls:
result = SearchResult(url, 1)
else:
result = SearchResult(url, 0)
session.web_results.append(result)
result = SearchResult(search.final_click_url, 1)
session.web_results.append(result)
sessions.append(session)
return sessions
def test_import_invalid_data(self):
"""Test import_data() function"""
test_db = setup_database()
delete_collections(test_db)
record, error = import_data(test_db, 'input', 'product.csv',
'dump_customer.csv', 'rentals.csv')
with self.assertRaises(AssertionError):
self.assertTupleEqual(error, (0, 0, 0))
self.assertTupleEqual(record, (3, 2, 5))
delete_collections(test_db)
def test_import_data(self):
"""Test import_data() function"""
test_db = setup_database()
delete_collections(test_db)
record, error = import_data(test_db, 'input', 'product.csv',
'customer.csv', 'rentals.csv')
self.assertTupleEqual(record, (3, 2, 5))
self.assertTupleEqual(error, (0, 0, 0))
print_collections(test_db)
delete_collections(test_db)
def main():
#############################
##### Define options #####
#############################
city = 'Cincinnati'
state = 'Ohio'
browser = 'FIREFOX' #FIREFOX or CHROME. Havent tested with Chrome yet
headless = False #Open the browser in headless mode = True/False
implicitly_wait = 20 #Seconds to wait implicitly if not explicitly set
database_file = r'data/pythonsqlite.db'
#############################
##### Database Setup ####
#############################
#Create empty database with auction and auction_items table
database.setup_database(database_file)
connection = database.create_connection(database_file)
#############################
###### Web Scraping ######
#############################
#Driver setup
driver, wait5, wait_halfsec = crawler.setup_driver(headless, browser,
implicitly_wait)
#Get all auctions
all_auctions = crawler.find_all_auctions_by_city(driver, wait5, city,
state)
#Add items to all auctions
crawler.add_items_to_all_auctions(driver, wait5, wait_halfsec,
all_auctions, connection)
#############################
##### Cleanup #####
#############################
connection.close()
crawler.clean_up(driver)
#Debug
print("--- %s seconds ---" % (time.time() - total_time))
def __init__(self):
self.config_file = configparser.ConfigParser()
self.cli_args = self.__read_cli_args()
self.config_file.read(self.__read_config_file())
self.welcome()
self.__import_everything_flag = False
self.__import_check()
self.__sc_session: scoped_session = database.setup_database(
self).get_scoped_session()
self.static_data_import = static_data.static_data_import(
self, self.__import_everything_flag)
self.sso = EVEsso.EVEsso(self)
self.channel_manager = cm.Channel_manager(self)
self.zk_obj = zk.zk(self)
self.motd = self.__read_motd()
def test_show_rentals(self):
"""Test show_rentals() function"""
expect_response = {
'user002': {
'name': 'Maya Data',
'address': '4936 Elliot Avenue',
'phone_number': '206-777-1927',
'email': '*****@*****.**'
},
}
test_db = setup_database()
delete_collections(test_db)
import_data(test_db, 'input', 'product.csv', 'customer.csv',
'rentals.csv')
result = show_rentals(test_db['rentals'], test_db['customer'],
'prd003')
self.assertDictEqual(result, expect_response)
delete_collections(test_db)
def __init__(self):
self.cli_args = InsightUtilities.InsightArgumentParser.get_cli_args()
self.welcome()
self.config = InsightUtilities.ConfigLoader()
self._header_dict = {}
self._import_everything_flag = False
self._import_check()
self._db_manager = database.setup_database(self)
self._sc_session: scoped_session = self._db_manager.get_scoped_session(
)
InsightUtilities.DBSessions(sc_session=self._sc_session)
self.static_data_import = static_data.static_data_import(
self, self._import_everything_flag)
self.routes = RouteMapper.RouteMapper(self)
self.routes.setup_load()
self.sso = EVEsso.EVEsso(self)
self.channel_manager = cm.Channel_manager(self)
self.zk_obj = zk.zk(self)
self._admin_module = InsightAdmins.InsightAdmins()
def test_show_available_products(self):
"""Test show_available_products() function"""
expect_response = {
'prd001': {
'description': '60-inch TV stand',
'product_type': 'livingroom',
'quantity_available': 5
},
'prd003': {
'description': 'bicycle',
'product_type': 'outdoor',
'quantity_available': 10
},
}
test_db = setup_database()
delete_collections(test_db)
import_data(test_db, 'input', 'product.csv', 'customer.csv',
'rentals.csv')
result = show_available_products(test_db['product'])
self.assertDictEqual(result, expect_response)
delete_collections(test_db)
def save_to_csv(training,
test,
training_file='data/training_set.csv',
test_file='data/test_set.csv'):
training.to_csv(training_file, index=False)
test.to_csv(test_file, index=False)
if __name__ == '__main__':
"""
Load sessions from the DB, split into training/test set,
and save to CSVs so they can be read in by other scripts.
"""
conn = setup_database()
searches = get_searches(conn)
training, test = train_test_split(searches)
print(
f'split into {len(training)} training rows and {len(test)} test rows')
save_to_csv(training, test)
training2, test2 = load_from_csv()
print('testing output')
assert_frame_equal(training.reset_index(drop=True),
training2.reset_index(drop=True))
assert_frame_equal(test.reset_index(drop=True),
test2.reset_index(drop=True))
print('ok')
def initialize():
setup_database()
update_poi()
'wget',
corpus_location,
'-P',
corpus_path,
])
else:
print "Corpus already downloaded, skipping."
print "Unzipping songs..."
if len(os.listdir(corpus_path)) > 1:
print "Songs already unzipped, skipping."
else:
subprocess.check_call([
'tar',
'-xzf',
'./corpus/music_repo.tar',
'-C',
corpus_path,
])
print "Successfully unzipped songs."
if __name__ == '__main__':
if os.path.exists(db['path']):
print "Dropping existing database..."
drop_database()
print "Setting up the database..."
setup_database()
print "Finished database setup.\n"
download_corpus()
csv_import()
def setup():
clf = classify_model()
setup_database()
learn_response = 0
return clf, learn_response
def parse_and_execute_actions(args):
if args.action == 'none':
logging.info('Nothing to do, leaving now...')
return
elif args.action == 'create-config':
if args.default:
config.create_default_config(args.config)
elif args.interactive:
config.create_config_interactively(args.config)
else:
logging.warning('No way to create the config file was specified. '
'Choose --default or --interactive.')
return
config.validate_config(args.config)
cfg = config.load_config(args.config)
if args.action == 'setup-database':
database.setup_database(cfg)
return
db, cursor = database.connect_to_database(cfg)
if not project.is_machine_registered(cfg, db, cursor):
project.register_render_machine(cfg, db, cursor)
if args.action == 'start':
if not project.check_if_project_name_valid(args):
logging.error('Project name "{}" is not a valid choice!'.format(
args.project_name))
elif project.check_if_project_name_taken(args, db, cursor):
logging.error('Project name "{}" is already taken!'.format(
args.project_name))
else:
project.start_project(args, cfg, db, cursor)
elif args.action == 'cancel':
# TODO add security check
if args.project:
project.cancel_project(args, cfg, db, cursor)
elif args.frames:
project.cancel_frames(args, cfg, db, cursor)
else:
logging.warning(
'Missing additional parameter, specify whether the project (--project) or frames '
'(--frames <...>) should be deleted.')
elif args.action == 'finish':
project.finish_project(args, cfg, db, cursor)
elif args.action == 'check': # status report
if not project.check_if_project_name_valid(args):
logging.error('Project name "{}" is not a valid choice!'.format(
args.project_name))
elif not project.check_if_project_name_taken(args, db, cursor):
logging.error('Project name "{}" does not exist!'.format(
args.project_name))
else:
project.get_project_info(args, cfg, db, cursor)
if args.history:
pass
elif args.frame_list:
print(project.get_project_frame_list(args, cfg, db, cursor))
elif args.action == 'render':
if args.one_batch:
project.render_frames(args, cfg, db, cursor)
elif args.some_batches > 0:
for _ in range(args.some_batches):
project.render_frames(args, cfg, db, cursor)
else:
while project.has_project_open_frames(args, cfg, db, cursor):
project.render_frames(args, cfg, db, cursor)
elif args.action == 'free':
if args.free_failed:
project.free_failed_frames(args, cfg, db, cursor)
elif args.free_waiting:
project.free_waiting_frames(args, cfg, db, cursor)
database.close_connection(db, cursor)
def expand_content_ids(df, on=None):
conn = setup_database()
content_items = get_content_items(conn)
return df.join(content_items, how='left', on=on)
from flask import Flask, request, json, Response
from Utils.utils import ToDict
from database import setup_database
from Controller.product_controller import product_api
from Controller.user_controller import user_api
from Controller.transaction_controller import transaction_api
from flask_cors import CORS
app = Flask(__name__)
CORS(app)
app.debug = True
app.register_blueprint(product_api)
app.register_blueprint(user_api)
app.register_blueprint(transaction_api)
if __name__ == '__main__':
setup_database(app)
app.run('0.0.0.0', port=5000)
if args.recover:
with open(args.recover, 'rb') as f:
workpop, completedcount, stalework = load(f)
for __ in range(workpop):
__ = next(workgenerator)
if args.aggressive_recover:
ioloop.IOLoop.current().run_sync(lambda: advance_work(server_config))
try:
start = datetime.now()
# Don't set up tables when recovering. We have explicitly coded to exit
# if the tables already exist. Not sure if we need to modify this
if not args.recover and not args.aggressive_recover:
ioloop.IOLoop.current().run_sync(lambda: setup_database(
server_config['database'],
server_config.get('use temp table', False)))
app = make_app(static_files, server_config, client_config)
app.listen(server_config['port'])
exitcall = ioloop.PeriodicCallback(
lambda: try_exit(server_config, args.config), 1000)
if 'stats' in server_config:
if 'interval' not in server_config['stats']:
server_config['stats'] = 1
serverstatcall = ioloop.PeriodicCallback(
write_stats, server_config['stats']['interval'] * 1000)
db_helpers = [
Process(target=result_handler,
args=(server_config['database'], received_queue,
workdone, parent,
server_config.get('use temp table',
corpus_location,
'-P',
corpus_path,
])
else:
print "Corpus already downloaded, skipping."
print "Unzipping songs..."
if len(os.listdir(corpus_path)) > 1:
print "Songs already unzipped, skipping."
else:
subprocess.check_call([
'tar',
'-xzf',
'./corpus/music_repo.tar',
'-C',
corpus_path,
])
print "Successfully unzipped songs."
if __name__ == '__main__':
if os.path.exists(db['path']):
print "Dropping existing database..."
drop_database()
print "Setting up the database..."
setup_database()
print "Finished database setup.\n"
download_corpus()
csv_import()