def test_load_state(self):
"""
Test that data can be loaded to the application
from an existing database
"""
amity = self.load_data_into_amity(Amity())
test_db = Database(amity)
test_db.save_state({"--db": "test.db"})
amity2 = Amity()
test_db2 = Database(amity2)
# Load data from previously created database
test_db2.load_state({"<sqlite_database>": "test.db"})
# Data is entered into the application
print('in test_load_state', amity.people)
print('in test_load_state', amity.rooms)
print(len(amity.rooms))
print(amity.livingspaces)
self.assertEqual(2, len(amity.rooms))
self.assertEqual(1, len(amity.livingspaces))
self.assertEqual(1, len(amity.offices))
self.assertEqual(2, len(amity.people))
self.assertEqual(1, len(amity.fellows))
self.assertEqual(1, len(amity.staff))
os.remove("test.db")
def __createDatabase(self, projectName: str = None) -> Database:
if projectName:
return Database(projectName)
databaseFile = self.shell.create_named_temporary_file(
suffix=".legion",
prefix="legion-",
directory="./tmp/",
delete_on_close=False) # to store the db file
return Database(databaseFile.name)
def openExistingProject(self, filename):
try:
print('[+] Opening project..')
self.istemp = False # indicate the file is NOT temporary and should NOT be deleted later
self.projectname = str(
filename) # set the new projectname and outputfolder vars
if not str(filename).endswith('.sprt'):
self.outputfolder = str(
filename
) + '-tool-output' # use the same name as the file for the folder (without the extension)
else:
self.outputfolder = str(filename)[:-5] + '-tool-output'
self.usernamesWordlist = Wordlist(
self.outputfolder +
'/sparta-usernames.txt') # to store found usernames
self.passwordsWordlist = Wordlist(
self.outputfolder +
'/sparta-passwords.txt') # to store found passwords
self.runningfolder = tempfile.mkdtemp(
suffix="-running",
prefix="sparta-") # to store tool output of running processes
self.db = Database(self.projectname) # use the new db
self.cwd = ntpath.dirname(
str(self.projectname)
) + '/' # update cwd so it appears nicely in the window title
except:
print('\t[-] Something went wrong while opening the project..')
print("[-] Unexpected error:", sys.exc_info()[0])
def post(self, exp_id):
""" Retrieve a list of experiments running on this server
:requires: A secure cookie obtained by logging in.
:param string name: Name of the experiment
:param string getaction: String of python code for get action code
:param string setreward: String of python code for set reward code
:param bool adviceid: Bool indicating whether adviceIds are used
:param bool hourly: Bool indicating whether the state of Theta should be stored hourly (apscheduler)
:returns: A JSON containing error yes / no.
:raises AUTH_ERROR: If no secure cookie avaiable.
"""
if self.get_secure_cookie("user"):
exp_obj = {}
exp_obj["name"] = self.get_body_argument("name")
exp_obj["getAction"] = self.get_body_argument("getaction")
exp_obj["setReward"] = self.get_body_argument("setreward")
if self.get_body_argument("hourly"):
exp_obj["hourlyTheta"] = True
else:
exp_obj["hourlyTheta"] = False
db = Database()
response = {}
response["id"] = db.edit_experiment(exp_obj, exp_id)
self.write(json.dumps(response))
else:
self.write("AUTH_ERROR")
def setup_logging():
"""Setup logging information.
Level Numeric Value
=========================
CRITICAL 50
ERROR 40
WARNING 30
INFO 20
DEBUG 10
"""
db = Database()
config_path = 'config/logging.yaml'
with open(config_path, 'rt') as f:
config = yaml.safe_load(f.read())
mail_settings = db.get_settings_from_db(
('smtp_server', 'from_addr', 'to_addr'))
for handler in config['handlers'].keys():
handler = config['handlers'][handler]
class_name = handler['class']
if 'SMTPHandler' in class_name:
handler['mailhost'] = mail_settings['smtp_server']
handler['fromaddr'] = mail_settings['from_addr']
handler['toaddrs'] = mail_settings['to_addr']
logging.config.dictConfig(config)
def post_index():
"""
Handles post request for index
"""
username = request.form.get('username')
weight = float(request.form.get('weight'))
if username is None or weight is None or weight < 0:
return render_template(
'new_user.html',
err='Invalid Inputs, ensure that username is set and weight is greater than 0',
page_title='New User | kCalTracker'
)
db = Database()
session = db.get_session()
try:
user_info = UserInfo(name=username, weight_kg=weight)
session.add(user_info)
session.commit()
return my_runs()
except InvalidRequestError as err:
session.rollback()
return render_template(
'new_user.html',
err='Failed to add user to database, please check your inputs',
page_title='New User | kCalTracker'
)
finally:
session.close()
def main():
_db = Database("app.db")
data_access = DataAccess(db=_db)
crawl = TwitterCrawlManager()
friends_crawler = FriendsCrawler(crawl=crawl, data_access=data_access)
friends_crawler.extract_friends("less_than_5000_infctd.csv")
data_access.generate_dataset("less_than_5000_infctd_out.csv")
def createTemporaryFiles(self):
try:
print('[+] Creating temporary files..')
self.istemp = True # indicates that file is temporary and can be deleted if user exits without saving
tf = tempfile.NamedTemporaryFile(
suffix=".sprt", prefix="sparta-",
delete=False) # to store the database file
self.outputfolder = tempfile.mkdtemp(
suffix="-tool-output",
prefix="sparta-") # to store tool output of finished processes
self.runningfolder = tempfile.mkdtemp(
suffix="-running",
prefix="sparta-") # to store tool output of running processes
os.makedirs(self.outputfolder +
'/screenshots') # to store screenshots
os.makedirs(self.runningfolder + '/nmap') # to store nmap output
os.makedirs(self.runningfolder + '/hydra') # to store hydra output
self.usernamesWordlist = Wordlist(
self.outputfolder +
'/sparta-usernames.txt') # to store found usernames
self.passwordsWordlist = Wordlist(
self.outputfolder +
'/sparta-passwords.txt') # to store found passwords
self.projectname = tf.name
self.db = Database(self.projectname)
except:
print('\t[-] Something went wrong creating the temporary files..')
print("[-] Unexpected error:", sys.exc_info()[0])
def select_model(self, user, version=None, model_id=None):
"""
Set a specific model to be used, setting it in the db models table
Args:
user (str): registered user which the model belongs
version (str): version of the desired model (if None, the class attribute is used)
model_id (str): id of the desired model (if None, the class attribute is used)
"""
db = Database()
if not version:
version = self.version
db.update(table='models',
column='version',
value=version,
where_col='usr',
where_val=user)
if not model_id:
model_id = self.model_id
db.update(table='models',
column='id',
value=model_id,
where_col='usr',
where_val=user)
self.get_model_info(user)
def create_db():
db = Database(':memory:')
db.execute_stmt('''
create table reorg2(
id integer primary key autoincrement,
filename text,
age integer,
condition text,
speaker text,
sentence_type text,
question_type text,
start_time real,
stop_time real,
duration real,
lag_time real,
mother_max_pitch real,
mother_min_pitch real,
mother_mean_pitch real,
mother_pitch_delta real,
mother_pitch_category text,
baby_max_pitch real,
baby_min_pitch real,
baby_mean_pitch reak,
baby_pitch_delta real,
baby_pitch_category text
);
''')
return db
def create_db():
col_datatypes = (
str,
int,
str,
str,
str,
str,
float,
float,
float,
float,
float,
float,
float,
float,
str,
float,
float,
float,
float,
str,
)
#remove old db if it exists
if os.path.exists(db_path):
os.remove(db_path)
db = CSVDatabase.create_with_files([data_path], col_datatypes)
db.execute_script(sql_path)
db.dump_to_file(db_path)
db.close()
return Database(db_path)
def __init__(self, exp_id, key="notUsedForLoopBack"):
self.db = Database()
self.mongo_db = MongoLog()
self.exp_id = exp_id # sets the experimentID
self.properties = self.db.get_one_experiment(self.exp_id)
self.key = key
self.valid = False # should be taken from Redis
async def start(self):
"""Я сказал стартуем"""
func_name = "[START] "
self.log.info('%sStart', func_name)
try:
database = Database()
sql = database.session.query(TypesModel).\
filter_by(file=DAILY[0][0]).one_or_none()
if sql is None: # if first run
self.log.info(' First run detected - db is empty')
await self.insert_default_rows(database) # Fill sql with default data
#
with Pool(len(DAILY)) as pool:
result = pool.map_async(download_and_parse, range(len(DAILY)))
for horo_id, fail in enumerate(result.get()):
if fail:
await self.report_error(func_name, fail)
else:
self.log.info(
'%sFile %s downloaded and parsed',
func_name,
DAILY[horo_id][0]
)
else:
await self.check_for_relevance()
await self.write_to_cache(database)
except Exception as error:
await self.report_error(func_name, error)
else:
database.close()
self.log.debug('%sEnd', func_name)
def run():
totals = {
'FAN': 0,
'MAN': 0,
'total': 0,
}
for cur_env in envs:
for filename in file_bounds[cur_env]:
start, stop = file_bounds[cur_env][filename]
db = Database('%s%s.db' % (db_path, cur_env))
for speaker in totals:
row = db.select(
'data', ['count(id)'],
where_cond=
'col39 = ? and col45 >= ? and col45 + col38 <= ? and col46 = ?',
params=[speaker, start, stop, filename])
totals[speaker] += row[0][0]
row = db.select(
'data', ['count(id)'],
where_cond='col45 >= ? and col45 + col38 <= ? and col46 = ?',
params=[start, stop, filename])
totals['total'] += row[0][0]
print 'Total:'
print totals
def get(self, exp_id):
""" Retrieve a specific experiment running on this server
+--------------------------------------------------------------------+
| Example |
+====================================================================+
| http://example.com/exp/EXP_ID |
+--------------------------------------------------------------------+
:requires: A secure cookie obtained by logging in.
:param int exp_id: Experiment ID for the experiment that is to be retrieved.
:returns: A JSON containing all the info for the expriment.
:raises 401: If the experiment does not belong to this user or the exp_id is wrong.
:raises 401: If user is not logged in or if there is no secure cookie available.
"""
if self.get_current_user():
if self.validate_user_experiment(exp_id):
db = Database()
response = db.get_one_experiment(exp_id)
self.write(json.dumps(response))
else:
raise ExceptionHandler(
reason="Experiment could not be validated.",
status_code=401)
else:
raise ExceptionHandler(reason="Could not validate user.",
status_code=401)
def delete(self, exp_id):
""" Delete an experiment
+--------------------------------------------------------------------+
| Example |
+====================================================================+
| http://example.com/exp/EXP_ID |
+--------------------------------------------------------------------+
:requires: A secure cookie obtained by logging in.
:param int exp_id: The ID of the experiment to be deleted.
:returns: A JSON showing the deleted experiment.
:raises 401: If the experiment does not belong to this user or the exp_id is wrong.
:raises 401: If user is not logged in or if there is no secure cookie available.
"""
if self.get_current_user():
if self.validate_user_experiment(exp_id):
db = Database()
mongo_db = MongoLog()
response = db.delete_experiment(exp_id)
mongo_db.log_deleted_experiment(response)
self.write(json.dumps(response['exp_id']))
else:
raise ExceptionHandler(
reason="Experiment could not be validated.",
status_code=401)
else:
raise ExceptionHandler(reason="Could not validate user.",
status_code=401)
def spot_trading_opportunities(self, instrument_eod_data, tfs_settings,
account_size, port_config):
"""Checks the EOD data and checks if there are trading
opportunities.
:param instrument_eod_data: dataframe row containing
the EOD data for an instrument
:param tfs_settings: tfs section of the settings file
:param account_size: equity value of IB account
:param port_config: portfolio data from config file
"""
db = Database()
eod_data = self._transform_eod_data(instrument_eod_data) # dict
ticker = eod_data['ticker']
eod_data['instrument_id'] = db.get_instrument_id(ticker).instr_id[0]
eod_data['position_info'] = db.get_position_size(ticker)
stop, target = self._get_stop_and_target(eod_data)
if not pd.isnull(stop):
instrument_eod_data.loc['stop_price'] = stop
instrument_eod_data.loc['next_price_target'] = target
new_positions = self._check_positions(
eod_data,
tfs_settings,
account_size,
port_config)
return new_positions, instrument_eod_data
def process_dir(path, env, par_code, writer):
trs_filenames = glob.glob('%s*.trs' % (path))
trans_count = 0
lena_count = 0
lena_db = Database('%s%s.db' % (lena_db_path, env))
for filename in trs_filenames:
print '\n\tProcessing file %s' % (os.path.basename(filename))
parser = TRSParser(filename)
segs = get_trans_segs(parser.parse())
if segs:
zone_start = segs[0].start
zone_end = segs[-1].end
print '\tExamining range: %s (%0.2f) - %s (%0.2f)' % (get_time_str(
zone_start), zone_start, get_time_str(zone_end), zone_end)
trans_count += get_trans_child_vocs(segs)
lena_count += get_lena_child_vocs(lena_db, filename, zone_start,
zone_end)
lena_db.close()
writer.writerow([par_code, trans_count, lena_count])
def getGuildIds(guild_id) -> Ids:
with Database(database=toDBFilepath(guild_id)) as db:
results = db.search(table="registry_int",
columns=["title", "itemvalue"],
condition={"title": "%.id"})
ret = Ids({r["title"]: r["itemvalue"] for r in results})
return ret
def setUp(self):
self.mock_cli_input = MockCLIInput()
self.mock_cli_output = MockCLIOutput()
self.ui = UIWrapper(self.mock_cli_output)
self.rules = Rules()
self.game = Game(Player("X", self.mock_cli_input, self.ui), Player("O", MockCLIInput(), self.ui), self.ui, Validations(), self.rules, Board())
self.engine = create_engine(TEST_DB_ADDRESS)
self.db = Database(self.engine)
def dump_to_file(self, filename):
file_db = Database(filename)
sql_iter = self.conn.iterdump()
for line in sql_iter:
file_db.execute_stmt(line)
file_db.close()
def entry(guild_id: int, discord_user_id: int, channel_id: int) -> Entry:
with Database(toDBFilepath(guild_id=guild_id), isolation_level="EXCLUSIVE") as db:
db.insert(table="entries",
candidate={"discord_user_id": discord_user_id,
"contact_channel_id": channel_id})
db.commit()
return getFromDiscordId(guild_id, discord_user_id)[0]
def generate():
_db = Database("app.db")
data_access = DataAccess(db=_db)
data_access.generate_dataset("all_friends.csv")
friends_list = pd.read_csv(get_output_file_path("all_friends.csv"))
users = pd.read_csv(get_input_file_path("all_infected.csv"))
user_ids = set(users['id'])
friends_list['friends'] = friends_list.apply(lambda x: list(set(json.loads(x['friends'])) & user_ids), axis=1)
friends_list.to_csv(get_output_file_path("filtered_friends.csv"), index=False, header=False, sep="\t")
def __enter__(self):
"""'with' Controlled execution entrance"""
self.db = Database(get_app_setting('SQL_DATABASE'),
get_app_setting('SQL_USERID'),
get_app_setting('SQL_PASSWORD'))
self.db.connect()
return self
def test_save_state(self):
"""Test that application data can be saved to user-defined database"""
amity = self.load_data_into_amity(Amity())
test_db = Database(amity)
test_db.save_state({"--db": "test.db"})
# File is created
self.assertTrue(os.path.exists("test.db"))
os.remove("test.db")
def create_db():
from tempfile import mkdtemp
home = mkdtemp()
manually_create_scheme(join(home, '.test_db'))
scheme = SchemeFactory('test_db', Config, home).add_folder('square', Square)\
.add_object('function', Function)\
.product()
return Database(scheme)
def save(self):
db = Database()
db.insert_article({
'link': self.article_link,
'title': self.article_title,
'neg': self.sentiment['neg'],
'neu': self.sentiment['neu'],
'pos': self.sentiment['pos'],
'compound': self.sentiment['compound']
})
def getInt(guild_id: int,
key: str,
default_value: int = None) -> Optional[int]:
with Database(database=toDBFilepath(guild_id)) as db:
ret = db.select("registry_int",
columns=["itemvalue"],
condition={"title": key})
if ret:
return ret[0]["itemvalue"]
return default_value
def validate_user_experiment(self, exp_id):
db = Database()
user_id = int(self.get_current_user())
properties = db.get_one_experiment(exp_id)
if not properties:
return False
if int(properties['user_id']) == user_id:
return True
else:
return False
def init_db_sqlite(config: ApplicationConfig, context: Context):
url = r'sqlite:///db.sqlite'
engine = create_engine(
config.database.url,
pool_pre_ping=True,
)
database = Database(connection=engine)
database.check_connection()
context.set('database', database)
