def setup_logging():
"""Setup logging information.
Level Numeric Value
=========================
CRITICAL 50
ERROR 40
WARNING 30
INFO 20
DEBUG 10
"""
db = Database()
config_path = 'config/logging.yaml'
with open(config_path, 'rt') as f:
config = yaml.safe_load(f.read())
mail_settings = db.get_settings_from_db(
('smtp_server', 'from_addr', 'to_addr'))
for handler in config['handlers'].keys():
handler = config['handlers'][handler]
class_name = handler['class']
if 'SMTPHandler' in class_name:
handler['mailhost'] = mail_settings['smtp_server']
handler['fromaddr'] = mail_settings['from_addr']
handler['toaddrs'] = mail_settings['to_addr']
logging.config.dictConfig(config)
def run():
totals = {
'FAN': 0,
'MAN': 0,
'total': 0,
}
for cur_env in envs:
for filename in file_bounds[cur_env]:
start, stop = file_bounds[cur_env][filename]
db = Database('%s%s.db' % (db_path, cur_env))
for speaker in totals:
row = db.select(
'data', ['count(id)'],
where_cond=
'col39 = ? and col45 >= ? and col45 + col38 <= ? and col46 = ?',
params=[speaker, start, stop, filename])
totals[speaker] += row[0][0]
row = db.select(
'data', ['count(id)'],
where_cond='col45 >= ? and col45 + col38 <= ? and col46 = ?',
params=[start, stop, filename])
totals['total'] += row[0][0]
print 'Total:'
print totals
def spot_trading_opportunities(self, instrument_eod_data, tfs_settings,
account_size, port_config):
"""Checks the EOD data and checks if there are trading
opportunities.
:param instrument_eod_data: dataframe row containing
the EOD data for an instrument
:param tfs_settings: tfs section of the settings file
:param account_size: equity value of IB account
:param port_config: portfolio data from config file
"""
db = Database()
eod_data = self._transform_eod_data(instrument_eod_data) # dict
ticker = eod_data['ticker']
eod_data['instrument_id'] = db.get_instrument_id(ticker).instr_id[0]
eod_data['position_info'] = db.get_position_size(ticker)
stop, target = self._get_stop_and_target(eod_data)
if not pd.isnull(stop):
instrument_eod_data.loc['stop_price'] = stop
instrument_eod_data.loc['next_price_target'] = target
new_positions = self._check_positions(
eod_data,
tfs_settings,
account_size,
port_config)
return new_positions, instrument_eod_data
def process_dir(path, env, par_code, writer):
trs_filenames = glob.glob('%s*.trs' % (path))
trans_count = 0
lena_count = 0
lena_db = Database('%s%s.db' % (lena_db_path, env))
for filename in trs_filenames:
print '\n\tProcessing file %s' % (os.path.basename(filename))
parser = TRSParser(filename)
segs = get_trans_segs(parser.parse())
if segs:
zone_start = segs[0].start
zone_end = segs[-1].end
print '\tExamining range: %s (%0.2f) - %s (%0.2f)' % (get_time_str(
zone_start), zone_start, get_time_str(zone_end), zone_end)
trans_count += get_trans_child_vocs(segs)
lena_count += get_lena_child_vocs(lena_db, filename, zone_start,
zone_end)
lena_db.close()
writer.writerow([par_code, trans_count, lena_count])
def delete(self, exp_id):
""" Delete an experiment
+--------------------------------------------------------------------+
| Example |
+====================================================================+
| http://example.com/exp/EXP_ID |
+--------------------------------------------------------------------+
:requires: A secure cookie obtained by logging in.
:param int exp_id: The ID of the experiment to be deleted.
:returns: A JSON showing the deleted experiment.
:raises 401: If the experiment does not belong to this user or the exp_id is wrong.
:raises 401: If user is not logged in or if there is no secure cookie available.
"""
if self.get_current_user():
if self.validate_user_experiment(exp_id):
db = Database()
mongo_db = MongoLog()
response = db.delete_experiment(exp_id)
mongo_db.log_deleted_experiment(response)
self.write(json.dumps(response['exp_id']))
else:
raise ExceptionHandler(
reason="Experiment could not be validated.",
status_code=401)
else:
raise ExceptionHandler(reason="Could not validate user.",
status_code=401)
def post(self, exp_id):
""" Retrieve a list of experiments running on this server
:requires: A secure cookie obtained by logging in.
:param string name: Name of the experiment
:param string getaction: String of python code for get action code
:param string setreward: String of python code for set reward code
:param bool adviceid: Bool indicating whether adviceIds are used
:param bool hourly: Bool indicating whether the state of Theta should be stored hourly (apscheduler)
:returns: A JSON containing error yes / no.
:raises AUTH_ERROR: If no secure cookie avaiable.
"""
if self.get_secure_cookie("user"):
exp_obj = {}
exp_obj["name"] = self.get_body_argument("name")
exp_obj["getAction"] = self.get_body_argument("getaction")
exp_obj["setReward"] = self.get_body_argument("setreward")
if self.get_body_argument("hourly"):
exp_obj["hourlyTheta"] = True
else:
exp_obj["hourlyTheta"] = False
db = Database()
response = {}
response["id"] = db.edit_experiment(exp_obj, exp_id)
self.write(json.dumps(response))
else:
self.write("AUTH_ERROR")
def create_db():
db = Database(':memory:')
db.execute_stmt('''
create table reorg2(
id integer primary key autoincrement,
filename text,
age integer,
condition text,
speaker text,
sentence_type text,
question_type text,
start_time real,
stop_time real,
duration real,
lag_time real,
mother_max_pitch real,
mother_min_pitch real,
mother_mean_pitch real,
mother_pitch_delta real,
mother_pitch_category text,
baby_max_pitch real,
baby_min_pitch real,
baby_mean_pitch reak,
baby_pitch_delta real,
baby_pitch_category text
);
''')
return db
def createTemporaryFiles(self):
try:
print('[+] Creating temporary files..')
self.istemp = True # indicates that file is temporary and can be deleted if user exits without saving
tf = tempfile.NamedTemporaryFile(
suffix=".sprt", prefix="sparta-",
delete=False) # to store the database file
self.outputfolder = tempfile.mkdtemp(
suffix="-tool-output",
prefix="sparta-") # to store tool output of finished processes
self.runningfolder = tempfile.mkdtemp(
suffix="-running",
prefix="sparta-") # to store tool output of running processes
os.makedirs(self.outputfolder +
'/screenshots') # to store screenshots
os.makedirs(self.runningfolder + '/nmap') # to store nmap output
os.makedirs(self.runningfolder + '/hydra') # to store hydra output
self.usernamesWordlist = Wordlist(
self.outputfolder +
'/sparta-usernames.txt') # to store found usernames
self.passwordsWordlist = Wordlist(
self.outputfolder +
'/sparta-passwords.txt') # to store found passwords
self.projectname = tf.name
self.db = Database(self.projectname)
except:
print('\t[-] Something went wrong creating the temporary files..')
print("[-] Unexpected error:", sys.exc_info()[0])
async def start(self):
"""Я сказал стартуем"""
func_name = "[START] "
self.log.info('%sStart', func_name)
try:
database = Database()
sql = database.session.query(TypesModel).\
filter_by(file=DAILY[0][0]).one_or_none()
if sql is None: # if first run
self.log.info(' First run detected - db is empty')
await self.insert_default_rows(database) # Fill sql with default data
#
with Pool(len(DAILY)) as pool:
result = pool.map_async(download_and_parse, range(len(DAILY)))
for horo_id, fail in enumerate(result.get()):
if fail:
await self.report_error(func_name, fail)
else:
self.log.info(
'%sFile %s downloaded and parsed',
func_name,
DAILY[horo_id][0]
)
else:
await self.check_for_relevance()
await self.write_to_cache(database)
except Exception as error:
await self.report_error(func_name, error)
else:
database.close()
self.log.debug('%sEnd', func_name)
def post_index():
"""
Handles post request for index
"""
username = request.form.get('username')
weight = float(request.form.get('weight'))
if username is None or weight is None or weight < 0:
return render_template(
'new_user.html',
err='Invalid Inputs, ensure that username is set and weight is greater than 0',
page_title='New User | kCalTracker'
)
db = Database()
session = db.get_session()
try:
user_info = UserInfo(name=username, weight_kg=weight)
session.add(user_info)
session.commit()
return my_runs()
except InvalidRequestError as err:
session.rollback()
return render_template(
'new_user.html',
err='Failed to add user to database, please check your inputs',
page_title='New User | kCalTracker'
)
finally:
session.close()
def openExistingProject(self, filename):
try:
print('[+] Opening project..')
self.istemp = False # indicate the file is NOT temporary and should NOT be deleted later
self.projectname = str(
filename) # set the new projectname and outputfolder vars
if not str(filename).endswith('.sprt'):
self.outputfolder = str(
filename
) + '-tool-output' # use the same name as the file for the folder (without the extension)
else:
self.outputfolder = str(filename)[:-5] + '-tool-output'
self.usernamesWordlist = Wordlist(
self.outputfolder +
'/sparta-usernames.txt') # to store found usernames
self.passwordsWordlist = Wordlist(
self.outputfolder +
'/sparta-passwords.txt') # to store found passwords
self.runningfolder = tempfile.mkdtemp(
suffix="-running",
prefix="sparta-") # to store tool output of running processes
self.db = Database(self.projectname) # use the new db
self.cwd = ntpath.dirname(
str(self.projectname)
) + '/' # update cwd so it appears nicely in the window title
except:
print('\t[-] Something went wrong while opening the project..')
print("[-] Unexpected error:", sys.exc_info()[0])
def post(self, exp_id):
""" Retrieve a list of experiments running on this server
:requires: A secure cookie obtained by logging in.
:param string name: Name of the experiment
:param string getaction: String of python code for get action code
:param string setreward: String of python code for set reward code
:param bool adviceid: Bool indicating whether adviceIds are used
:param bool hourly: Bool indicating whether the state of Theta should be stored hourly (apscheduler)
:returns: A JSON containing error yes / no.
:raises AUTH_ERROR: If no secure cookie avaiable.
"""
if self.get_secure_cookie("user"):
exp_obj = {}
exp_obj["name"] = self.get_body_argument("name")
exp_obj["getAction"] = self.get_body_argument("getaction")
exp_obj["setReward"] = self.get_body_argument("setreward")
if self.get_body_argument("hourly"):
exp_obj["hourlyTheta"] = True
else:
exp_obj["hourlyTheta"] = False
db = Database()
response = {}
response["id"] = db.edit_experiment(exp_obj, exp_id)
self.write(json.dumps(response))
else:
self.write("AUTH_ERROR")
def __init__(self, exp_id, key="notUsedForLoopBack"):
self.db = Database()
self.mongo_db = MongoLog()
self.exp_id = exp_id # sets the experimentID
self.properties = self.db.get_one_experiment(self.exp_id)
self.key = key
self.valid = False # should be taken from Redis
def get(self, exp_id):
""" Retrieve a specific experiment running on this server
+--------------------------------------------------------------------+
| Example |
+====================================================================+
| http://example.com/exp/EXP_ID |
+--------------------------------------------------------------------+
:requires: A secure cookie obtained by logging in.
:param int exp_id: Experiment ID for the experiment that is to be retrieved.
:returns: A JSON containing all the info for the expriment.
:raises 401: If the experiment does not belong to this user or the exp_id is wrong.
:raises 401: If user is not logged in or if there is no secure cookie available.
"""
if self.get_current_user():
if self.validate_user_experiment(exp_id):
db = Database()
response = db.get_one_experiment(exp_id)
self.write(json.dumps(response))
else:
raise ExceptionHandler(
reason="Experiment could not be validated.",
status_code=401)
else:
raise ExceptionHandler(reason="Could not validate user.",
status_code=401)
def post(self, exp_id):
""" Retrieve a list of experiments running on this server
Input (POST) arguments: (self.get_body_argument("name"))
name : Name of the experiment
getaction : String of python code for get action code
setreward : String of python code for set reward code
adviceid : Bool indicating whether adviceIds are used
hourly : Bool indicating whether the state of Theta should be stored hourly (apscheduler)
Returns:
A JSON Blob containing error yes / no
"""
if self.get_secure_cookie("user"):
exp_obj = {}
exp_obj["name"] = self.get_body_argument("name")
exp_obj["getAction"] = self.get_body_argument("getaction")
exp_obj["setReward"] = self.get_body_argument("setreward")
db = Database()
response = {}
response["id"] = db.edit_experiment(exp_obj, exp_id)
self.write(json.dumps(response))
else:
self.write("AUTH_ERROR")
def __enter__(self):
"""'with' Controlled execution entrance"""
self.db = Database(get_app_setting('SQL_DATABASE'),
get_app_setting('SQL_USERID'),
get_app_setting('SQL_PASSWORD'))
self.db.connect()
return self
def test_save_state(self):
"""Test that application data can be saved to user-defined database"""
amity = self.load_data_into_amity(Amity())
test_db = Database(amity)
test_db.save_state({"--db": "test.db"})
# File is created
self.assertTrue(os.path.exists("test.db"))
os.remove("test.db")
def validate_user_experiment(self, exp_id):
db = Database()
user_id = int(self.get_current_user())
properties = db.get_one_experiment(exp_id)
if not properties:
return False
if int(properties['user_id']) == user_id:
return True
else:
return False
def init_db_sqlite(config: ApplicationConfig, context: Context):
url = r'sqlite:///db.sqlite'
engine = create_engine(
config.database.url,
pool_pre_ping=True,
)
database = Database(connection=engine)
database.check_connection()
context.set('database', database)
def setup():
print('Connecting to the database')
Database.init()
db = Database.get()
db.connect()
print('Setting up MeasurementUnits!')
Migration.update_database()
print('...')
MasterControl.setup()
db.close()
def save(self):
db = Database()
db.insert_article({
'link': self.article_link,
'title': self.article_title,
'neg': self.sentiment['neg'],
'neu': self.sentiment['neu'],
'pos': self.sentiment['pos'],
'compound': self.sentiment['compound']
})
def __createDatabase(self, projectName: str = None) -> Database:
if projectName:
return Database(projectName)
databaseFile = self.shell.create_named_temporary_file(
suffix=".legion",
prefix="legion-",
directory="./tmp/",
delete_on_close=False) # to store the db file
return Database(databaseFile.name)
class Experiment():
def __init__(self, exp_id, key):
self.db = Database()
self.advice_db = Advice()
self.exp_id = exp_id # sets the experimentID
self.properties = self.db.get_one_experiment(self.exp_id)
self.key = key
self.valid = False # should be taken from Redis
self.log_advice_bool = (self.properties['adviceID'] == 'true') # should be taken from Redis
def is_valid(self):
"""Checks wheter the exp_id and key match for the current experiment.
Input arguments:
none
Returns:
A boolean: true if a valid key is provided, false otherwise.
"""
# After database has a decent is_valid function, change to self.db.is_valid(exp_id)
self.valid = True
return self.valid
def advice(self):
return self.log_advice_bool
def gen_advice_id(self, context, action):
return self.advice_db.log_advice(context, action)
def get_by_advice_id(self, _id):
return self.advice_db.get_advice(_id)
### THESE NEED CHANGING FOR THE NEW LOCATION
def run_action_code(self, context, action): #or make it get_action_code and return the code instead
self.action = action
self.context = context
code = self.db.experiment_properties("exp:%s:properties" % (self.exp_id), "getAction")
exec(code)
return self.action
def run_reward_code(self, context):
self.context = context
code = self.db.experiment_properties("exp:%s:properties" % (self.exp_id), "setReward")
exec(code)
return True
### END OF CHANGE
def set_theta(self, values, context = None, action=None, all_action=False, all_context=False, name="theta"):
key = "exp:%s:" % (self.exp_id) +name
return self.db.set_theta(values, key, context, action, all_action, all_context)
def get_theta(self, context = None, action=None, all_action=False, all_context=False, all_float=True, name="theta"):
key = "exp:%s:" % (self.exp_id) +name
return self.db.get_theta(key, context, action, all_action, all_context, all_float)
def version_model(self):
db = Database()
self.version = datetime.now().strftime("%Y-%m-%d")
db.update(f"model_info", 'version', self.version, 'token',
self.data_source.token)
versions = len(glob(f'{self.model_dir}/*')) # TODO: USE DB(?)
if not versions == 0:
self.version = f'{self.version}-v{int(versions - 2) + 1}'
self.update_model_dir(f'{self.model_name}-{self.model_id}')
def test_save_state(self):
"""Test that application data can be saved to user-defined database"""
amity = self.load_data_into_amity(Amity())
test_db = Database(amity)
test_db.save_state({
"--db": "test.db"
})
# File is created
self.assertTrue(os.path.exists("test.db"))
os.remove("test.db")
def setUp(self):
self.actual_db = self.real_db
self.emptyTable('game_restriction')
self.emptyTable('members')
# Create a Database object to call the relevant functions.
self.db = Database()
# Insert dummy data into the database.
self.db.flag_gaming_channel("1", "Hearthstone", 1)
self.db.insert_coins(85431603408420864, 4000, 'Lenny#1112')
self.db.insert_coins(235892294857916417, 23, 'Christian#1111')
self.db.insert_coins(278286021903515656, 20000,
'BuffBot#0334') # Mocking the bot user.
def export_data(db_path, save_dir, progress_fraction_fcn):
db = Database(db_path)
rows = db.select('clips', ['distinct Batch_Num'],
where_cond='Batch_Num is not null')
batch_nums = map(lambda line: line[0], rows)
for i in range(len(batch_nums)):
_export_batch(db, batch_nums[i], save_dir)
progress_fraction_fcn((i + 1) / float(len(batch_nums)))
db.close()
def parse(horo_id, xml: ET.Element):
"""Парсим XML и сохраняем в базу"""
func_name = "[PARSE] "
dates = []
horo_texts = [[] for i in range(len(ZODIACS))]
try:
database = Database()
# Parse xml
for i, div in enumerate(xml):
if div.tag == 'date':
for date in div.attrib.values():
dates.append(datetime.strptime(date, '%d.%m.%Y').date())
for j in range(len(ZODIACS)):
if div.tag in ZODIACS[j]:
for horo_day in div:
horo_texts[j].append(horo_day.text.replace('\n', ''))
# horo_day
# 0 - yesterday
# 1 - today
# 2 - tomorrow
# 3 - tomorrow2
daily = {}
for horo_day in range(4):
tmp = [horo_texts[i][horo_day] for i in range(len(ZODIACS))]
texts = {
'horo_date': dates[horo_day],
'aries': tmp[0],
'taurus': tmp[1],
'gemini': tmp[2],
'cancer': tmp[3],
'leo': tmp[4],
'virgo': tmp[5],
'libra': tmp[6],
'scorpio': tmp[7],
'sagittarius': tmp[8],
'capricorn': tmp[9],
'aquarius': tmp[10],
'pisces': tmp[11]
}
daily[horo_day] = {
'horo_id': horo_id + 1,
'texts': texts
}
save_to_db(daily, database)
# Save xml to file
with open('xmls/%s.xml' % DAILY[horo_id][0], 'wb') as file:
ET.ElementTree(xml).write(file, encoding='utf-8')
except Exception as error:
raise Exception(func_name + str(error)) from error
else:
database.close()
return False # Is fail?
class StationMapper:
def __init__(self):
self.db = Database()
def _convert_to_station_name(self, station_id):
result = self.db._get_station_by_id(station_id)
for data in result:
return data[0]
def _convert_to_station_id(self, station_name):
result = self.db._get_station_by_name(station_name)
for data in result:
return data[0]
def save_to_db(params: dict, database: Database):
"""Сохранение в базу"""
func_name = "[SAVE TO DB] "
try:
for horo_day, value in params.items(): # SQL update horoscope texts
sql = update(DailyModel).\
where(
(DailyModel.horo_id == value['horo_id']) & (DailyModel.horo_day == horo_day)
).values(value['texts'])
database.execute(sql)
database.commit()
except Exception as error:
raise Exception(func_name + str(error)) from error
async def save_to_db(self, params: dict, database: Database):
"""Сохранение в базу"""
func_name = "[SAVE TO DB] "
self.log.debug('%sStart', func_name)
try:
sql = update(WeeklyModel).\
where(WeeklyModel.horo_id == params['horo_id']).values(params)
database.execute(sql)
database.commit()
except Exception as error:
raise Exception(func_name + str(error)) from error
finally:
self.log.debug('%sEnd', func_name)
def log_theta():
""" For every experiment, if Theta logging flag is set. Log theta from
redis to mongodb.
"""
redis_db = Database()
mongo_db = MongoLog()
experiment_ids = redis_db.get_experiment_ids()
for experiment_id in experiment_ids:
exp = Experiment(experiment_id)
if exp.properties["hourly_theta"] == "True":
theta = exp.get_theta()
theta["exp_id"] = experiment_id
mongo_db.log_hourly_theta(theta)
def test_load_state(self):
"""
Test that data can be loaded to the application
from an existing database
"""
amity = self.load_data_into_amity(Amity())
test_db = Database(amity)
test_db.save_state({"--db": "test.db"})
amity2 = Amity()
test_db2 = Database(amity2)
# Load data from previously created database
test_db2.load_state({"<sqlite_database>": "test.db"})
# Data is entered into the application
print('in test_load_state', amity.people)
print('in test_load_state', amity.rooms)
print(len(amity.rooms))
print(amity.livingspaces)
self.assertEqual(2, len(amity.rooms))
self.assertEqual(1, len(amity.livingspaces))
self.assertEqual(1, len(amity.offices))
self.assertEqual(2, len(amity.people))
self.assertEqual(1, len(amity.fellows))
self.assertEqual(1, len(amity.staff))
os.remove("test.db")
def generate_model_id(self):
db = Database()
self.model_id = str(uuid4()).split('-')[0]
model = f"{self.model_name}-{self.model_id}"
db.update(f"model_info", 'model', model, 'token',
self.data_source.token)
self.update_model_dir(model)
try:
os.listdir(f"{self.models_dir}/{model}")
except FileNotFoundError:
Path(f'{os.getcwd()}/{self.models_dir}/{model}').mkdir(
parents=True, exist_ok=True)
return self.model_id
def get(self, exp_id):
""" Retrieve a specific experiment running on this server
Input arguments :
id : String experiment ID (in Query string)
Returns:
A JSON Blob containing all the info for the expriment
"""
if self.get_secure_cookie("user"):
db = Database()
response = db.get_one_experiment(exp_id)
self.write(json.dumps(response))
else:
self.write("AUTH_ERROR")
def get(self):
""" Retrieve a list of experiments running on this server
Input (GET) arguments (Optional):
max : integer indicating the max length of the list
Returns:
A JSON Blob containing a list of expid : name pairs, or a single entry
"""
if self.get_secure_cookie("user"):
db = Database()
response = db.get_all_experiments()
self.write(json.dumps(response))
else:
self.write("AUTH_ERROR")
def get(self, exp_id):
""" Delete an experiment given an experiment id
Input argument:
exp_id : The ID of the experiment to be deleted
Returs:
A JSON Blob indicating the response
"""
if self.get_secure_cookie("user"):
db = Database()
response = db.delete_experiment(exp_id)
self.write(json.dumps(response))
else:
self.write("AUTH_ERROR")
def run_etl(config):
"""
Manages extract, transform and load flow. The flow imports rows data
from source databases, map the data into Python objects and finally
loads the objects into target database.
"""
try:
target_database = Database(config['target'])
except DatabaseError as err:
sys.stderr.write("ERROR: {}\n".format(err))
return 1
# Prep transform
transform = Transform(target_database, config['general'])
try:
new_date = transform.get_new_date()
except TransformError as err:
sys.stderr.write("ERROR: {}\n".format(err))
return 1
# Prep load
load = Load(target_database)
# Commits new date
tid = load.date(new_date, config['general']['time_interval'])
transform.tid = tid
logging.debug("tid={}".format(tid))
# Instantiates a currency converter using base currency as target currency
exchange = Exchange(config['general'])
transform.exchange = exchange
# Commits ETL by source
retcode = 0
for source, source_settings in config['sources'].items():
logging.debug("source={} source_settings={}".format(
source, source_settings))
source_currency = source_settings['currency']
try:
# This fetches external data so it's important to fail early
exchange.set_exchange_rate(source_currency=source_currency)
except ExchangeError as err:
sys.stderr.write("ERROR: {}\n".format(err))
return 1
retcode += run_source_etl(source, source_settings, transform, load)
target_database.close()
return retcode
def __init__(self, exp_id, key = "notUsedForLoopBack"):
self.db = Database()
self.mongo_db = MongoLog()
self.exp_id = exp_id # sets the experimentID
self.properties = self.db.get_one_experiment(self.exp_id)
self.key = key
self.valid = False # should be taken from Redis
def log_theta():
""" For every experiment, if Theta logging flag is set. Log theta from
redis to mongodb.
"""
redis_db = Database()
mongo_db = MongoLog()
experiment_ids = redis_db.get_experiment_ids()
for experiment_id in experiment_ids:
exp = Experiment(experiment_id)
properties = redis_db.get_one_experiment(experiment_id)
if properties['hourlyTheta'] == "True":
theta = exp.get_theta()
theta['exp_id'] = experiment_id
mongo_db.log_hourly_theta(theta)
print("We did it, we stored some stuff!")
def __init__(self):
self.db = Database()
self.builder = Gtk.Builder()
self.builder.add_from_file("glade/main.glade")
self.builder.connect_signals({
"onDeleteWindow": Gtk.main_quit
})
def __init__(self, exp_id, key):
self.db = Database()
self.advice_db = Advice()
self.exp_id = exp_id # sets the experimentID
self.properties = self.db.get_one_experiment(self.exp_id)
self.key = key
self.valid = False # should be taken from Redis
self.log_advice_bool = (self.properties['adviceID'] == 'true') # should be taken from Redis
def get(self):
""" Retrieve a list of experiments running on this server
+--------------------------------------------------------------------+
| Example |
+====================================================================+
| http://example.com/admin/exp/list.json?max=10 |
+--------------------------------------------------------------------+
:requires: A secure cookie obtained by logging in.
:param int max: Indicating the max length of the list.
:returns: A JSON containing a list of {expid : name} pairs
:raises AUTH_ERROR: If not secure cookie available.
"""
if self.get_secure_cookie("user"):
db = Database()
response = db.get_all_experiments()
self.write(json.dumps(response))
else:
self.write("AUTH_ERROR")
def get(self, exp_id):
""" Delete an experiment given an experiment id
+--------------------------------------------------------------------+
| Example |
+====================================================================+
| http://example.com/admin/exp/1/delete.json |
+--------------------------------------------------------------------+
:requires: A secure cookie obtained by logging in.
:param int exp_id: The ID of the experiment to be deleted.
:returns: A JSON showing the deleted experiment.
:raises AUTH_ERROR: if no secure cookie available.
"""
if self.get_secure_cookie("user"):
db = Database()
response = db.delete_experiment(exp_id)
self.write(json.dumps(response))
else:
self.write("AUTH_ERROR")
def get(self, exp_id):
""" Retrieve a specific experiment running on this server
+--------------------------------------------------------------------+
| Example |
+====================================================================+
| http://example.com/admin/exp/1/get.json |
+--------------------------------------------------------------------+
:requires: A secure cookie obtained by logging in.
:param int exp_id: String experiment ID (in Query string)
:returns: A JSON containing all the info for the expriment.
:raises AUTH_ERROR: If no secure cookie available.
"""
if self.get_secure_cookie("user"):
db = Database()
response = db.get_one_experiment(exp_id)
self.write(json.dumps(response))
else:
self.write("AUTH_ERROR")
def test_load_state(self):
"""
Test that data can be loaded to the application
from an existing database
"""
amity = self.load_data_into_amity(Amity())
test_db = Database(amity)
test_db.save_state({
"--db": "test.db"
})
amity2 = Amity()
test_db2 = Database(amity2)
# Load data from previously created database
test_db2.load_state({
"<sqlite_database>": "test.db"
})
# Data is entered into the application
print('in test_load_state', amity.people)
print('in test_load_state', amity.rooms)
print(len(amity.rooms))
print(amity.livingspaces)
self.assertEqual(2, len(amity.rooms))
self.assertEqual(1, len(amity.livingspaces))
self.assertEqual(1, len(amity.offices))
self.assertEqual(2, len(amity.people))
self.assertEqual(1, len(amity.fellows))
self.assertEqual(1, len(amity.staff))
os.remove("test.db")
class Experiment():
def __init__(self, exp_id, key = "notUsedForLoopBack"):
self.db = Database()
self.advice_db = Advice()
self.exp_id = exp_id # sets the experimentID
self.properties = self.db.get_one_experiment(self.exp_id)
self.key = key
self.valid = False # should be taken from Redis
def is_valid(self):
"""Checks wheter the exp_id and key match for the current experiment.
Input arguments:
none
Returns:
A boolean: true if a valid key is provided (a prime), false otherwise.
"""
key = self.db.experiment_properties("exp:%s:properties" % (self.exp_id), "key")
if key == self.key:
self.valid = True
return self.valid
def run_action_code(self, context, action={}):
self.action = action
self.context = context
code = self.db.experiment_properties("exp:%s:properties" % (self.exp_id), "getAction")
#logging.debug(code)
exec(code)
return self.action
def run_reward_code(self, context, action, reward):
self.context = context
self.action = action
self.reward = reward
code = self.db.experiment_properties("exp:%s:properties" % (self.exp_id), "setReward")
exec(code)
return True
def log_data(self, value):
self.advice_db.log_row(value)
return True
def set_theta(self, values, context = None, action=None, all_action=False, all_context=False, name="theta"):
key = "exp:%s:" % (self.exp_id) +name
return self.db.set_theta(values, key, context, action, all_action, all_context)
def get_theta(self, context = None, action=None, all_action=False, all_context=False, all_float=True, name="theta"):
key = "exp:%s:" % (self.exp_id) +name
return self.db.get_theta(key, context, action, all_action, all_context, all_float)
def is_prime(self, n):
if n < 2: return False
for number in islice(count(2), int(sqrt(n)-1)):
if not n%number:
return False
return True
def post(self):
""" Create a new experiment
:requires: A secure cookie obtained by logging in.
:param string name: Name of the experiment
:param string getaction: String of python code for get action code
:param string setreward: String of python code for set reward code
:param bool adviceid: Bool indicating whether adviceIds are used
:param bool hourly: Bool indicating whether the state of Theta should be stored hourly.
:returns: A JSON of the form:
{ id : the assigned experiment id,
name : the name of the experiment (checked for duplicates),
error : (optional) error message }
:raises AUTH_ERROR: If no secure cookie available.
"""
if self.get_secure_cookie("user"):
exp_obj = {}
exp_obj["name"] = self.get_body_argument("name")
exp_obj["getAction"] = self.get_body_argument("getaction")
exp_obj["setReward"] = self.get_body_argument("setreward")
if self.get_body_argument("hourly"):
exp_obj["hourlyTheta"] = True
else:
exp_obj["hourlyTheta"] = False
# Generate key (also stored in REDIS)
exp_obj["key"] = hex(random.getrandbits(42))[2:-1]
db = Database()
insertid = db.insert_experiment(exp_obj)
response = {}
response["name"] = exp_obj["name"]
response["id"] = insertid
response["key"] = exp_obj["key"]
response["error"] = False
self.write(json.dumps(response))
else:
self.write("AUTH_ERROR")
def run_source_etl(source, source_settings, transform, load):
"""
Commits the extract, transform and load flow for the given source.
"""
cid = int(source_settings['company_id'])
logging.debug("cid={}".format(cid))
try:
source_database = Database(source_settings)
except DatabaseError as err:
sys.stderr.write("ERROR: {}\n".format(err))
return 1
extract = Extract(source_database)
source_departments = extract.departments()
target_departments = transform.departments(source_departments, cid)
departments_map = load.departments(target_departments)
logging.debug("departments_map={}".format(
json.dumps(departments_map, indent=2)))
source_jobroles = extract.jobroles()
target_jobroles = transform.jobroles(source_jobroles)
jobroles_map = load.jobroles(target_jobroles)
logging.debug("jobroles_map={}".format(
json.dumps(jobroles_map, indent=2)))
source_employees = extract.employees()
target_employees = transform.employees(source_employees, cid)
employees_map = load.employees(target_employees, departments_map,
jobroles_map)
logging.debug("employees_map={}".format(
json.dumps(employees_map, indent=2)))
logging.info("{} employees added from {}".format(
len(employees_map), source))
source_database.close()
return 0
def post(self):
""" Create a new experiment
Input (POST) arguments: (self.get_body_argument("name"))
name : Name of the experiment
getaction : String of python code for get action code
setreward : String of python code for set reward code
adviceid : Bool indicating whether adviceIds are used
hourly : Bool indicating whether the state of Theta should be stored hourly (apscheduler)
Returns:
A JSON Blob containing
id : the assigned experiment id
name : the name of the experiment (checked for duplicates)
error : (optional) error message
"""
if self.get_secure_cookie("user"):
exp_obj = {}
exp_obj["name"] = self.get_body_argument("name")
exp_obj["getAction"] = self.get_body_argument("getaction")
exp_obj["setReward"] = self.get_body_argument("setreward")
# Generate key (also stored in REDIS)
exp_obj["key"] = hex(random.getrandbits(42))[2:-1]
db = Database()
insertid = db.insert_experiment(exp_obj)
response = {}
response["name"] = exp_obj["name"]
response["id"] = insertid
response["key"] = exp_obj["key"]
response["error"] = False
self.write(json.dumps(response))
else:
self.write("AUTH_ERROR")
class MainWindow(Gtk.Window):
def __init__(self):
self.db = Database()
self.builder = Gtk.Builder()
self.builder.add_from_file("glade/main.glade")
self.builder.connect_signals({
"onDeleteWindow": Gtk.main_quit
})
def main(self):
for cat in self.db.query('SELECT * FROM categories'):
print("Product #%d: %s" % (cat[0], cat[1]))
window = self.builder.get_object("mainWindow")
window.show_all()
Gtk.main()
class Experiment():
""" Class that organizes experiments.
:var int exp_id: The exp_id that is tied to the experiment and the \
database.
"""
def __init__(self, exp_id, key = "notUsedForLoopBack"):
self.db = Database()
self.mongo_db = MongoLog()
self.exp_id = exp_id # sets the experimentID
self.properties = self.db.get_one_experiment(self.exp_id)
self.key = key
self.valid = False # should be taken from Redis
def is_valid(self):
"""Checks wheter the exp_id and key match for the current experiment.
:returns: A boolean: true if a valid key is provided, false otherwise.
"""
key = self.db.experiment_properties("exp:%s:properties" % (self.exp_id), "key")
if key == self.key:
self.valid = True
return self.valid
def run_action_code(self, context, action={}):
""" Takes getAction code from Redis and executes it
:param dict context: Context is a dictionary with the context for the \
getAction algorithm
:param dict action: Action is pre-created such that the exec(code) \
function can return an action dict for this function (This is because \
of the behavior of Python.).
:returns: A dict of action of which the content is \
determined by the getAction code.
"""
self.action = action
self.context = context
code = self.db.experiment_properties("exp:%s:properties" % (self.exp_id), "getAction")
#logging.debug(code)
exec(code)
return self.action
def run_reward_code(self, context, action, reward):
""" Takes setReward code from Redis and executes it
:param dict context: The context that may be needed for the algorithm.
:param string action: The action that is needed for the algorith. Is \
actually free of type, but generally a string is used.
:param int reward: Generally an int, in 0 or 1. Can be of other type, \
but must be specified by used algorithm.
:returns: Boolean True if executed correctly.
"""
self.context = context
self.action = action
self.reward = reward
code = self.db.experiment_properties("exp:%s:properties" % (self.exp_id), "setReward")
exec(code)
return True
def log_data(self, value):
""" Raw logging that is used in the getAction and setReward codes.
.. note:: Needs less ambiguity when it comes to the use of the \
specific database. As we will use MongoDB for multiple \
different logging purposes.
:param dict value: The value that needs to be logged. Since MongoDB is \
used, a dictionary is needed.
:returns: True if executed correctly.
"""
value["exp_id"] = self.exp_id
self.mongo_db.log_row(value)
return True
def set_theta(self, thetas, key = None, value = None, name = "_theta"):
""" Set the new theta (parameters) in the database.
:param dict thetas: The thetas that will eb stored. Typically a \
dictionary or a class of base.py. The function will check if it is a \
class and whether it has a get_dict function. It is okay to give a \
class with these conditions - it will call the get_dict function and \
store the dictionary.
:param string key: The key with which the theta will be associated. If \
only a key is given, all the thetas that belong to that key will be \
returned. Typically a key distinguishes experiments from each other.
:param string value: The value with which the theta will be assiocated. \
Typically the value distinguishes the different versions within an \
experiment. If no value is given, all thetas belonging to the \
key/experiment will be returned.
:param string name: The name of the parameter set.
"""
try:
check_dict = getattr(thetas, "get_dict")
except AttributeError:
check_dict = False
if check_dict and callable(check_dict):
thetas = thetas.get_dict()
db_key = "exp:%s:" % (self.exp_id) + name
if key is not None and value is not None:
db_key = db_key + ":%s:%s" % (key, value)
return self.db.set_theta(thetas, db_key)
def get_theta(self, key = None, value = None, name = "_theta", all_float = False):
""" Get the theta (parameters) from the database.
:param string key: The key with which the theta will be associated. If \
only a key is given, all the thetas that belong to that key will be \
returned. Typically a key distinguishes experiments from each other. \
:param string value: The value with which the theta will be assiocated. \
Typically the value distinguishes the different versions within an \
experiment. If no value is given, all thetas belonging to the \
key/experiment will be returned.
:param string name: The name of the parameters. Typically theta is \
okay.
:param bool all_float: If all_float is True, it will try to convert \
every value within the theta to a float.
:returns: A dictionary with the parameter set.
"""
db_key = "exp:%s:" % (self.exp_id) + name
all_values = False
if key is not None and value is not None:
db_key = db_key + ":%s:%s" % (key, value)
if key is not None and value is None:
db_key = db_key + ":%s" % (key)
all_values = True
return self.db.get_theta(db_key, all_values, all_float)
def delete_theta(self, key = None, value = None, name = "_theta"):
db_key = "exp:%s:" % (self.exp_id) + name
if key is not None and value is not None:
db_key = db_key + ":%s:%s" % (key, value)
return self.db.delete_theta(db_key)
def get_log_data(self):
""" Get all the logged data from the experiment
:returns dict logs: Dict of dict of all the manual logs
"""
return self.mongo_db.get_log_rows(self.exp_id)
def get_hourly_theta(self):
""" Get all the hourly logged thetas (if flag is set)
:returns dict of dict hourly: All the hourly logged thetas
"""
return self.mongo_db.get_hourly_theta(self.exp_id)
def debug(self, obj):
self.context['_debug'] = obj
self.clear_flag = False
def wait(self):
self.p.wait()
self.ret_code = self.p.returncode
self.clear_flag = True
return self.ret_code
def __del__(self):
if self.p.returncode is None:
self.p.kill()
print 'killed'
if __name__ == '__main__':
db = Database('~/host_list.txt')
if len(sys.argv) < 2:
screen_wrapper = ScreenWrapper()
with screen_wrapper as screen:
ih = MainMenu(screen, db.get_item_list())
ih.run()
screen_wrapper.restore_screen()
choice = ih.get_user_choice()
if choice is None:
sys.exit()
ssh_args = choice.host_info
else:
ssh_args = ' '.join(sys.argv[1:])
p = RunClass('ssh {args}'.format(args=ssh_args))
def main():
""" Generate the Database """
controller = MainWindowController()
controller.run()
Database.clearSession()