def _update(self):
updateTime = self.lastUpdateTime + datetime.timedelta(
seconds=self.interval)
# check update flag is enabled
# check Brewfather URL is defined
# check the updated interval has elapsed since last update
# check relevant data for the system is valid
# check the data has been updated since last update
if self.bUpdate and self.sURL != "" and datetime.datetime.now() > updateTime and \
self.postdata["name"] != "" and \
self.bNewData:
try:
self.jsondump = json.dumps(self.postdata).encode('utf8')
req = request.Request(
self.sURL,
data=self.jsondump,
headers={'content-type': 'application/json'})
response = request.urlopen(req)
self.bNewData = False
self.lastUpdateTime = datetime.datetime.now()
logger.info("BrewFather: " + self.postdata["temp"] + "C, " +
self.postdata["gravity"] + "SG, " +
self.postdata["aux_temp"] + "C, " +
self.lastUpdateTime.strftime("%d.%m.%Y %H:%M:%S"))
except:
logger.error("Exception posting to Brewfather: " +
self.getLastJSON())
else:
logger.debug("Update parameters:\nbUpdate = "+str(self.bUpdate)+"\nsUrl = "+self.sURL+"\npostdata.Name = "+self.postdata["name"] + \
"\npostdata.temp = "+self.postdata["temp"]+"\npostdata.gravity = "+self.postdata["gravity"] + "\npostdata.aux_temp = "+self.postdata["aux_temp"] + \
"\nupdateTime = "+updateTime.strftime("%d.%m.%Y %H:%M:%S") + \
"\nlastUpdateTime = "+self.lastUpdateTime.strftime("%d.%m.%Y %H:%M:%S") + \
"\nCurrent Time = "+datetime.datetime.now().strftime("%d.%m.%Y %H:%M:%S"))
def write_dataset(self, dataset, abs_path):
"""
Writes dataset to a file
Arguments:
- dataset: pandas dataframe
Dataset to be written
- abs_path: str
Absolute path to the file where the dataset must be written
Returns:
- No return value
"""
logger.info("Creating {0}".format(abs_path))
ext = abs_path.split('.')[-1]
if ext == 'csv':
dataset.to_csv(abs_path, index=False)
elif ext == 'xls':
dataset.to_excel(abs_path, index=False)
else:
logger.error("Expected extensions csv or xls, got {0}".format(ext))
self.error()
logger.info("{0} successfully generated".format(abs_path))
return None
def import_dataset(self, abs_path):
"""
Imports the dataset
Arguments:
- abs_path: str
Absolute path to the dataset file
Returns:
- data: pandas dataframe
Dataset as a pandas dataframe
"""
self.check_file_existence(abs_path)
logger.info("Reading {0}".format(abs_path))
ext = abs_path.split('.')[-1]
if ext == 'csv':
data = pd.read_csv(abs_path)
elif ext == 'xls':
data = pd.read_excel(abs_path)
else:
logger.error("Expected extensions csv or xls, got {0}".format(ext))
self.error()
logger.info("{0} has {1} rows and {2} columns".format(
abs_path, data.shape[0], data.shape[1]))
return data
def _readConf(self):
try:
if os.path.isfile(CONFIGFILE) == False:
logger.error("Controller configuration file is not valid: "+CONFIGFILE)
ini = configparser.ConfigParser()
ini.read(CONFIGFILE)
if 'Controller' in ini:
logger.debug("Reading Controller config")
config = ini['Controller']
try:
if config["MessageLevel"] == "DEBUG":
logger.setLevel(logging.DEBUG)
elif config["MessageLevel"] == "WARNING":
logger.setLevel(logging.WARNING)
elif config["MessageLevel"] == "ERROR":
logger.setLevel(logging.ERROR)
elif config["MessageLevel"] == "INFO":
logger.setLevel(logging.INFO)
else:
logger.setLevel(logging.INFO)
except KeyError:
logger.setLevel(logging.INFO)
try:
if config["BeerTempAdjust"] != "":
self.beerTAdjust = float(config.get("BeerTempAdjust"))
else:
raise Exception
except:
self.beerTAdjust = 0.0
logger.warning("Invalid BeerTempAdjust in configuration; using default: "+str(self.beerTAdjust))
try:
if config["ChamberTempAdjust"] != "":
self.chamberTAdjust = float(config.get("ChamberTempAdjust"))
else:
raise Exception
except:
self.chamberTAdjust = 0.0
logger.warning("Invalid BeerTempAdjust in configuration; using default: "+str(self.chamberTAdjust))
try:
if config["OnDelay"] != "" and int(config["OnDelay"]) >= 0:
self.onDelay = int(config.get("OnDelay"))*60
else:
raise Exception
except:
self.onDelay = DEFAULT_ON_DELAY
logger.warning("Invalid OnDelay in configuration; using default: "+str(self.onDelay)+" seconds")
except:
logger.warning("Problem read from configuration file: "+CONFIGFILE)
logger.debug("Controller config updated")
def slice_df_on_revenue(self):
if self.combined_df_created:
filt = self.combined_df["TotalRevenue"].notnull()
first_revenue = self.combined_df[filt].index[0]
self.combined_df = self.combined_df.loc[first_revenue:]
else:
logger.error(
f"Combined DF is not created for {self.symbol}. Cannot slice")
def DT(input_list, results_path, seed=123, criterion='gini', splitter='best', max_depth=None):
"""
1. Perform pruned decision tree classification on X and Y
2. Get heatmap of confusion matrix
3. Get decision tree
4. Get ROC curve
Arguments:
- input_list: list, length = 4
Absolute path to [X_train, Y_train, X_test, Y_test]
- results_path: str
Absolute path to the directory where the files will be saved
- seed: int, optional, default = 123
Random seed
- criterion: {gini, entropy}, optional, default = gini
Function to measure the quality of a split
- splitter: {best, random}, optional, default = best
Strategy used to choose the split at each node
- max_depth: int, value <= length of tree, optional, default = None
Maximum depth of the tree
Returns:
- Trained pruned decision tree classifier
"""
h = Helpers()
v = Variables()
# Diagnostics
h.check_dir(results_path)
if len(input_list) != 4:
logger.error("{0} files found in input_list, expected 4".format(len(input_list)))
h.error()
X_train, Y_train, X_test, Y_test = input_list
h.check_file_existence(X_train)
h.check_file_existence(Y_train)
h.check_file_existence(X_test)
h.check_file_existence(Y_test)
# Import datasets
X_train = h.import_dataset(X_train)
Y_train = h.import_dataset(Y_train)
X_test = h.import_dataset(X_test)
Y_test = h.import_dataset(Y_test)
# Train DT
dt = DecisionTreeClassifier(criterion=criterion, splitter=splitter, max_depth=max_depth, random_state=seed)
dt.fit(X_train, Y_train)
# get accuracy, confusion matrix and ROC AUC
h.get_metrics(dt, [X_train, Y_train, X_test, Y_test], results_path)
# build decision tree
h.decision_tree_viz(dt, os.path.join(results_path,'decision_tree.png'))
return dt
def get_data_from_file(self):
try:
logger.info('open file (groupes.txt)')
all_name = open('files/groupes.txt', 'r').read()
except ValueError:
logger.error('file not found')
print(f'error file not found = {ValueError}')
else:
self.names_list = list(map(str, all_name.split()))
def do_process(self, event):
if event.event_type == "created":
filename = event.src_path
try:
self.decode_pcap(filename)
except Exception as e:
logger.error("error spotted on line 30 watcher.py file")
logger.error(e)
def make_request(self):
try:
request = requests.get(self.url)
logger.info(f'URL request code {request.status_code}')
pupdate_table = Selector(text=request.text)
data_table = self._select_data_table_from_page(pupdate_table)
return data_table
except:
logger.error(f'URL request failed')
pass
def readYAML(self, filename="test.yaml"):
try:
with open(filename) as file:
yml = yaml.load(file)
print(yml)
self.yml = yml
except Exception as e:
logger.error("Exception Occured while loading YAML...",
file=sys.stderr)
logger.error(e)
sys.exit(1)
def impute_combined_df(self, column_name):
if not self.combined_df_created:
logger.error(
f"Combined DF is not created for {self.symbol}. Cannot impute")
else:
price_nulls = self.combined_df[
self.combined_df[column_name].isnull()].index
for nulls in price_nulls:
null_loc = self.combined_df.index.get_loc(nulls)
if np.isnan(self.combined_df[column_name].iloc[null_loc]):
og_null_loc = null_loc
total_nulls = 1
ends_with_null = False
starts_with_null = False
while np.isnan(
self.combined_df[column_name].iloc[null_loc]):
if null_loc == 0:
starts_with_null = True
if null_loc == len(self.combined_df[column_name]) - 1:
ends_with_null = True
break
if np.isnan(
self.combined_df[column_name].iloc[null_loc +
1]):
total_nulls += 1
null_loc += 1
first_point = og_null_loc - 1
second_point = null_loc
if ends_with_null:
impute_value = self.combined_df[column_name].iloc[
first_point]
self.combined_df[column_name].iloc[
og_null_loc:] = impute_value
elif starts_with_null:
impute_value = self.combined_df[column_name].iloc[
null_loc]
self.combined_df[
column_name].iloc[:null_loc] = impute_value
else:
rise = (
self.combined_df[column_name].iloc[second_point] -
self.combined_df[column_name].iloc[first_point])
run = second_point - first_point
slope = np.divide(rise, run)
for nums in np.arange(start=1, stop=total_nulls + 1):
self.combined_df[column_name].iloc[
first_point +
nums] = (slope * nums +
self.combined_df[column_name].
iloc[first_point])
def get_usersList(self):
usersAPI = self.usersListUrl
headers = {
'Authorization': 'Basic ' + self.authKey,
'Content-Type': 'application/json; charset=UTF-8'
}
try:
response = requests.get(usersAPI, headers=headers)
response.raise_for_status
except requests.exceptions.RequestException as e:
logger.error("error in geeting usersList" + str(e))
raise SystemExit(e)
return json.loads(response.text)
def _send_packet(self, packet):
"""Send an packet through RFM95"""
id_from, id_to, id_packet, flags, message = packet
try:
# tx_header = (To,From,ID,Flags)
self.rfm95.send(message, destination=id_to, node=id_from,
identifier=id_packet, flags=flags)
# self.rfm95.send(message, tx_header=(
# id_to, id_from, id_packet, flags))
except Exception as error:
logger.error(
'sending of packet {} failed : {}'.format(id_packet, error))
else:
logger.info('message (id : {}) from {} to {} sent : {}'.format(
id_packet, id_from, id_to, message))
def post_picture(self, externalID):
userURL = self.usersListUrl + '/' + externalID
headers = {'Authorization': 'Basic ' + self.authKey}
files = {
'avatar':
(externalID + '.' + self.imageFormat,
open(self.folderPath + '/' + externalID + '.' + self.imageFormat,
'rb'), 'image/' + self.imageFormat)
}
try:
response = requests.put(userURL, headers=headers, files=files)
response.raise_for_status
logger.info("Image upload successful!")
except requests.exceptions.RequestException as e:
logger.error("error in uploading the image: " + str(e))
raise SystemExit(e)
def check_integer(self, number):
"""
Ensure that input is a number
Arguments:
- number: Input
Returns:
- No return values
"""
try:
assert (str(number).isnumeric())
except AssertionError:
logger.error("Expected type {0}, got type {1} for {2}".format(
type(number), 'int', number))
self.error()
def upload(self, event):
#checking the event type when the user changes or modifies somthing in the specified folder
#(event_type=created or modified or deleted)
# event_type ="created"| src_path="path of the file"|
es = self.connect_elasticsearch()
if event.event_type == "created":
file_to_be_uploaded = event.src_path
packets = []
try:
#read the entire content from the json file
os.chmod(file_to_be_uploaded, 0o777)
main_name = os.path.basename(file_to_be_uploaded)
file_details = re.search(rr, main_name).groupdict()
# print(file_details)
iter = 0
for line in open(file_to_be_uploaded, errors="ignore"):
packets.append(json.loads(line))
if iter % 2 == 1:
packets[iter]["file_details"] = file_details
iter = iter + 1
extracted_name = os.path.basename(file_to_be_uploaded)
response = es.bulk(packets)
if response["errors"]:
with open("./" + logs + "/failiurelogs.txt",
"a") as myfile:
myfile.write(file_to_be_uploaded + "\n")
myfile.close()
logging.error("fail:" + extracted_name)
else:
logging.info("success:" + extracted_name)
#once the file has been uploaded to elasticsearch just delete the file in the decodedfiles directory
#os.chmod(file_to_be_uploaded, 0o777)
#os.remove(file_to_be_uploaded)
except Exception as e:
logging.error(
"exception in importing to elasticsearch line 42")
with open("./" + logs + "/failiurelogs.txt", "a") as myfile:
myfile.write(file_to_be_uploaded)
myfile.write(str(e) + "\n")
myfile.close()
logging.info("fail:" + extracted_name)
def check_file_existence(self, filename):
"""
Check if a file exists and throw error if not
Arguments:
- filename: str
Absolute path of the file whose existence is to be determined
Returns:
- No return values
"""
try:
assert (os.path.exists(filename))
except AssertionError:
logger.error("{0} does not exist".format(filename))
self.error()
return None
def check_dir(self, path):
"""
Check if directory is present and create it if not present
Arguments:
- path: str
Absolute path to the directory
Returns:
- No return values
"""
if not os.path.exists(path):
try:
logger.info("Attempting to create {0} directory".format(path))
os.makedirs(path)
logger.info("{0} directory successfully created".format(path))
except OSError:
logger.error("Unable to create {0} directory".format(path))
self.error()
def check_extension(self, filename, ext):
"""
Validates the extension of the file
Arguments:
- filename: str
Name of the file
- ext: str
Extension of the file
Returns:
- No return values
"""
try:
current = filename.split('.')[-1]
assert (current == ext)
except AssertionError:
logger.error("Expected extension {0}, got {1}".format(
ext, current))
self.error()
def check_year(self, start, end):
"""
Check if end year is greater than the start year
Arguments:
- start: int
Starting year
- end: int
Ending year
Returns:
- No return values
"""
self.check_integer(start)
self.check_integer(end)
try:
assert (end > start)
except AssertionError:
logger.error("End year must be greater than start year")
self.error()
return None
def post_news(self):
rssURL = self.feedUrl
NewsAPI = self.newsUrl
feeds = self.get_feeds()
logger.info('{} items in the feed'.format(len(feeds.entries)))
for idx,entry in enumerate(feeds.entries):
payload = self.make_payload(entry)
body = payload
headers = {'Authorization': 'Basic ' + self.authKey,
'Content-Type': 'application/json; charset=UTF-8'}
entry
logger.info('posting news with title "{}"'.format(entry.title))
try:
response = requests.post(NewsAPI,body,headers=headers)
response.raise_for_status
except requests.exceptions.RequestException as e:
logger.error('error in making a news post: ' + str(e))
raise SystemExit(e)
if idx==3:
break
def _processControllerResponse(self, str):
try:
curTime = datetime.datetime.now()
sTime = self.lastUpdateTime.strftime("%d.%m.%Y %H:%M:%S")
lhs, rhs = str.split(":", 1)
if lhs == "C":
if rhs == "-":
logger.debug("("+sTime+"): Cooling is OFF")
if self.bCoolOn == True:
self.coolEndTime = curTime
self.bCoolOn = False
elif rhs == "+":
logger.debug("("+sTime+"): Cooling is ON")
self.bCoolOn = True
elif lhs == "H":
if rhs == "-":
logger.debug("("+sTime+"): Heating is OFF")
if self.bHeatOn == True:
self.heatEndTime = curTime
self.bHeatOn = False
elif rhs == "+":
logger.debug("("+sTime+"): Heating is ON")
self.bHeatOn = True
elif lhs == "F":
self.chamberTemp = float(rhs)
logger.debug("("+sTime+"): Fridge temperature: {:5.3f}C".format(round(float(self.getChamberTemp()),1)))
self.lastUpdateTime = curTime
elif lhs == "B":
self.beerTemp = float(rhs)
logger.debug("("+sTime+"): Beer temperature: {:5.3f}C".format(round(float(self.getBeerTemp()),1)))
self.lastUpdateTime = curTime
elif lhs == "S":
self.internalTemp = float(rhs)
logger.debug("("+sTime+"): Delta to safety temperature: "+rhs+"C")
self.lastUpdateTime = curTime
except BaseException as e:
logger.error("ERROR: %s\n" % str(e))
def create_file(self, filename):
"""
Create a file if it does not exist
Arguments:
- filename: str
Absolute path of the file that has to be created
Returns:
- No return values
"""
try:
os.path.exists(filename)
except IOError:
logger.info("Attempting to create {0}".format(filename))
try:
f = open(filename, 'w')
f.close()
except IOError:
logger.error("Unable to create {0}".format(filename))
self.error()
logger.info("{0} successfully created".format(filename))
return None
def _readConf(self):
try:
if os.path.isfile(CONFIGFILE) == False:
logger.error("BrewFather configuration file is not valid: " +
CONFIGFILE)
ini = configparser.ConfigParser()
ini.read(CONFIGFILE)
if 'BrewFather' in ini:
logger.debug("Reading BrewFather config")
config = ini['BrewFather']
try:
if config["MessageLevel"] == "DEBUG":
logger.setLevel(logging.DEBUG)
elif config["MessageLevel"] == "WARNING":
logger.setLevel(logging.WARNING)
elif config["MessageLevel"] == "ERROR":
logger.setLevel(logging.ERROR)
elif config["MessageLevel"] == "INFO":
logger.setLevel(logging.INFO)
else:
logger.setLevel(logging.INFO)
except KeyError:
logger.setLevel(logging.INFO)
if config["Update"] != "":
self.bUpdate = strtobool(config.get("Update"))
else:
raise Excpetion
if config["UpdateURL"] != "":
self.sURL = config.get("UpdateURL")
else:
raise Exception
try:
if config["UpdateIntervalSeconds"] != "":
if int(config["UpdateIntervalSeconds"]
) >= MINIMUM_INTERVAL:
self.interval = int(
config.get("UpdateIntervalSeconds"))
else:
logger.warning(
"Brewfather update interval cannot be less than 15min; using 900s"
)
self.interval = MINIMUM_INTERVAL
else:
raise Exception
except:
logger.warning(
"Error reading Brewfather update interval; using 900s")
self.interval = MINIMUM_INTERVAL
try:
if config["Device"] != "":
self.postdata["name"] = config["Device"]
else:
raise Exception
except:
self.postdata["name"] = "Fermonitor"
except:
self.bUpdate = False
logger.warning(
"Problem read from configuration file: " + CONFIGFILE +
". Updating BrewFather.app is disabled until configuration fixed. It could take a minute for updated values in config file to be used."
)
print("[BrewFather]\nUpdate = " + str(self.bUpdate) +
"\nUpdateURL = " + self.sURL + "\nUpdateIntervalSeconds = " +
str(self.interval))
logger.debug("BrewFather config:\n[BrewFather]\nUpdate = " +
str(self.bUpdate) + "\nUpdateURL = " + self.sURL +
"\nUpdateIntervalSeconds = " + str(self.interval))
def _readConf(self):
bDefaultInterval = True
bDefaultBtDeviceId = True
if os.path.isfile(CONFIGFILE) == False:
logger.error("Tilt configuration file is not valid: " + CONFIGFILE)
else:
ini = configparser.ConfigParser()
try:
logger.debug("Reading Tilt config: " + CONFIGFILE)
ini.read(CONFIGFILE)
if self.color in ini:
config = ini[self.color]
try:
if config["UpdateIntervalSeconds"] != "" and int(
config["UpdateIntervalSeconds"]
) >= MINIMUM_INTERVAL:
self.interval = int(
config.get("UpdateIntervalSeconds"))
bDefaultInterval = False
else:
self.interval = MINIMUM_INTERVAL
except KeyError:
pass
try:
if config["BluetoothDeviceId"] != "" and int(
config["BluetoothDeviceId"]) >= 0:
self.bluetoothDeviceId = int(
config.get("BluetoothDeviceId"))
bDefaultBtDeviceId = False
else:
self.bluetoothDeviceId = 0
except KeyError:
pass
try:
if config["MessageLevel"] == "DEBUG":
logger.setLevel(logging.DEBUG)
elif config["MessageLevel"] == "WARNING":
logger.setLevel(logging.WARNING)
elif config["MessageLevel"] == "ERROR":
logger.setLevel(logging.ERROR)
elif config["MessageLevel"] == "INFO":
logger.setLevel(logging.INFO)
else:
logger.setLevel(logging.INFO)
except KeyError:
logger.setLevel(logging.INFO)
else:
logger.error("[" + self.color +
"] section not found in ini file: " +
CONFIGFILE)
except:
pass
if bDefaultInterval or bDefaultBtDeviceId:
logger.warning("Problem read from configuration file: \""+CONFIGFILE+ \
"\". Using some default values[**] for Tilt configuration. It could take a minute for updated values in config file to be used.")
sConf = "Color = " + self.color
sConf = sConf + "\nUpdateIntervalSeconds = " + str(self.interval)
if bDefaultInterval:
sConf = sConf + "**"
sConf = sConf + "\nBluetoothDeviceId = " + str(
self.bluetoothDeviceId)
if bDefaultBtDeviceId:
sConf = sConf + "**"
print(sConf)
def _evaluate(self):
self.control
self.targetTemps
self.tempDates
self.bufferBeerTemp
self.bufferChamberScale
# use wired data initially
if self.control.isDataValid():
self.beerTemp = self.control.getBeerTemp()
self.beerWireTemp = self.control.getBeerTemp()
self.chamberTemp = self.control.getChamberTemp()
self.timeData = self.control.timeOfData()
else:
self.beerTemp = DEFAULT_TEMP
self.beerWireTemp = DEFAULT_TEMP
self.chamberTemp = DEFAULT_TEMP
self.control.stopHeatingCooling()
logger.error(
"Chamber paused (60s), heating and cooling stopped: controller data is invalid"
)
self.paused = True
time.sleep(PAUSE_DELAY)
return
# if Tilt is configured and available replace related values
if (self.tilt is not None):
tiltdatatime = self.tilt.timeOfData()
if tiltdatatime is not None and tiltdatatime > datetime.datetime.now(
) - datetime.timedelta(minutes=5):
self.beerTemp = self.tilt.getTemp()
self.beerSG = self.tilt.getGravity()
if self.timeData < tiltdatatime:
self.timeData = tiltdatatime
else:
self.beerSG = DEFAULT_SG
logger.error(
"Data from tilt unavailable, checking again in 60s: using wired temperatures"
)
time.sleep(PAUSE_DELAY)
self.paused = False
_curTime = datetime.datetime.now()
# check which of the temperature change dates have passed
datesPassed = 0
for dt in self.tempDates:
if dt < _curTime:
datesPassed = datesPassed + 1
# No configured dates have passed, leave chamer powered off
if datesPassed == 0:
# Turn off heating and cooling
logger.debug(
"Leaving chamber heating/cooling off until first date reached: "
+ self.tempDates[datesPassed].strftime("%d.%m.%Y %H:%M:%S"))
self.control.stopHeatingCooling()
return
# check if last date has been reached. If so, heating/cooling should stop
elif datesPassed == len(self.tempDates):
logger.debug("Last date reached turning heating/cooling off: " +
self.tempDates[datesPassed -
1].strftime("%d.%m.%Y %H:%M:%S"))
self.control.stopHeatingCooling()
return
# date is within configured range
else:
self.targetTemp = self.targetTemps[datesPassed - 1]
# beer is warmer than target + buffer, consider cooling
if self.beerTemp > (self.targetTemp + self.bufferBeerTemp):
# check how much cooler chamber is compared to target, do not want it too low or beer temperature will overshoot too far.
if (self.targetTemp -
self.chamberTemp) < self.bufferChamberScale * (
self.beerTemp - self.targetTemp):
# Turn cooling ON
logger.debug("Cooling to be turned ON - Target: " +
str(self.targetTemp) + "; Beer: " +
str(self.beerTemp) + "; Chamber: " +
str(self.chamberTemp) + "; Beer Buffer: " +
str(self.bufferBeerTemp) +
"; Chamber Scale: " +
str(self.bufferChamberScale))
self.control.startCooling()
else:
logger.debug("Chamber is cold enough to cool beer")
self.control.stopHeatingCooling()
# beer is cooler than target + buffer, consider heating
elif self.beerTemp < (self.targetTemp - self.bufferBeerTemp):
# check how much hotter chamber is compared to target, do not want it too high or beer temperature will overshoot too far.
if (self.chamberTemp -
self.targetTemp) < self.bufferChamberScale * (
self.targetTemp - self.beerTemp):
# Turn heating ON
logger.debug("Heating to be turned ON - Target: " +
str(self.targetTemp) + "; Beer: " +
str(self.beerTemp) + "; Chamber: " +
str(self.chamberTemp) + "; Beer Buffer: " +
str(self.bufferBeerTemp) +
"; Chamber Scale: " +
str(self.bufferChamberScale))
self.control.startHeating()
else:
logger.debug("Chamber is warm enough to heat beer")
self.control.stopHeatingCooling()
# beer is within range of target +/- buffer
else:
logger.debug("No heating/cooling needed - Target: " +
str(self.targetTemp) + "; Beer: " +
str(self.beerTemp) + "; Chamber: " +
str(self.chamberTemp) + "; Beer Buffer: " +
str(self.bufferBeerTemp) + "; Chamber Scale: " +
str(self.bufferChamberScale))
self.control.stopHeatingCooling()
return
def _readConf(self):
try:
if os.path.isfile(CONFIGFILE) == False:
logger.error("Chamber configuration file is not valid: " +
CONFIGFILE)
ini = configparser.ConfigParser()
ini.read(CONFIGFILE)
if 'Chamber' in ini:
logger.debug("Reading Chamber config")
config = ini['Chamber']
try:
if config["MessageLevel"] == "DEBUG":
logger.setLevel(logging.DEBUG)
elif config["MessageLevel"] == "WARNING":
logger.setLevel(logging.WARNING)
elif config["MessageLevel"] == "ERROR":
logger.setLevel(logging.ERROR)
elif config["MessageLevel"] == "INFO":
logger.setLevel(logging.INFO)
else:
logger.setLevel(logging.INFO)
except KeyError:
logger.setLevel(logging.INFO)
# Read temperatures to target for each date
try:
if config["Temps"] != "":
self.targetTemps = []
t = config["Temps"].split(",")
for x in t:
self.targetTemps.append(float(x))
else:
raise Exception
except:
self.targetTemps = [DEFAULT_TEMP]
logger.warning("Invalid temp values; using default: " +
str(self.targetTemps[0]))
# Read dates when temperature should change
try:
if config["Dates"] != "":
self.tempDates = []
dts = config["Dates"].split(",")
for x in dts:
self.tempDates.append(
datetime.datetime.strptime(
x, '%d/%m/%Y %H:%M:%S'))
else:
raise Exception
except:
self.tempDates = [
datetime.datetime.now(),
datetime.datetime.now()
]
logger.warning(
"Invalid date values; using default. Heating/cooling will NOT start"
)
if len(self.tempDates) != len(self.targetTemps) + 1:
self.tempDates = [
datetime.datetime.now(),
datetime.datetime.now()
]
self.targetTemps = [DEFAULT_TEMP]
logger.warning(
"Invalid date or time values; using default. Heating/cooling will NOT start"
)
try:
if config["BeerTemperatureBuffer"] != "" and float(
config["BeerTemperatureBuffer"]) >= 0.0:
self.bufferBeerTemp = float(
config.get("BeerTemperatureBuffer"))
else:
raise Exception
except:
self.bufferBeerTemp = DEFAULT_BUFFER_BEER_TEMP
logger.warning(
"Invalid beer temperature buffer in configuration; using default: "
+ str(self.bufferBeerTemp))
try:
if config["ChamberScaleBuffer"] != "" and float(
config["ChamberScaleBuffer"]) >= 0.0:
self.bufferChamberScale = float(
config.get("ChamberScaleBuffer"))
else:
raise Exception
except:
self.bufferChamberScale = DEFAULT_BUFFER_CHAMBER_SCALE
logger.warning(
"Invalid chamber scale buffer in configuration; using default: "
+ str(self.bufferChamberScale))
except:
logger.warning("Problem read from configuration file: " +
CONFIGFILE)
logger.debug("Chamber config updated")
def run_financials(self):
if self.financials_are_run:
logger.info(f"Financials skipping for {self.symbol}...")
else:
logger.info(f"Beginning Financials update for {self.symbol}")
self.financials_are_run = True
yahoo_api_base = "query1.finance.yahoo.com"
# BALANCE SHEET SECTION
query_url = f"http://{yahoo_api_base}/v10/finance/quoteSummary/{self.symbol}?modules=balanceSheetHistory"
json = requests.get(query_url).json()
time.sleep(2)
if json["quoteSummary"]["error"]:
logger.error(json["quoteSummary"]["error"]["description"])
else:
# The below is a list where each element is one year
annual_balance_sheet = json["quoteSummary"]["result"][0][
"balanceSheetHistory"]["balanceSheetStatements"]
# Pull all of our parameters of interest into a Python list to use as columns for a Pandas DataFrame
balance_timestamps = []
short_term_assets, total_assets = [], []
short_term_liabilities, total_liabilities = [], []
fields = [
"totalCurrentAssets",
"totalAssets",
"totalCurrentLiabilities",
"totalLiab",
]
for year in annual_balance_sheet:
if all(fields in year for fields in fields):
if (year["totalCurrentAssets"] and year["totalAssets"]
and year["totalCurrentLiabilities"]
and year["totalLiab"]):
balance_timestamps.append(year["endDate"]["fmt"])
short_term_assets.append(
year["totalCurrentAssets"]["raw"])
total_assets.append(year["totalAssets"]["raw"])
short_term_liabilities.append(
year["totalCurrentLiabilities"]["raw"])
total_liabilities.append(year["totalLiab"]["raw"])
if balance_timestamps:
self.balance_sheet = pd.DataFrame(
{
"ShortTermAssets": short_term_assets,
"TotalAssets": total_assets,
"ShortTermLiabilities": short_term_liabilities,
"TotalLiabilities": total_liabilities,
},
index=pd.to_datetime(balance_timestamps),
)
self.balance_sheet.index.rename("Timestamp", inplace=True)
self.balance_sheet.to_csv(
f"ticker_info/balance_sheet/{self.symbol}_balance_sheet.csv"
)
else:
logger.warning(
f"Did not receive any Balance Sheet information for {self.symbol}"
)
# EPS SECTION
query_url = f"http://{yahoo_api_base}/v10/finance/quoteSummary/{self.symbol}?modules=earningsHistory"
json = requests.get(query_url).json()
time.sleep(2)
if json["quoteSummary"]["error"]:
logger.error(json["quoteSummary"]["error"]["description"])
else:
# List of each quarter for the last year
quarterly_eps = json["quoteSummary"]["result"][0][
"earningsHistory"]["history"]
eps_timestamp, eps = [], []
fields = ["quarter", "epsActual"]
for quarter in quarterly_eps:
if all(fields in quarter for fields in fields):
if quarter["quarter"] and quarter["epsActual"]:
eps_timestamp.append(quarter["quarter"]["fmt"])
eps.append(quarter["epsActual"]["raw"])
if eps_timestamp:
self.eps = pd.DataFrame(
{"EPS": eps}, index=pd.to_datetime(eps_timestamp))
self.eps.index.rename("Timestamp", inplace=True)
self.eps.to_csv(f"ticker_info/eps/{self.symbol}_eps.csv")
else:
logger.warning(
f"Did not receive any EPS data for {self.symbol}")
# REVENUE SECTION
query_url = f"http://{yahoo_api_base}/v10/finance/quoteSummary/{self.symbol}?modules=incomeStatementHistory"
json = requests.get(query_url).json()
time.sleep(2)
if json["quoteSummary"]["error"]:
logger.error(json["quoteSummary"]["error"]["description"])
else:
# List of annual income statement each year for the last 4 years
annual_income_statement = json["quoteSummary"]["result"][0][
"incomeStatementHistory"]["incomeStatementHistory"]
revenue_timestamp = []
total_revenue, gross_profit, net_income = [], [], []
fields = ["totalRevenue", "grossProfit", "netIncome"]
for year in annual_income_statement:
if all(fields in year for fields in fields):
if (year["totalRevenue"] and year["grossProfit"]
and year["netIncome"]):
revenue_timestamp.append(year["endDate"]["fmt"])
total_revenue.append(year["totalRevenue"]["raw"])
gross_profit.append(year["grossProfit"]["raw"])
net_income.append(year["netIncome"]["raw"])
if revenue_timestamp:
self.income_statement = pd.DataFrame(
{
"TotalRevenue": total_revenue,
"GrossProfit": gross_profit,
"NetIncome": net_income,
},
index=pd.to_datetime(revenue_timestamp),
)
self.income_statement.index.rename("Timestamp",
inplace=True)
self.income_statement.to_csv(
f"ticker_info/revenue/{self.symbol}_revenue.csv")
else:
logger.warning(
f"Did not receive any revenue data for {self.symbol}")
def NN(input_list, results_path, seed=123, hidden_layer_sizes=(30,30,30), activation='relu', solver='adam', regularization=0.0001, batch_size='auto', learning_rate_sch='constant', learning_rate_init=0.001, max_iter=100, tol=1e-4, momentum=0.9, nesterovs_momentum=True, early_stopping=False, validation_fraction=0.1, beta_1=0.9, beta_2=0.999, n_iter_no_change=10):
"""
1. Train a neural network to classify X based on Y
2. Get classification report
3. Get heatmap of confusion matrix
4. Get ROC curve
Arguments:
- input_list : list, length = 4
Absolute path to [X_train, Y_train, X_test, Y_test]
- results_path: str
Absolute path to the directory where the files will be saved
- seed: int, optional, default = 123
Random seed
- hidden_layer_sizes: tuple, length = n_layers-2, default = (30,30,30)
Number of neurons in each layer
- activation: {'identity', 'logistic', 'tanh', 'relu'}, optional, default = 'relu'
Activation function
- solver: {'lbfgs', 'sgd', 'adam'}, optional, default = 'adam'
Solver
- regularization: float, optional, default = 0.0001
L2 regularization term
- batch_size: int, optional, default = 'auto'
Minibatch size
- learning_rate_sch: {'constant', 'invscaling', 'adaptive'}, optional, default='constant'
Learning rate schedules
- learning_rate_init: float, optional, default = 0.001
Initial learning rate used
- max_iter: int, optional, default = 100
Maximum number of iterations
- tol: float, optional, default = 1e-4
Tolerance for the optimization
- momentum: float, optional, default = 0.9
Momentum for gradient descent update
- nesterovs_momentum: boolean, optional, default = True
Whether to use Nesterov's momentum
- early_stopping: bool, optional, default = False
Whether to use early stopping to terminate training when validation score is not improving
- validation_fraction: float, optional, default = 0.1
Proportion of training data to set aside as validation set for early stopping
- beta_1: float in [0,1), optional, default = 0.9
Exponential decay rate for estimates of first moment vector in adam
- beta_2: float in [0,1), optional, default = 0.999
Exponential decay rate for estimates of second moment vector in adam
- n_iter_no_change: int, optional, default = 10
Maximum number of epochs to not meet tol improvement
Returns:
- Trained neural network
"""
h = Helpers()
v = Variables()
# Diagnostics
h.check_dir(results_path)
if len(input_list) != 4:
logger.error("{0} files found in input_list, expected 4".format(len(input_list)))
h.error()
X_train, Y_train, X_test, Y_test = input_list
h.check_file_existence(X_train)
h.check_file_existence(Y_train)
h.check_file_existence(X_test)
h.check_file_existence(Y_test)
# Import datasets
X_train = h.import_dataset(X_train)
Y_train = h.import_dataset(Y_train)
X_test = h.import_dataset(X_test)
Y_test = h.import_dataset(Y_test)
# Train NN
nn = MLPClassifier(hidden_layer_sizes = hidden_layer_sizes,
activation=activation,
solver=solver,
alpha=regularization,
batch_size=batch_size,
learning_rate=learning_rate_sch,
learning_rate_init=learning_rate_init,
max_iter=max_iter,
random_state=seed,
tol=tol,
momentum=momentum,
nesterovs_momentum=nesterovs_momentum,
early_stopping=early_stopping,
validation_fraction=validation_fraction,
beta_1=beta_1,
beta_2=beta_2,
n_iter_no_change=n_iter_no_change)
nn.fit(X_train, Y_train)
# get accuracy, confusion matrix and ROC AUC
h.get_metrics(nn, [X_train, Y_train, X_test, Y_test], results_path)
return nn
def LR(input_list, results_path, seed=123, k_folds=10):
"""
1. Perform k-fold logistic regression on X and Y
2. Get heatmap of confusion matrix
3. Get ROC curve
Arguments:
- input_list: list, length = 2 or 4
Absolute path to [X,Y] or [X_train, Y_train, X_test, Y_test]
- results_path: str
Absolute path to the directory where the figures must be saved
- seed: int, optional, default = 123
Random seed
- k_folds: int, optional, default = 10
Number of folds for cross-validation
Returns:
- Trained logistic regression model
"""
h = Helpers()
v = Variables()
# Diagnostics
h.check_dir(results_path)
num_files = len(input_list)
if num_files == 2:
X, Y = input_list
h.check_file_existence(X)
h.check_file_existence(Y)
elif num_files == 4:
X_train, Y_train, X_test, Y_test = input_list
h.check_file_existence(X_train)
h.check_file_existence(Y_train)
h.check_file_existence(X_test)
h.check_file_existence(Y_test)
else:
logger.error(
"{0} files found in input_list, expected 2 or 4".format(num_files))
h.error()
# Import datasets
if num_files == 2:
X = h.import_dataset(X)
Y = h.import_dataset(Y)
else:
X_train = h.import_dataset(X_train)
Y_train = h.import_dataset(Y_train)
X_test = h.import_dataset(X_test)
Y_test = h.import_dataset(Y_test)
# Train LR model
if num_files == 2:
lr = LogisticRegressionCV(solver='liblinear',
cv=k_folds,
random_state=seed)
lr.fit(X, Y)
# get accuracy, classification report, confusion matrix and ROC AUC
h.get_metrics(lr, [X, Y], results_path)
else:
lr = LogisticRegression(solver='liblinear', random_state=seed)
lr.fit(X_train, Y_train)
# get accuracy, classification matrix, confusion matrix and ROC AUC
h.get_metrics(lr, [X_train, Y_train, X_test, Y_test], results_path)
return lr
