def get_target_paths(self):
"""
Within a network_location, we match URLs with this list of regular
expressions, which tell us to map from a source URL to a target URL.
If there are multiple regular expressions which match a source URL,
the order of appearance will be used to resolve ambiguity.
"""
INVALID_TARGET_PATH = "Invalid target path in {network_location}!"
# An "identity" capture from source URL to target URL.
WILD_TARGET_PATH = { "(.*)": "{0}" }
target_paths = self.configuration.get("target_paths", [WILD_TARGET_PATH])
# target_paths: [ target_path, ... ]
assert isinstance(target_paths, types.ListType)
for target_path in target_paths:
try:
# target_path: { "regex_with_groups", "target_with_group_captures" }
# e.g. { ".*(/some/directory)/$", "{0}/index.html" }
assert isinstance(target_path, types.DictType)
assert len(target_path) == 1
except:
error_message = \
INVALID_TARGET_PATH.format(network_location=self.network_location)
Logger.exception(error_message)
raise InvalidConfiguration(error_message)
return target_paths
def retrieve(self, url, filename=None, reporthook=None, data=None):
INTERPOSITION_MESSAGE = "Interposing for {url}"
Logger.info(INTERPOSITION_MESSAGE.format(url=url))
# What is the actual target to download given the URL? Sometimes we would
# like to transform the given URL to the intended target; e.g. "/simple/"
# => "/simple/index.html".
target_filepath = self.get_target_filepath(url)
# TODO: Set valid headers fetched from the actual download.
# NOTE: Important to guess the mime type from the target_filepath, not the
# unmodified URL.
content_type, content_encoding = mimetypes.guess_type(target_filepath)
headers = {
# NOTE: pip refers to this same header in at least these two duplicate
# ways.
"content-type": content_type,
"Content-Type": content_type,
}
# Download the target filepath determined by the original URL.
temporary_directory, temporary_filename = self.download_target(target_filepath)
if filename is None:
# If no filename is given, use the temporary file.
filename = temporary_filename
else:
# Otherwise, copy TUF-downloaded file in its own directory
# to the location user specified.
shutil.copy2(temporary_filename, filename)
return filename, headers
def wrapper(self, *args, **kwargs):
# We assume that the first argument to instancemethod is a URL-like object;
# that is, either a string or a urllib2.Request.
url_object = args[0]
data = kwargs.get("data")
# If this is a urllib2.Request...
if isinstance(url_object, urllib2.Request):
# If this is a GET HTTP method...
if url_object.get_method() == "GET":
# ...then you should check with TUF.
url = url_object.get_full_url()
else:
# ...otherwise, revert to default behaviour.
Logger.warn(NON_GET_HTTP_METHOD_MESSAGE.format(method=url_object.get_method(),
url=url_object.get_full_url()))
return instancemethod(self, *args, **kwargs)
# ...otherwise, we assume this is a string.
else:
url = url_object
updater = __updater_controller.get(url)
# If TUF has not been configured for this URL...
if updater is None:
# ...then revert to default behaviour.
return instancemethod(self, *args, **kwargs)
else:
# ...otherwise, use TUF to get this document.
return updater.open(url, data=data)
def __urllib2_urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
"""Create a file-like object for the specified URL to read from."""
# We assume that the first argument to instancemethod is a URL-like object;
# that is, either a string or a urllib2.Request.
updater = None
# If this is a urllib2.Request...
if isinstance(url, urllib2.Request):
# If this is a GET HTTP method...
if url.get_method() == "GET":
# ...then you should check with TUF.
updater = __updater_controller.get(url.get_full_url())
else:
# ...otherwise, revert to default behaviour.
Logger.warn(NON_GET_HTTP_METHOD_MESSAGE.format(method=url.get_method(),
url=url.get_full_url()))
return urllib2.urlopen(url, data=data, timeout=timeout)
else:
# ...otherwise, we assume this is a string.
updater = __updater_controller.get(url)
if updater is None:
return urllib2.urlopen(url, data=data, timeout=timeout)
else:
response = updater.open(url, data=data)
# See urllib2.AbstractHTTPHandler.do_open
# TODO: let Updater handle this
response.msg = ""
return response
def get(self, url):
"""Get an Updater, if any, for this URL.
Assumptions:
- @url is a string."""
GENERIC_WARNING_MESSAGE = "No updater or interposition for url={url}"
DIFFERENT_NETLOC_MESSAGE = "We have an updater for netloc={netloc1} but not for netlocs={netloc2}"
HOSTNAME_FOUND_MESSAGE = "Found updater for hostname={hostname}"
HOSTNAME_NOT_FOUND_MESSAGE = "No updater for hostname={hostname}"
updater = None
try:
parsed_url = urlparse.urlparse(url)
hostname = parsed_url.hostname
port = parsed_url.port or 80
netloc = parsed_url.netloc
network_location = "{hostname}:{port}".format(hostname=hostname, port=port)
# Sometimes parsed_url.netloc does not have a port (e.g. 80),
# so we do a double check.
network_locations = set((netloc, network_location))
updater = self.__updaters.get(hostname)
if updater is None:
Logger.warn(HOSTNAME_NOT_FOUND_MESSAGE.format(hostname=hostname))
else:
# Ensure that the updater is meant for this (hostname, port).
if updater.configuration.network_location in network_locations:
Logger.info(HOSTNAME_FOUND_MESSAGE.format(hostname=hostname))
# Raises an exception in case we do not recognize how to
# transform this URL for TUF. In that case, there will be no
# updater for this URL.
target_filepath = updater.get_target_filepath(url)
else:
# Same hostname, but different (not user-specified) port.
Logger.warn(DIFFERENT_NETLOC_MESSAGE.format(
netloc1=updater.configuration.network_location, netloc2=network_locations))
updater = None
except:
Logger.exception(GENERIC_WARNING_MESSAGE.format(url=url))
updater = None
finally:
if updater is None:
Logger.warn(GENERIC_WARNING_MESSAGE.format(url=url))
return updater
def add(self, configuration):
"""Add an Updater based on the given Configuration."""
UPDATER_ADDED_MESSAGE = "Updater added for {configuration}."
repository_mirror_hostnames = self.__check_configuration(configuration)
# If all is well, build and store an Updater, and remember hostnames.
self.__updaters[configuration.hostname] = Updater(configuration)
self.__repository_mirror_hostnames.update(repository_mirror_hostnames)
Logger.info(UPDATER_ADDED_MESSAGE.format(configuration=configuration))
def __init__(self, configuration):
CREATED_TEMPDIR_MESSAGE = "Created temporary directory at {tempdir}"
self.configuration = configuration
# A temporary directory used for this updater over runtime.
self.tempdir = tempfile.mkdtemp()
Logger.debug(CREATED_TEMPDIR_MESSAGE.format(tempdir=self.tempdir))
# must switch context before instantiating updater
# because updater depends on some module (tuf.conf) variables
self.switch_context()
self.updater = tuf.client.updater.Updater(self.configuration.hostname,
self.configuration.repository_mirrors)
def __check_configuration(self, configuration):
"""
If the given Configuration is invalid, I raise an exception.
Otherwise, I return some information about the Configuration,
such as repository mirror hostnames.
"""
INVALID_REPOSITORY_MIRROR = "Invalid repository mirror {repository_mirror}!"
# Updater has a "global" view of configurations, so it performs
# additional checks after Configuration's own local checks.
assert isinstance(configuration, Configuration)
# Restrict each (incoming, outgoing) hostname pair to be unique across
# configurations; this prevents interposition cycles, amongst other
# things.
# GOOD: A -> { A:X, A:Y, B, ... }, C -> { D }, ...
# BAD: A -> { B }, B -> { C }, C -> { A }, ...
assert configuration.hostname not in self.__updaters
assert configuration.hostname not in self.__repository_mirror_hostnames
# Parse TUF server repository mirrors.
repository_mirrors = configuration.repository_mirrors
repository_mirror_hostnames = set()
for repository_mirror in repository_mirrors:
mirror_configuration = repository_mirrors[repository_mirror]
try:
url_prefix = mirror_configuration["url_prefix"]
parsed_url = urlparse.urlparse(url_prefix)
mirror_hostname = parsed_url.hostname
# Restrict each (incoming, outgoing) hostname pair to be unique
# across configurations; this prevents interposition cycles,
# amongst other things.
assert mirror_hostname not in self.__updaters
assert mirror_hostname not in self.__repository_mirror_hostnames
# Remember this mirror's hostname for the next network_location.
repository_mirror_hostnames.add(mirror_hostname)
except:
error_message = \
INVALID_REPOSITORY_MIRROR.format(repository_mirror=repository_mirror)
Logger.exception(error_message)
raise InvalidConfiguration(error_message)
return repository_mirror_hostnames
def __read_configuration(configuration_handler,
filename="tuf.interposition.json",
parent_repository_directory=None,
parent_ssl_certificates_directory=None):
"""
A generic function to read TUF interposition configurations off a file, and
then handle those configurations with a given function. configuration_handler
must be a function which accepts a tuf.interposition.Configuration
instance.
Returns the parsed configurations as a dictionary of configurations indexed
by hostnames."""
INVALID_TUF_CONFIGURATION = "Invalid configuration for {network_location}!"
INVALID_TUF_INTERPOSITION_JSON = "Invalid configuration in {filename}!"
NO_CONFIGURATIONS = "No configurations found in configuration in {filename}!"
# Configurations indexed by hostnames.
parsed_configurations = {}
try:
with open(filename) as tuf_interposition_json:
tuf_interpositions = json.load(tuf_interposition_json)
configurations = tuf_interpositions.get("configurations", {})
if len(configurations) == 0:
raise InvalidConfiguration(NO_CONFIGURATIONS.format(filename=filename))
else:
for network_location, configuration in configurations.iteritems():
try:
configuration_parser = ConfigurationParser(network_location,
configuration, parent_repository_directory=parent_repository_directory,
parent_ssl_certificates_directory=parent_ssl_certificates_directory)
configuration = configuration_parser.parse()
configuration_handler(configuration)
parsed_configurations[configuration.hostname] = configuration
except:
Logger.exception(INVALID_TUF_CONFIGURATION.format(network_location=network_location))
raise
except:
Logger.exception(INVALID_TUF_INTERPOSITION_JSON.format(filename=filename))
raise
else:
return parsed_configurations
def get_repository_mirrors(self, hostname, port, ssl_certificates):
"""Parse TUF server repository mirrors."""
INVALID_REPOSITORY_MIRROR = "Invalid repository mirror {repository_mirror}!"
repository_mirrors = self.configuration["repository_mirrors"]
repository_mirror_network_locations = set()
for repository_mirror in repository_mirrors:
mirror_configuration = repository_mirrors[repository_mirror]
try:
url_prefix = mirror_configuration["url_prefix"]
parsed_url = urlparse.urlparse(url_prefix)
mirror_hostname = parsed_url.hostname
mirror_port = parsed_url.port or 80
mirror_scheme = parsed_url.scheme
mirror_netloc = "{hostname}:{port}".format(hostname = mirror_hostname,
port = mirror_port)
# TODO: warn is ssl_certificates is specified,
# but there is no mirror_scheme == "https"
if mirror_scheme == "https":
assert os.path.isfile(ssl_certificates)
# No single-edge cycle in interposition.
# GOOD: A -> { A:XYZ, ... }
# BAD: A -> { A, ... }
assert not (mirror_hostname == hostname and mirror_port == port)
# Unique network location over repository mirrors.
# GOOD: A -> { A:X, A:Y, ... }
# BAD: A -> { A:X, A:X, ... }
assert mirror_netloc not in repository_mirror_network_locations
# Remember this mirror's network location to check the rest of the mirrors.
repository_mirror_network_locations.add(mirror_netloc)
except:
error_message = \
INVALID_REPOSITORY_MIRROR.format(repository_mirror=repository_mirror)
Logger.exception(error_message)
raise InvalidConfiguration(error_message)
return repository_mirrors
def remove(self, configuration):
"""Remove an Updater matching the given Configuration."""
UPDATER_REMOVED_MESSAGE = "Updater removed for {configuration}."
assert isinstance(configuration, Configuration)
repository_mirror_hostnames = configuration.get_repository_mirror_hostnames()
assert configuration.hostname in self.__updaters
assert repository_mirror_hostnames.issubset(self.__repository_mirror_hostnames)
# If all is well, remove the stored Updater as well as its associated
# repository mirror hostnames.
del self.__updaters[configuration.hostname]
self.__repository_mirror_hostnames.difference_update(repository_mirror_hostnames)
Logger.info(UPDATER_REMOVED_MESSAGE.format(configuration=configuration))
def get_target_filepath(self, source_url):
"""Given source->target map, figure out what TUF *should* download given a
URL."""
WARNING_MESSAGE = "Possibly invalid target_paths for " + \
"{network_location}! No TUF interposition for {url}"
parsed_source_url = urlparse.urlparse(source_url)
target_filepath = None
try:
# Does this source URL match any regular expression which tells us
# how to map the source URL to a target URL understood by TUF?
for target_path in self.configuration.target_paths:
# target_path: { "regex_with_groups", "target_with_group_captures" }
# e.g. { ".*(/some/directory)/$", "{0}/index.html" }
source_path_pattern, target_path_pattern = target_path.items()[0]
source_path_match = re.match(source_path_pattern, parsed_source_url.path)
# TODO: A failure in string formatting is *critical*.
if source_path_match is not None:
target_filepath = target_path_pattern.format(*source_path_match.groups())
# If there is more than one regular expression which
# matches source_url, we resolve ambiguity by order of
# appearance.
break
# If source_url does not match any regular expression...
if target_filepath is None:
# ...then we raise a predictable exception.
raise URLMatchesNoPattern(source_url)
except:
Logger.exception(WARNING_MESSAGE.format(
network_location=self.configuration.network_location, url=source_url))
raise
else:
# TUF assumes that target_filepath does not begin with a '/'.
target_filepath = target_filepath.lstrip('/')
return target_filepath
def retrieve(self, url, filename=None, reporthook=None, data=None):
INTERPOSITION_MESSAGE = "Interposing for {url}"
# TODO: set valid headers
content_type, content_encoding = mimetypes.guess_type(url)
headers = {"content-type": content_type}
Logger.info(INTERPOSITION_MESSAGE.format(url=url))
target_filepath = self.get_target_filepath(url)
temporary_directory, temporary_filename = self.download_target(target_filepath)
if filename is None:
# If no filename is given, use the temporary file.
filename = temporary_filename
else:
# Otherwise, copy TUF-downloaded file in its own directory
# to the location user specified.
shutil.copy2(temporary_filename, filename)
return filename, headers
def Configure(self, argv):
self.args = argv
self.messagePrinter = Logger()
self.scriptPath, self.scriptExtension = os.path.splitext(argv[0])
self.scriptPath, self.scriptName = os.path.split(self.scriptPath)
self.argparser.parse_args(args=argv[1:], namespace=self)
self.messagePrinter.isDbgEnabled = self.debugMessages
self.messagePrinter.isInfoEnabled = self.verbose
self.messagePrinter.isErrEnabled = not self.silenceErrors
if self.scriptIni is None:
iniPath = ''
if self.scriptPath is not None:
iniPath = self.scriptPath
self.scriptIni = toPosixPath(os.path.join(iniPath, '{0}.ini'.format(self.scriptName)))
self.parser = configparser.ConfigParser(allow_no_value = True) # see https://docs.python.org/2/library/configparser.html
self.parser.optionxform = str # make case-sensitive as per https://docs.python.org/2/library/configparser.html
self.isConfigured = True
if os.path.exists(self.scriptIni):
self.parser.read(self.scriptIni)
elif os.path.exists(os.path.join(self.scriptPath, self.scriptIni)):
self.parser.read(os.path.join(self.scriptPath, self.scriptIni))
else:
msg = "No configuration file found. Searched, current directory and script directory directory for {0}.".format(self.scriptIni)
self.messagePrinter.error(msg)
self.isConfigured = False
self.databaseFilename = ':memory:'
self.sourcePath = './'
return
try:
if self.databaseFilename is None:
self.databaseFilename = self.parser.get("Output", "DatabaseFilename")
except:
self.databaseFilename = ':memory:'
try:
if self.sourcePath is None:
self.sourcePath = self.parser.get("Paths","SourceRoot")
# Make the read SourceRoot path relative to the INI file's path.
if self.isConfigured:
iniPath, iniFilename = os.path.split(self.scriptIni)
self.sourcePath = toPosixPath(os.path.normpath(os.path.join(iniPath, self.sourcePath)))
print('source-path: {0}'.format(self.sourcePath))
except:
self.sourcePath = './'
def __check_configuration_on_add(self, configuration):
"""
If the given Configuration is invalid, I raise an exception.
Otherwise, I return some information about the Configuration,
such as repository mirror hostnames.
"""
INVALID_REPOSITORY_MIRROR = "Invalid repository mirror {repository_mirror}!"
# Updater has a "global" view of configurations, so it performs
# additional checks after Configuration's own local checks.
assert isinstance(configuration, Configuration)
# Restrict each (incoming, outgoing) hostname pair to be unique across
# configurations; this prevents interposition cycles, amongst other
# things.
# GOOD: A -> { A:X, A:Y, B, ... }, C -> { D }, ...
# BAD: A -> { B }, B -> { C }, C -> { A }, ...
assert configuration.hostname not in self.__updaters
assert configuration.hostname not in self.__repository_mirror_hostnames
# Check for redundancy in server repository mirrors.
repository_mirror_hostnames = configuration.get_repository_mirror_hostnames()
for mirror_hostname in repository_mirror_hostnames:
try:
# Restrict each hostname in every (incoming, outgoing) pair to be
# unique across configurations; this prevents interposition cycles,
# amongst other things.
assert mirror_hostname not in self.__updaters
assert mirror_hostname not in self.__repository_mirror_hostnames
except:
error_message = \
INVALID_REPOSITORY_MIRROR.format(repository_mirror=mirror_hostname)
Logger.exception(error_message)
raise InvalidConfiguration(error_message)
return repository_mirror_hostnames
class ImportPayPal(object):
'''
This class imports a PayPal CSV file into the database.
'''
def __init__(self, fname):
self.logger = Logger(self, Logger.DEBUG)
self.logger.debug(sys._getframe().f_code.co_name)
self.fname = fname
self.data = Database.get_instance()
self.accepted = 0
self.rejected = 0
self.legend = [
'Date', 'Time', 'TimeZone', 'Name', 'Type', 'Status', 'Currency',
'Gross', 'Fee', 'Net', 'FromEmail', 'ToEmail', 'TransactionID',
'ShippingAddress', 'AddressStatus', 'ItemTitle', 'ItemID',
'Shipping', 'InsuranceAmount', 'SalesTax', 'Option1Name',
'Option1Value', 'Option2Name', 'Option2Value', 'ReferenceTxnID',
'InvoiceNumber', 'CustomNumber', 'Quantity', 'ReceiptID',
'Balance', 'AddressLine1', 'AddressLine2', 'City', 'State',
'PostalCode', 'Country', 'Phone', 'Subject', 'Note', 'CountryCode',
'BalanceImpact'
]
@debugger
def import_all(self):
'''
This is the top level interface for the importer. All of the other methods
are private.
'''
self.logger.debug("import all records from %s" % (self.fname))
#try:
# import the CSV file into the database.
self._read_file()
# get the various tables set up
codes = self._countries()
cust = self._customers()
vend = self._vendors()
sales = self._sales()
purch = self._purchases()
text = 'Imported records:\n'
text += ' %d country codes\n' % (codes)
text += ' %d unique customer entries\n' % (cust)
text += ' %d unique vendor entries\n' % (vend)
text += ' %d sale entries\n' % (sales)
text += ' %d purchase entries\n' % (purch)
text += ' %d CSV lines accepted\n' % (self.accepted)
text += ' %d CSV lines rejected\n' % (self.rejected)
showinfo('Import', text)
# except Exception as e:
# showerror('ERROR', str(e))
@debugger
def _read_file(self):
'''
Read the CSV file into an array of lines where each one is a dictionary with the
column names as keys.
'''
with open(self.fname, "r") as fh:
reader = csv.reader(fh)
line = []
for line in reader:
break
if not line[1] == 'Time':
raise ('File selected is not a PayPal CSV import file.')
for line in reader:
rec = {}
for idx, item in enumerate(line):
#if item == '':
# rec[self.legend[idx]] = None
#else:
rec[self.legend[idx]] = item
rec['imported_country'] = False
rec['imported_customer'] = False
rec['imported_vendor'] = False
rec['imported_sale'] = False
rec['imported_purchase'] = False
if not self.data.if_rec_exists('RawImport', 'TransactionID',
rec['TransactionID']):
self.data.insert_row('RawImport', rec)
self.accepted += 1
else:
self.rejected += 1
self.data.commit()
@debugger
def _countries(self):
'''
Read the new import and copy new country codes into the country codes table.
'''
data = self.data.get_row_list('RawImport', 'imported_country = false')
if data is None:
return 0
count = 0
for item in data:
if item['CountryCode'] != '' and not self.data.if_rec_exists(
'Country', 'abbreviation', item['CountryCode']):
rec = {
'name': item['Country'],
'abbreviation': item['CountryCode']
}
self.data.insert_row('Country', rec)
count += 1
self.data.update_row_by_id('RawImport', {'imported_country': True},
item['ID'])
self.data.commit()
return count
@debugger
def _customers(self):
'''
Find all of the new customer records and copy the data into the customers table.
'''
data = self.data.get_row_list(
'RawImport',
'imported_customer = false and BalanceImpact = \'Credit\'')
if data is None:
showinfo('INFO', 'There are no customer contacts to import.')
return 0
count = 0
for item in data:
if item['Type'] == 'Website Payment' or item[
'Type'] == 'General Payment':
# Yes it's a customer
if not self.data.if_rec_exists('Customer', 'name',
item['Name']):
rec = {
'date_created':
item['Date'],
'name':
item['Name'],
'address1':
item['AddressLine1'],
'address2':
item['AddressLine2'],
'state':
item['State'],
'city':
item['City'],
'zip':
item['PostalCode'],
'email_address':
item['FromEmail'],
'email_status_ID':
self.data.get_id_by_row('EmailStatus', 'name',
'primary'),
'phone_number':
item['Phone'],
'phone_status_ID':
self.data.get_id_by_row('PhoneStatus', 'name',
'primary'),
'description':
'Imported from PayPal',
'notes':
item['Subject'],
'country_ID':
self.data.get_id_by_row('Country', 'abbreviation',
item['CountryCode']),
'class_ID':
self.data.get_id_by_row('ContactClass', 'name',
'retail')
}
self.data.insert_row('Customer', rec)
count += 1
# BUG: (fixed) When there are multiple instances of a name, the sale or purch record does not get imported
# because the imported_customer field does not get updated due to the duplicate name interlock.
self.data.update_row_by_id('RawImport',
{'imported_customer': True},
item['ID'])
self.data.commit()
return count
@debugger
def _vendors(self):
'''
Find all of the new vendor records and copy the data into the vendor table.
'''
data = self.data.get_row_list(
'RawImport',
'imported_vendor = false and BalanceImpact = \'Debit\'')
if data is None:
showinfo('INFO', 'There are no customer contacts to import.')
return 0
count = 0
for item in data:
if item['Name'] != '' and item['Name'] != 'PayPal':
if not self.data.if_rec_exists('Vendor', 'name', item['Name']):
rec = {
'date_created':
item['Date'],
'name':
item['Name'],
'contact_name':
'',
'email_address':
item['ToEmail'],
'email_status_ID':
self.data.get_id_by_row('EmailStatus', 'name',
'primary'),
'phone_number':
'',
'phone_status_ID':
self.data.get_id_by_row('PhoneStatus', 'name',
'primary'),
'description':
item['ItemTitle'],
'notes':
item['Subject'],
'type_ID':
self.data.get_id_by_row('VendorType', 'name',
'unknown'),
}
self.data.insert_row('Vendor', rec)
self.data.update_row_by_id('RawImport',
{'imported_vendor': True},
item['ID'])
count += 1
self.data.commit()
return count
@debugger
def _sales(self):
'''
Find the sales records and copy the data into the sales database table.
'''
data = self.data.get_row_list(
'RawImport',
'imported_sale = false and imported_customer = true and BalanceImpact = \'Credit\''
)
if data is None:
showinfo('INFO', 'There are no sales transcations to import.')
return 0
count = 0
for item in data:
if item['Name'] != '' and item['Name'] != 'PayPal':
rec = {
'date':
item['Date'],
'customer_ID':
self.data.get_id_by_row('Customer', 'name', item['Name']),
'raw_import_ID':
int(item['ID']),
'status_ID':
self.data.get_id_by_row('SaleStatus', 'name', 'complete'),
'transaction_uuid':
item['TransactionID'],
'gross':
self.data.convert_value(item['Gross'], float),
'fees':
self.data.convert_value(item['Fee'], float),
'shipping':
self.data.convert_value(item['Shipping'], float),
'notes':
item['Subject'] + '\n' + item['ItemTitle'],
'committed':
False
}
self.data.insert_row('SaleRecord', rec)
count += 1
self.data.update_row_by_id('RawImport',
{'imported_sale': True}, item['ID'])
self.data.commit()
return count
@debugger
def _purchases(self):
'''
Find all of the purchase records and copy the data into the purchase database table.
'''
data = self.data.get_row_list(
'RawImport',
'imported_purchase = false and imported_vendor = true and BalanceImpact = \'Debit\''
)
if data is None:
showinfo('INFO', 'There are no purchase transcations to import.')
return 0
count = 0
for item in data:
if item['Name'] != '' and item['Name'] != 'PayPal':
gross = item['Gross']
tax = item['SalesTax']
shipping = item['Shipping']
rec = {
'date':
item['Date'],
'raw_import_ID':
int(item['ID']),
'vendor_ID':
self.data.get_id_by_row('Vendor', 'name', item['Name']),
'status_ID':
self.data.get_id_by_row('PurchaseStatus', 'name',
'complete'),
'type_ID':
self.data.get_id_by_row('PurchaseType', 'name', 'unknown'),
'transaction_uuid':
item['TransactionID'],
'gross':
self.data.convert_value(item['Gross'], float),
'tax':
self.data.convert_value(item['SalesTax'], float),
'shipping':
self.data.convert_value(item['Shipping'], float),
'notes':
item['Subject'] + '\n' + item['ItemTitle'],
'committed':
False
}
self.data.insert_row('PurchaseRecord', rec)
self.data.update_row_by_id('RawImport',
{'imported_purchase': True},
item['ID'])
count += 1
self.data.commit()
return count
current_ts = ts
row['diff'] = abs(float(row['diff']))
if diff < 0:
borrowers.append(row)
else:
lenders_len = len(lenders)
if lenders_len < 1:
lenders.append(row)
else:
for i, l in enumerate(lenders):
if l['diff'] < row['diff']:
lenders.insert(i, row)
break
else:
if i == (lenders_len - 1):
lenders.insert(i+1, row)
break
# log(lenders)
# log('-----------------------')
# log(str(counter) + ':' + str(ts))
input_csv_file.close()
output_csv_file.close()
if __name__ == "__main__":
log.debug_off()
start = datetime.now()
main(sys.argv[1:])
print("Finished in ", datetime.now()-start, '.')
class EntryBox(tk.Frame):
'''
Implement a consistent interface to the entry widget
'''
def __init__(self,
master,
table,
column,
width=None,
readonly=False,
*args,
**kargs):
'''
master = the frame to bind this frame to
name = the text of the label
table = the name of the database table that is associated with this widget
column = the name of the column this widget associates with
lw = label width
cw = control width
'''
self.logger = Logger(self, level=Logger.INFO)
self.logger.debug("EntryBox enter constructor")
super().__init__(master, *args, **kargs)
self.table = table
self.column = column
self.readonly = readonly
self.content = tk.StringVar(master)
self.entry = tk.Entry(self, textvariable=self.content, width=width)
self.entry.grid(row=0, column=1, padx=5, pady=5, sticky=tk.W)
if self.readonly:
self.entry.configure(state='readonly')
self.row = {
'table': self.table,
'column': self.column,
'self': self,
'hasid': None
}
self.logger.debug("EntryBox leave constructor")
@debugger
def read(self):
return self.content.get()
@debugger
def write(self, val):
if self.readonly:
self.entry.configure(state='normal')
self.content.set(val)
if self.readonly:
self.entry.configure(state='readonly')
@debugger
def clear(self):
self.content.set('')
@debugger
def get_line(self):
'''
Return the form entry to update the form.
'''
return self.row
class ButtonBox(tk.Frame):
'''
Make the button box and register the events.
'''
def __init__(self,
master,
form,
disable_select=False,
disable_new=False,
*args,
**kargs):
'''
master = The frame to bind the widgets to.
form = Name of the form to bind the events to.
'''
self.logger = Logger(self, level=Logger.INFO)
self.logger.debug("NotesBox enter constructor")
super().__init__(master, *args, **kargs)
row = 0
col = 0
self.form = form
self.events = EventHandler.get_instance()
tk.Button(self, text='Prev', command=self.prev_btn).grid(row=row,
column=col,
padx=5,
pady=5)
col += 1
tk.Button(self, text='Next', command=self.next_btn).grid(row=row,
column=col,
padx=5,
pady=5)
if not disable_select:
col += 1
tk.Button(self, text='Select',
command=self.select_btn).grid(row=row,
column=col,
padx=5,
pady=5)
if not disable_new:
col += 1
tk.Button(self, text='New', command=self.new_btn).grid(row=row,
column=col,
padx=5,
pady=5)
col += 1
tk.Button(self, text='Save', command=self.save_btn).grid(row=row,
column=col,
padx=5,
pady=5)
col += 1
tk.Button(self, text='Delete',
command=self.delete_btn).grid(row=row,
column=col,
padx=5,
pady=5)
@debugger
def register_events(self, next, prev, select, new, save, delete):
'''
next = callback for the next button
prev = callback for the prev button
select = callback for the select button
new = callback for the new button
save = callback for the save button
delete = callback for the delete button
'''
self.events.register_event('next_btn_%s' % (self.form), next)
self.events.register_event('prev_btn_%s' % (self.form), prev)
self.events.register_event('select_btn_%s' % (self.form), select)
self.events.register_event('new_btn_%s' % (self.form), new)
self.events.register_event('save_btn_%s' % (self.form), save)
self.events.register_event('delete_btn_%s' % (self.form), delete)
@debugger
def next_btn(self):
self.events.raise_event('next_btn_%s' % (self.form))
@debugger
def prev_btn(self):
self.events.raise_event('prev_btn_%s' % (self.form))
@debugger
def select_btn(self):
self.events.raise_event('select_btn_%s' % (self.form))
@debugger
def new_btn(self):
self.events.raise_event('new_btn_%s' % (self.form))
@debugger
def save_btn(self):
self.events.raise_event('save_btn_%s' % (self.form))
@debugger
def delete_btn(self):
self.events.raise_event('delete_btn_%s' % (self.form))
scores_template = '../../out/scores/crowdai/scores_{}_{}_{}_{}_{}.out'
indices_template = '../../out/models/crowdai/indices_{}_{}_{}_{}_{}.out'
batch_size = 3
img_height = 96
img_width = 1366
channels = 1
num_epochs = 500
# Decaying by factor of ~0.91 after each epoch (for batch_size 6)
lr_starting = 9e-4
lr_decay = 0.9999714
start_time = time.time()
logger = Logger(batch_size, num_epochs, lr_starting)
score_data = dict()
score_data['train_loss'] = []
score_data['valid_loss'] = []
score_data['f1_score'] = []
score_data['lr'] = []
optimizers = {
'sgd': SGD(lr=0.001, momentum=0.9, nesterov=True),
'adam': Adam(lr=lr_starting, decay=lr_decay)
}
def build_model(output_size):
channel_axis = 3
freq_axis = 1
class SelectItem(BaseDialog):
'''
Create a list of items called 'name' from a table and return the database
ID of the item in item_id.
'''
def __init__(self, master, table, thing=None):
self.logger = Logger(self, level=Logger.DEBUG)
self.logger.debug('SelectItem enter constructor')
self.table = table
if thing is None:
self.thing = 'Item'
else:
self.thing = thing
self.item_id = -1
super().__init__(master)
self.wait_window(self)
self.logger.debug('SelectItem leave constructor')
@debugger
def body(self, master):
self.title("Select %s" % (self.thing))
self.data = Database.get_instance()
padx = 6
pady = 2
frame = tk.Frame(master, bd=1, relief=tk.RIDGE)
frame.grid(row=0, column=0, padx=4, pady=7)
tk.Label(frame,
text="Select %s" % (self.thing),
font=("Helvetica", 14)).grid(row=0, column=0, columnspan=2)
######################
# Populate the combo boxes
lst = self.data.populate_list(self.table, 'name')
lst.sort()
######################
# Show the boxes
tk.Label(frame, text='Name:').grid(row=1, column=0)
self.cbb = ttk.Combobox(frame, values=lst)
self.cbb.grid(row=1, column=1, padx=padx, pady=pady)
try:
self.cbb.current(0)
except tk.TclError:
mb.showerror("TCL ERROR",
"No records are available to select for this table.")
@debugger
def validate(self):
# Since the name was selected from the list, there is no need to
# validate.
return True
@debugger
def apply(self):
''' Populate the form with the selected data. '''
id = self.data.get_id_by_name(self.table, self.cbb.get())
self.item_id = id
def main(svc_input, configs):
logger = Logger("查询日志", verbose=True)
log_file_name = "log%s_%s.txt" % (svc_input.replace("?", "#"), DateTimeUtil.get_current_datetime(is_date=True))
log_file_path = WindowsUtil.convert_win_path(os.path.join(temp_dir, log_file_name))
logger.info("[开始查询] %s" % svc_input)
try:
# 找到本地匹配的保修历史记录
history_zip = ZipFileSVC(zip_file_path=history_zipfile, mode='a')
start_time = DateTimeUtil.get_current_datetime()
# 创建出所有可能查询码
svc_generator = SVCGenerator(svc_input, logger)
logger.info("创建出所有可能查询码：%s" % len(svc_generator.target_svc_set))
# 根据本地匹配的非法查询码历史，筛选出目标查询码，以及非法查询码
existed_svc = history_zip.find_file_regex(svc_generator.regex)
svc_generator.generate_target_svc_batch(existed_svc, invalid_history_file_path)
# 调用戴尔查询API，并将API数据转化为实体类数据
output_dell_asset_list = list([])
if svc_generator.target_svc_set:
batch = Batch(logger, configs)
api_dell_asset_list = batch.begin(svc_generator.target_svc_set)
output_dell_asset_list = api_dell_asset_list
logger.info("从API中总共得到%s个结果" % (len(api_dell_asset_list)))
logger.info("将实体类序列化到本地临时TXT文件")
temp_text_files_path = DellAsset.serialize_txt_batch(api_dell_asset_list, temp_dir)
logger.info("将序列化临时文件存到本地zip历史记录，总数：%s" % len(temp_text_files_path))
history_zip.add_new_file_batch(temp_text_files_path)
logger.info("删除临时 %s 个TXT文件" % len(temp_text_files_path))
for file_path in temp_text_files_path:
FileUtil.delete_file(file_path)
logger.info("将API得到的实体类和历史记录实体类合并")
else:
logger.warn("目标查询码为空，仅从从历史记录中导出结果")
for svc in svc_generator.existed_svc_set:
dell_asset_content = history_zip.get_member_content(file_name="%s.txt" % svc)
output_dell_asset_list.append(DellAsset.deserialize_txt(dell_asset_content))
logger.info("添加历史记录，总共得到%s个结果" % (len(output_dell_asset_list)))
excel_output_path = WindowsUtil.convert_win_path(os.path.join(excel_dir, "%s.xlsx" % svc_generator.get_file_name()))
DellAsset.save_as_excel_batch(output_dell_asset_list, excel_output_path)
if FileUtil.is_path_existed(excel_output_path):
logger.info("存为Excel文档成功")
end_time = DateTimeUtil.get_current_datetime()
logger.info("总用时 %s " % DateTimeUtil.datetime_diff(start_time, end_time))
logger.info("[查询结束] 总共%s个结果 保存在：%s" % (len(output_dell_asset_list), excel_output_path))
else:
logger.error("[保存结果失败] %s" % excel_output_path)
except Exception as e:
# 若程序出现错误失败，发送邮件
logger.error("[查询失败] 已发送报告 请等待解决")
logger.error("%s\n%s" % (e, traceback.format_exc()))
logger.save(log_file_path)
email_api_key = configs["email_api_key"]
email = Email(email_api_key, subject="[查询失败] %s %s" % (DateTimeUtil.get_current_datetime(is_date=True), svc_input))
email.add_attachment(log_file_path)
email.send(cc_mode=logger.has_error)
class Importer(object):
def __init__(self): #, master):
self.logger = Logger(self, Logger.DEBUG)
self.logger.debug(sys._getframe().f_code.co_name)
#self.master = master
self.legend = [
'Date', 'Time', 'TimeZone', 'Name', 'Type', 'Status', 'Currency',
'Gross', 'Fee', 'Net', 'FromEmail', 'ToEmail', 'TransactionID',
'ShippingAddress', 'AddressStatus', 'ItemTitle', 'ItemID',
'Shipping', 'InsuranceAmount', 'SalesTax', 'Option1Name',
'Option1Value', 'Option2Name', 'Option2Value', 'ReferenceTxnID',
'InvoiceNumber', 'CustomNumber', 'Quantity', 'ReceiptID',
'Balance', 'AddressLine1', 'AddressLine2', 'City', 'State',
'PostalCode', 'Country', 'Phone', 'Subject', 'Note', 'CountryCode',
'BalanceImpact'
]
self.data = Database.get_instance()
@debugger
def import_all(self):
''' Perform all of the steps to import the entire CSV file '''
total = self.read_all()
if total > 0:
#data = self.get_sales()
codes = self.import_country_codes()
cust = self.import_customer_contacts()
sales = self.do_sales_transactions()
#data = self.get_purchases()
#codes += self.import_country_codes()
vend = self.import_vendor_contacts()
purch = self.do_purchase_transactions()
text = 'Imported records:\n'
text += ' %d total lines in import\n' % (total)
text += ' %d country codes\n' % (codes)
text += ' %d unique customer entries\n' % (cust)
text += ' %d unique vendor entries\n' % (vend)
text += ' %d sale entries\n' % (sales)
text += ' %d purchase entries' % (purch)
mb.showinfo('Import', text)
else:
mb.showinfo("Import", 'There were no records found to import.')
@debugger
def validate_email(self, email):
if bool(re.search(r"^[\w\.\+\-]+\@[\w]+\.[a-z]{2,3}$", email)):
return email
else:
return ''
@debugger
def read_file(self):
''' Handle the file IO for the CSV file '''
#master = tkinter.Tk()
fh = askopenfile(mode='r',
filetypes=[('Spread Sheets', '*.csv *.CSV'),
('All Files', '*')])
if fh is None:
return None # error or cancel
raw = []
reader = csv.reader(fh)
for line in reader:
raw.append(line)
fh.close()
#master.destroy()
#print(raw)
return (raw, fh.name)
@debugger
def read_all(self):
''' Read all of the lines in the CSV into the database '''
raw_data = self.read_file()
if raw_data is None:
return 0
raw = raw_data[0]
name = raw_data[1]
if self.data.if_rec_exists('ImportedFileNames', 'name', name):
mb.showwarning(
'WARNING',
'File name \"%s\" has already been imported. Rename file and try again.'
% (name))
return 0
else:
tmp = {
'name': name,
'date': time.strftime('%m/%d/%Y', time.localtime())
}
self.data.insert_row('ImportedFileNames', tmp)
data = []
count = 0
for line in raw:
tmp = {}
for idx, item in enumerate(line):
tmp[self.legend[idx]] = item
data.append(tmp)
if tmp['Status'] == 'Completed': # and tmp['Name'] != '' and tmp['Name'] != 'PayPal':
tmp['imported_country'] = False
tmp['imported_customer'] = False
tmp['imported_vendor'] = False
tmp['imported_sale'] = False
tmp['imported_purchase'] = False
if not self.data.if_rec_exists('RawImport', 'TransactionID',
tmp['TransactionID']):
self.data.insert_row('RawImport', tmp)
count += 1
self.data.commit()
return count
@debugger
def import_country_codes(self):
''' Find the new country codes and store them '''
data = self.data.get_row_list('RawImport', 'imported_country = false')
if data is None:
return
count = 0
for item in data:
if item['CountryCode'] != '' and not self.data.if_rec_exists(
'Country', 'abbreviation', item['CountryCode']):
rec = {
'name': item['Country'],
'abbreviation': item['CountryCode']
}
self.data.insert_row('Country', rec)
count += 1
self.data.update_row_by_id('RawImport', {'imported_country': True},
item['ID'])
self.data.commit()
return count
@debugger
def import_customer_contacts(self):
''' Store contact information for customers '''
data = self.data.get_row_list(
'RawImport',
'imported_customer = false and BalanceImpact = \'Credit\'')
if data is None:
mb.showinfo('INFO', 'There are no customer contacts to import.')
return
count = 0
for item in data:
if item['Type'] == 'Website Payment' or item[
'Type'] == 'General Payment':
# Yes it's a customer
if not self.data.if_rec_exists('Customer', 'name',
item['Name']):
rec = {
'date_created':
item['Date'],
'name':
item['Name'],
'address1':
item['AddressLine1'],
'address2':
item['AddressLine2'],
'state':
item['State'],
'city':
item['City'],
'zip':
item['PostalCode'],
'email_address':
item['FromEmail'],
'email_status_ID':
self.data.get_id_by_row('EmailStatus', 'name',
'primary'),
'phone_number':
item['Phone'],
'phone_status_ID':
self.data.get_id_by_row('PhoneStatus', 'name',
'primary'),
'description':
'Imported from PayPal',
'notes':
item['Subject'],
'country_ID':
self.data.get_id_by_row('Country', 'abbreviation',
item['CountryCode']),
'class_ID':
self.data.get_id_by_row('ContactClass', 'name',
'retail')
}
self.data.insert_row('Customer', rec)
count += 1
# BUG: (fixed) When there are multiple instances of a name, the sale or purch record does not get imported
# because the imported_customer field does not get updated due to the duplicate name interlock.
self.data.update_row_by_id('RawImport',
{'imported_customer': True},
item['ID'])
self.data.commit()
#mb.showinfo('INFO', 'Imported %d customer contacts.'%(count))
return count
@debugger
def import_vendor_contacts(self):
''' Store contact information for vendors '''
data = self.data.get_row_list(
'RawImport',
'imported_vendor = false and BalanceImpact = \'Debit\'')
if data is None:
mb.showinfo('INFO', 'There are no customer contacts to import.')
return
count = 0
for item in data:
if item['Name'] != '' and item['Name'] != 'PayPal':
if not self.data.if_rec_exists('Vendor', 'name', item['Name']):
rec = {
'date_created':
item['Date'],
'name':
item['Name'],
'contact_name':
'',
'email_address':
self.validate_email(item['ToEmail']),
'email_status_ID':
self.data.get_id_by_row('EmailStatus', 'name',
'primary'),
'phone_number':
'',
'phone_status_ID':
self.data.get_id_by_row('PhoneStatus', 'name',
'primary'),
'description':
item['ItemTitle'],
'notes':
item['Subject'],
'type_ID':
self.data.get_id_by_row('VendorType', 'name',
'unknown'),
}
self.data.insert_row('Vendor', rec)
self.data.update_row_by_id('RawImport',
{'imported_vendor': True},
item['ID'])
count += 1
self.data.commit()
#mb.showinfo('INFO', 'Imported %d vendor contacts.'%(count))
return count
@debugger
def do_purchase_transactions(self):
''' Handle all of the transaction details for purchases '''
data = self.data.get_row_list(
'RawImport',
'imported_purchase = false and imported_vendor = true and BalanceImpact = \'Debit\''
)
if data is None:
mb.showinfo('INFO',
'There are no purchase transcations to import.')
return
count = 0
for item in data:
if item['Name'] != '' and item['Name'] != 'PayPal':
gross = item['Gross']
tax = item['SalesTax']
shipping = item['Shipping']
rec = {
'date':
item['Date'],
'raw_import_ID':
int(item['ID']),
'vendor_ID':
self.data.get_id_by_row('Vendor', 'name', item['Name']),
'status_ID':
self.data.get_id_by_row('PurchaseStatus', 'name',
'complete'),
'type_ID':
self.data.get_id_by_row('PurchaseType', 'name', 'unknown'),
'transaction_uuid':
item['TransactionID'],
'gross':
self.data.convert_value(item['Gross'], float),
'tax':
self.data.convert_value(item['SalesTax'], float),
'shipping':
self.data.convert_value(item['Shipping'], float),
'notes':
item['Subject'] + '\n' + item['ItemTitle'],
'committed':
False
}
self.data.insert_row('PurchaseRecord', rec)
self.data.update_row_by_id('RawImport',
{'imported_purchase': True},
item['ID'])
count += 1
self.data.commit()
#mb.showinfo('INFO', 'Imported %d purchase transactions.'%(count))
return count
@debugger
def do_sales_transactions(self):
''' Do all sales transactions '''
data = self.data.get_row_list(
'RawImport',
'imported_sale = false and imported_customer = true and BalanceImpact = \'Credit\''
)
if data is None:
mb.showinfo('INFO', 'There are no sales transcations to import.')
return
count = 0
for item in data:
if item['Name'] != '' and item['Name'] != 'PayPal':
rec = {
'date':
item['Date'],
'customer_ID':
self.data.get_id_by_row('Customer', 'name', item['Name']),
'raw_import_ID':
int(item['ID']),
'status_ID':
self.data.get_id_by_row('SaleStatus', 'name', 'complete'),
'transaction_uuid':
item['TransactionID'],
'gross':
self.data.convert_value(item['Gross'], float),
'fees':
self.data.convert_value(item['Fee'], float),
'shipping':
self.data.convert_value(item['Shipping'], float),
'notes':
item['Subject'] + '\n' + item['ItemTitle'],
'committed':
False
}
self.data.insert_row('SaleRecord', rec)
count += 1
self.data.update_row_by_id('RawImport',
{'imported_sale': True}, item['ID'])
self.data.commit()
#mb.showinfo('INFO', 'Imported %d sale transactions.'%(count))
return count
#!/usr/bin/python3
import tkinter
import traceback, sys
from main_frame import MainFrame
from utility import Logger
#from data_store import DataStore
#from configuration import Configuration
if __name__ == "__main__":
try:
logger = Logger(__name__, Logger.DEBUG)
logger.debug(sys._getframe().f_code.co_name)
top = tkinter.Tk()
app = MainFrame(top)
top.wm_title("Shop Timer")
logger.debug("start main loop")
top.mainloop()
logger.debug("end main loop")
except Exception as e:
traceback.print_exception(*sys.exc_info())
class LowerFrame(tkinter.Frame):
'''
This class manages the lower from of the display.
'''
def __init__(self, master):
self.logger = Logger(self, Logger.INFO)
self.logger.debug("constructor")
self.data_store = DataStore.get_instance()
self.master = master
#self.line_data = []
self.line_widgits = []
register_event('UPDATE_LOWER_FRAME_EVENT', self.update_frame)
register_event('UPDATE_LINES_EVENT', self.set_state)
register_event('UPDATE_NOTES_EVENT', self.update_notes)
register_event('CHANGE_UNITS_EVENT', self.change_units)
tkinter.Label(self.master, width=12,
text="Hole").grid(row=0, column=0, sticky=tkinter.W)
tkinter.Label(self.master, width=5,
text="Interval").grid(row=0, column=1, sticky=tkinter.W)
tkinter.Label(self.master, width=10,
text="Note").grid(row=0, column=2, sticky=tkinter.W)
tkinter.Label(self.master, width=10,
text="Frequency").grid(row=0, column=3, sticky=tkinter.W)
tkinter.Label(self.master, width=18,
text="Hole Size").grid(row=0, column=4, sticky=tkinter.W)
tkinter.Label(self.master, width=11,
text="Hole Location").grid(row=0,
column=5,
sticky=tkinter.W)
tkinter.Label(self.master, width=12,
text="Hole Diff").grid(row=0, column=6, sticky=tkinter.W)
tkinter.Label(self.master, width=12,
text="Cutoff Freq").grid(row=0,
column=7,
sticky=tkinter.W)
# create all of the lines
for n in range(12):
lw = LineWidgit(self.master, n)
self.line_widgits.append(lw)
for idx in range(self.data_store.get_number_holes()):
self.line_widgits[idx].grid(row=idx + 1,
column=0,
columnspan=8,
sticky=tkinter.W)
self.line_widgits[idx].set_state()
self.logger.debug("end constructor")
@debugger
def update_frame(self):
# hide all of the lines
for idx in range(12):
self.line_widgits[idx].grid_forget()
# expose the correct number of lines
for idx in range(self.data_store.get_number_holes()):
self.line_widgits[idx].grid(row=idx + 1,
column=0,
columnspan=8,
sticky=tkinter.W)
self.line_widgits[idx].set_state()
self.update_notes()
@debugger
def set_state(self):
'''
Copy data from the datastore into all lines in GUI
'''
for line in self.line_widgits:
line.set_state()
@debugger
def get_state(self):
'''
Copy data from the line widgits into the data_store
'''
for line in self.line_widgits:
line.get_state()
@debugger
def update_notes(self):
sel = self.data_store.get_bell_note_select()
for index, line in enumerate(self.line_widgits):
sel += self.data_store.get_hole_interval(index)
self.data_store.set_hole_freq(
index, self.data_store.note_table[sel]["frequency"])
self.data_store.set_hole_note(
index, self.data_store.note_table[sel]["note"])
line.set_state()
raise_event("CALCULATE_EVENT")
@debugger
def change_units(self):
'''
When this is called, it is assumed that the data_store has the wrong
units. The values are converted to the current units and the GUI
is updated.
'''
for line in self.line_widgits:
line.change_units()
class Form: #(tk.Frame):
'''
Base class for a form. This contains everything needed to display a form and
associate the elements of a form with a database record. This allows a complete
form to be added to a notebook panel.
A single generic driver routine is used to get or set the form with the database.
The idea is to capture all of the calls to tkinter having to do with the forms.
'''
# widgets that may be added. Used internally to identify them.
TITLE = 0
ENTRY = 1
TEXT = 2
COMBO = 3
LABEL = 4
BUTTON = 5
# used by the supplemental_form class in setup_forms.py
INDIRECT_LABEL = 6
COMMIT_BTN = 7
PRODUCT = 8
DIR_BROWSER = 9
IMPORT_BTN = 10
def __init__(self,
notebook,
index,
table=None,
height=700,
width=1000,
span=4):
'''
Create a form object. A form must be associated with exactly one database table. The form
is filled in with exactly one row from the database table. If the form does not use all
of the row, that is handled transparently.
This class contains frames. It is not, itself, a frame.
notebook = The notebook container
index = The index of the tab to bind the form to
table = The database table where the data from the form is located
height = The height of the form in pixels
width = the width of the form in pixels
span = The number of columns in the form
'''
self.logger = Logger(self, level=Logger.DEBUG)
self.logger.debug("enter constructor")
self.notebook = notebook # container for the form.
self.nb_index = index # index of the tab to bind the form to
self.table = table # database table to save and retrieve data
# frame to bind the widgets to
self.owner = self.notebook.get_frame(self.nb_index)
# load_form should be over ridden to handle special cases.
self.notebook.set_callback(self.nb_index, self.load_form)
# database singleton object
self.data = Database.get_instance()
# the rows may not be sequentially numbered.
if not table is None:
self.row_list = self.data.get_id_list(self.table)
self.row_index = 0
else:
self.row_list = None
self.row_index = 0
# dict of dicts that lists all of the controls by name.
self.controls = {}
# actual height and width of the form frame.
self.height = height
self.width = width
self.logger.debug("window size = %d, %d" % (self.height, self.width))
# keep track of the current layout position.
self.row = 0
self.col = 0
self.btn_row = 0
self.btn_width = 10
self.padx = 5
self.pady = 5
self.btn_padx = 1
self.btn_pady = 1
self.span = span # number of columns for controls
self.ctrl_width = 60 # default control width in chars
self.text_height = 20 # default text height
# create the frames
#self.base_frame = tk.Frame(self.owner)
#self.ctl_frame = ScrollableFrame(self.base_frame, height, width).get_frame()
self.ctl_frame = ScrollableFrame(self.owner, height, width).get_frame()
#self.ctl_frame = ScrollableFrame(self.base_frame).get_frame()
#self.ctl_frame = tk.Frame(self.base_frame)
#self.btn_frame = tk.Frame(self.base_frame)
self.btn_frame = tk.Frame(self.owner)
self.btn_frame.grid(row=0, column=0, sticky='es')
self.ctl_frame.grid(row=0, column=1, sticky='wn')
#self.base_frame.grid(row=0, column=0)
self.logger.debug("leave constructor")
@debugger
def add_title(self, name):
'''
Add the form title. This is not a tracked item.
'''
lab = tk.Label(self.ctl_frame, text=name, font=("Helvetica", 14))
lab.grid(row=0,
column=0,
padx=self.padx,
pady=self.pady,
columnspan=self.span,
sticky=(tk.E, tk.W))
self.row += 1
@debugger
def add_entry(self,
name,
column,
_type,
inc_row=True,
readonly=False,
span=None,
**kargs):
'''
Add an entry line to the form. The name is displayed as the label and is also used
to access the control when reading it or writing to it.
Kargs are passed directly to the widget constructor.
name = The name to use to access the control object.
column = Name of the database column.
_type = The type of data stored in the database.
inc_row = whether to increment the row after adding this line.
'''
if not 'width' in kargs:
kargs['width'] = self.ctrl_width
self.logger.debug("kargs = %s" % (str(kargs)))
lab = tk.Label(self.ctl_frame, text=name + ':')
strvar = tk.StringVar(self.ctl_frame)
item = tk.Entry(self.ctl_frame, textvariable=strvar, **kargs)
if readonly:
item.configure(state='readonly')
lab.grid(row=self.row,
column=self.col,
padx=self.padx,
pady=self.pady,
sticky=tk.E)
self.col += 1
if span is None:
span = self.span - 1
item.grid(row=self.row,
column=self.col,
padx=self.padx,
pady=self.pady,
columnspan=span,
sticky=tk.W)
if inc_row:
self.row += 1
self.col = 0
else:
self.col += 1
def getter():
return _type(strvar.get())
def setter(s):
state = item.configure()['state']
if state == 'readonly':
item.configure(state='normal')
strvar.set(str(s))
if state == 'readonly':
item.configure(state='readonly')
def clear():
state = item.configure()['state']
if state == 'readonly':
item.configure(state='normal')
strvar.set('')
if state == 'readonly':
item.configure(state='readonly')
self.controls[name] = {
'column': column,
'obj': item,
'get': getter,
'set': lambda s: setter(s),
'clear': clear,
'type': _type,
'kind': Form.ENTRY
}
@debugger
def add_text(self, name, column, inc_row=True, **kargs):
'''
Add a multi-line text entry box with scroll bars to the form. The name is displayed
as the label and is also used to access the control when reading it or writing to it.
Kargs are passed directly to the widget constructor.
name = The name of the widgit and the label text
column = the column in the database to retrieve
inc_row = whether to increment the row after adding the widget
'''
lab = tk.Label(self.ctl_frame, text=name + ':')
if not 'width' in kargs:
kargs['width'] = self.ctrl_width
if not 'height' in kargs:
kargs['height'] = self.text_height
self.logger.debug("kargs = %s" % (str(kargs)))
frame = tk.Frame(self.ctl_frame, bd=1, relief=tk.RIDGE)
text = tk.Text(frame, wrap=tk.NONE, **kargs)
text.insert(tk.END, '')
# see https://www.homeandlearn.uk/tkinter-scrollbars.html
self.vsb = tk.Scrollbar(frame, orient=tk.VERTICAL)
self.vsb.config(command=text.yview)
text.config(yscrollcommand=self.vsb.set)
self.vsb.pack(side=tk.RIGHT, fill=tk.Y)
self.hsb = tk.Scrollbar(frame, orient=tk.HORIZONTAL)
self.hsb.config(command=text.xview)
text.config(xscrollcommand=self.hsb.set)
self.hsb.pack(side=tk.BOTTOM, fill=tk.X)
text.pack(side=tk.LEFT)
lab.grid(row=self.row,
column=self.col,
padx=self.padx,
pady=self.pady,
sticky=tk.E)
self.col += 1
frame.grid(row=self.row,
column=self.col,
padx=self.padx,
pady=self.pady,
columnspan=self.span - 1,
sticky=tk.W)
if inc_row:
self.row += 1
self.col = 0
else:
self.col += 1
def getter():
return text.get(1.0, tk.END)
def setter(s):
text.delete('1.0', tk.END)
if not s is None:
text.insert(tk.END, str(s))
def clear():
text.delete('1.0', tk.END)
self.controls[name] = {
'column': column,
'obj': text,
'get': getter,
'set': lambda s: setter(s),
'clear': clear,
'kind': Form.TEXT
}
@debugger
def add_combo(self, name, table, column, inc_row=True, **kargs):
'''
Add a combo box line to the form. The name is displayed as the label and is also used
to access the control when reading it or writing to it. Combo boxes use a different table
to store their data in. This widgit automatically retrieves that data when it is displayed.
The type is always strings.
Kargs are passed directly to the widget constructor.
name = the name of the row and the label text
table = the name of the table to retrieve the data for the combo box
column = the name of the column to retrieve the value to set the box to
inc_row = whether to increment the row after adding the widget
'''
lab = tk.Label(self.ctl_frame, text=name + ':')
if not 'width' in kargs:
kargs['width'] = self.ctrl_width
# Note that the method that displays the frame has to populate this.
combo = ttk.Combobox(self.ctl_frame, state='readonly', **kargs)
lab.grid(row=self.row,
column=self.col,
padx=self.padx,
pady=self.pady,
sticky=tk.E)
self.col += 1
combo.grid(row=self.row,
column=self.col,
padx=self.padx,
pady=self.pady,
sticky=tk.W)
if inc_row:
self.row += 1
self.col = 0
else:
self.col += 1
def getter():
return combo.current() + 1
def setter(s):
combo.current(int(s) - 1)
def clear():
try:
combo.current(0)
except tk.TclError:
pass # empty content is not an error
def populate():
combo['values'] = self.data.populate_list(table, 'name')
self.controls[name] = {
'column': column,
'table': table,
'obj': combo,
'get': getter,
'set': lambda s: setter(s),
'clear': clear,
'populate': populate,
'kind': Form.COMBO
}
@debugger
def add_dynamic_label(self, name, column, inc_row=True, **kargs):
'''
Add a label that can be changed according to a database column.
'''
lab = tk.Label(self.ctl_frame, text=name + ':')
value = tk.StringVar(self.ctl_frame)
val = tk.Label(self.ctl_frame, textvariable=value, **kargs)
lab.grid(row=self.row,
column=self.col,
padx=self.padx,
pady=self.pady,
sticky=tk.E)
self.col += 1
val.grid(row=self.row,
column=self.col,
padx=self.padx,
pady=self.pady,
sticky=tk.W)
if inc_row:
self.row += 1
self.col = 0
else:
self.col += 1
def getter():
return value.get()
def setter(s):
value.set(str(s))
def clear():
value.set('')
self.controls[name] = {
'column': column,
'obj': lab,
'get': getter,
'set': lambda s: setter(s),
'clear': clear,
'kind': Form.LABEL
}
@debugger
def add_notebook(self, nb_class):
'''
Add a notebook to the form.
'''
#nb_class(self.base_frame)
nb_class(self.owner)
@debugger
def add_button(self, name, command=None, **kargs):
'''
Add a button to the button line on the form. The name is used as the label on the
button. If a callback is not provided, then a default callback will be run. Buttons are
added in the order they are created and arranged accroding default patterns.
Args and kargs are passed directly to the widget constructor.
'''
if command is None:
if name == "Next":
command = self.next_callback
elif name == "Prev":
command = self.prev_callback
elif name == "Select":
command = self.select_callback
elif name == "New":
command = self.new_callback
elif name == "Save":
command = self.save_callback
elif name == "Delete":
command = self.delete_callback
else:
raise "unknown name and no command to exec"
btn = tk.Button(self.btn_frame,
text=name,
command=command,
width=self.btn_width,
**kargs)
#btn.pack(padx=self.btn_padx, pady=self.btn_pady, side=tk.TOP)
btn.grid(row=self.btn_row,
column=0,
padx=self.btn_padx,
pady=self.btn_pady)
self.btn_row += 1
@debugger
def get_form(self):
'''
Return a dictionary of all of the values that are currently in the form. Form fields are
returned in the type that will be stored in the database.
'''
values = {}
for item in self.controls:
values[item] = self.get_form_field(item)
return values
@debugger
def set_form(self, values):
'''
Accept a dictionary of all of the values for the form and make them visible. If a
form field is different from the type that will be stored in the database, then the
value will be changed to the correct type.
'''
for item in values:
self.set_form_field(item, values[item])
@debugger
def clear_form(self):
'''
Clear the form with blank/default values.
'''
for item in self.controls:
self.clear_form_field(item)
@debugger
def get_form_field(self, name):
'''
Return the contents of a form field in the type that the form stores. If the database
has stored an int, then an int is returned, rather than a string. If the field is a
combo box, then setting the values is expected to be an array of text to populate the
control.
'''
return self.controls[name]['get']()
@debugger
def set_form_field(self, name, value):
'''
Set the value of the field according to the type that the field stores. The type will
be automatically converted from a string, if the type to be stored in the is different.
'''
self.controls[name]['set'](value)
@debugger
def clear_form_field(self, name):
'''
Clear the form field to the default value.
'''
self.controls[name]['clear']()
@debugger
def commit_form(self):
'''
Write the contents of the form to the database. The form does not need to be
visible for this to take place. Nothing is written to the database until this method
is called. This assumes that all of the data is in the same table.
'''
if not self.row_list is None:
vals = {}
row_id = self.row_list[self.row_index]
for item in self.controls:
cval = self.controls[item]['get']()
col = self.controls[item]['column']
if not cval is None and col != '':
vals[col] = cval
if self.data.if_rec_exists(self.table, 'ID', row_id):
self.data.update_row(self.table, vals, "ID=%d" % (row_id))
else:
self.data.insert_row(self.table, vals)
self.data.commit()
self.row_list = self.data.get_id_list(self.table)
else:
self.logger.debug("Form has no table")
@debugger
def load_form(self):
'''
Read the contents of the form from the database and place the values in the
form controls. If the form is currently visible, then display the values.
'''
if not self.row_list is None:
try:
row_id = self.row_list[self.row_index]
row = self.data.get_row_by_id(self.table, row_id)
for item in self.controls:
if self.controls[item]['kind'] == Form.COMBO:
self.controls[item]['populate']()
self.controls[item]['set'](
row[self.controls[item]['column']])
elif self.controls[item]['kind'] == Form.PRODUCT:
self.controls[item]['set'](row_id)
elif self.controls[item]['kind'] == Form.DIR_BROWSER:
self.controls[item]['set']()
elif self.controls[item]['kind'] == Form.IMPORT_BTN:
self.controls[item]['set']()
else:
self.controls[item]['set'](
row[self.controls[item]['column']])
except IndexError as e:
showerror(
'No Records',
'No records exist for this form.\n\nThere are %d records in the table.'
% (len(self.row_list)))
@debugger
def configure_obj(self, name, **kargs):
'''
Call "configure" on an underlying object with the parameters indicated.
'''
self.controls[name]['obj'].configure(**kargs)
@debugger
def prev_callback(self):
'''
Default callback for the Prev button.
'''
if not self.row_list is None:
self.row_index -= 1
if self.row_index < 0:
self.row_index = 0
showinfo('First Record', 'This is the first record.')
else:
self.load_form()
@debugger
def next_callback(self):
'''
Default callback for the next button.
'''
if not self.row_list is None:
self.row_index += 1
if self.row_index > len(self.row_list) - 1:
self.row_index = len(self.row_list) - 1
showinfo('Last Record', 'This is the last record.')
else:
self.load_form()
@debugger
def select_callback(self):
'''
Default callback for the select button. This displays a dialog of the "Name" field
of all of the rows that are defined in the table. If the name field does not exist,
throw an exception.
'''
if not self.row_list is None:
item = SelectItem(self.owner, self.table, 'name')
self.row_index = self.row_list.index(item.item_id)
self.load_form()
@debugger
def new_callback(self):
'''
Default callback for the "New" button. This clears the form to default values.
'''
for item in self.controls:
self.controls[item]['clear']()
@debugger
def save_callback(self):
'''
Default callback for the "Save" button. This writes the for to the database.
'''
if askyesno('Save record?', 'Are you sure you want to save this?'):
self.commit_form()
@debugger
def delete_callback(self):
'''
Default callback for the delete button. This deletes the current row from the
database and displays the next one. If deleting the last row, then the first
row is displayed.
'''
if askyesno('Delete record?', 'Are you sure you want to delete this?'):
self.data.delete_row(self.table, self.row_list[self.row_index])
self.data.commit()
@debugger
def set_layout_row(self, num):
'''
Set the layout row to the given number.
'''
self.row = num
@debugger
def set_layout_col(self, num):
'''
Set the layout column to the given number.
'''
self.col = num
@debugger
def get_layout_row(self):
'''
Returnthe layout row.
'''
return self.row
@debugger
def get_layout_col(self):
'''
Return the layout column.
'''
return self.col
class ScrollableFrame:
'''
This is the scrollable frame class that is used by the notebook class. It
could be used generically, but it was designed for use here.
The scrolling functionality supports both a scroll bar as well as the
mouse wheel.
'''
def __init__(self, container, height=800, width=1000, *args, **kwargs):
self.logger = Logger(self, level=Logger.INFO)
self.logger.debug("enter constructor")
self.canvas = tk.Canvas(container, height=height, width=width)
self.canvas.grid(row=0, column=0, sticky='news')
self.scrollbar = tk.Scrollbar(container,
orient="vertical",
command=self.canvas.yview)
self.scrollbar.grid(row=0, column=1, sticky='nsw')
self.canvas.configure(yscrollcommand=self.scrollbar.set)
self.canvas.configure(yscrollincrement='20')
self.canvas.configure(scrollregion=self.canvas.bbox("all"))
self.scrollwindow = tk.Frame(self.canvas)
self.canvas.create_window((0, 0),
window=self.scrollwindow,
anchor="nw")
self.scrollwindow.bind("<Configure>", self.configure_window)
self.scrollwindow.bind("<Enter>", self.enter_handler)
self.scrollwindow.bind("<Leave>", self.leave_handler)
self.scrollwindow.bind('<Button-4>', self.mouse_wheel)
self.scrollwindow.bind('<Button-5>', self.mouse_wheel)
self.canvas.focus_set()
self.logger.debug("leave constructor")
def get_frame(self):
''' Get the master frame for embedded widgets. '''
#print('here')
return self.scrollwindow
# PORTABILITY! This may not be the same on every platform. It works under
# linux using xfce. Other must be tested. The problem that this solves is
# that mousewheel events are only routed to the widget that the mouse is
# pointing to. If the mouse is hovering over a label or something, the
# canvas never gets the event. Surprisingly, the enter and leave events
# are both sent when the mousewheel is rolled. I am sure that this is an
# "unsupported" feature. If the mousewheel stops working, this is where
# to look for a fix.
def enter_handler(self, event):
#print('enter', event.state)
state = (event.state & 0x1800) >> 11
direction = 0
if state == 2:
direction = 1
if state == 1:
direction = -1
#print(direction)
self.canvas.yview_scroll(direction, tk.UNITS)
def leave_handler(self, event):
#print('leave', event.state)
pass
@debugger
def configure_window(self, event):
#print('here', self.mark)
self.canvas.configure(scrollregion=self.canvas.bbox("all"))
def mouse_wheel(self, event):
direction = 0
if event.num == 5:
direction = 1
if event.num == 4:
direction = -1
#print(direction)
self.canvas.yview_scroll(direction, tk.UNITS)
def __init__(self,
notebook,
index,
table=None,
height=700,
width=1000,
span=4):
'''
Create a form object. A form must be associated with exactly one database table. The form
is filled in with exactly one row from the database table. If the form does not use all
of the row, that is handled transparently.
This class contains frames. It is not, itself, a frame.
notebook = The notebook container
index = The index of the tab to bind the form to
table = The database table where the data from the form is located
height = The height of the form in pixels
width = the width of the form in pixels
span = The number of columns in the form
'''
self.logger = Logger(self, level=Logger.DEBUG)
self.logger.debug("enter constructor")
self.notebook = notebook # container for the form.
self.nb_index = index # index of the tab to bind the form to
self.table = table # database table to save and retrieve data
# frame to bind the widgets to
self.owner = self.notebook.get_frame(self.nb_index)
# load_form should be over ridden to handle special cases.
self.notebook.set_callback(self.nb_index, self.load_form)
# database singleton object
self.data = Database.get_instance()
# the rows may not be sequentially numbered.
if not table is None:
self.row_list = self.data.get_id_list(self.table)
self.row_index = 0
else:
self.row_list = None
self.row_index = 0
# dict of dicts that lists all of the controls by name.
self.controls = {}
# actual height and width of the form frame.
self.height = height
self.width = width
self.logger.debug("window size = %d, %d" % (self.height, self.width))
# keep track of the current layout position.
self.row = 0
self.col = 0
self.btn_row = 0
self.btn_width = 10
self.padx = 5
self.pady = 5
self.btn_padx = 1
self.btn_pady = 1
self.span = span # number of columns for controls
self.ctrl_width = 60 # default control width in chars
self.text_height = 20 # default text height
# create the frames
#self.base_frame = tk.Frame(self.owner)
#self.ctl_frame = ScrollableFrame(self.base_frame, height, width).get_frame()
self.ctl_frame = ScrollableFrame(self.owner, height, width).get_frame()
#self.ctl_frame = ScrollableFrame(self.base_frame).get_frame()
#self.ctl_frame = tk.Frame(self.base_frame)
#self.btn_frame = tk.Frame(self.base_frame)
self.btn_frame = tk.Frame(self.owner)
self.btn_frame.grid(row=0, column=0, sticky='es')
self.ctl_frame.grid(row=0, column=1, sticky='wn')
#self.base_frame.grid(row=0, column=0)
self.logger.debug("leave constructor")
def main(argv):
workspace = ''
# datasets = {
# 'pecanstreet': ['California', 'Austin', 'New York'],
# 'eGauge':['Colorado']
# }
datasets = {
'pecanstreet': ['California'],
'eGauge':['Colorado']
}
data_years = {
'California': '2015', # 2015 full year for PecanStreet California dataset UTC
'Austin': '2018', # 2018 full year for PecanStreet Austin dataset UTC
'New York': '2019', # 2019/5/1-10/31 half year for PecanStreet New York dataset UTC
'Colorado': '2015' # 2015 full year for eGauge dataset
}
discharge_speed = '100'
for dataset in datasets:
for location in datasets[dataset]:
print(f'Start simulate {location} by LRF...')
input_path = workspace + 'data/' + dataset + '/' + location + '/' + data_years[location] + '.csv'
output_path = workspace + 'data/' + dataset + '/' + location + '/logs/LRF.csv'
# init csv file header
output_csv_header = ['timestamp', 'datetime', 'from', 'to', 'amount', 'type']
# type: share, grid, own
with open(output_path, 'w', newline='') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=output_csv_header)
writer.writeheader()
csvfile.close()
lenders = []
borrowers = []
discharge_rate = int(discharge_speed)
df = pd.read_csv(input_path)
total = len(df)
counter = 0
current_ts = 0
# house_info_path = workspace + 'data/metadata.csv'
house_info_path =workspace + 'data/' + dataset + '/' + location + '/metadata.csv'
discharge_rates = {}
# Init charge rate list
with open(house_info_path) as house_info_csv_file:
reader = csv.DictReader(house_info_csv_file)
for row in reader:
discharge_rates[row['house_id']] = discharge_rate
house_info_csv_file.close()
with open(output_path, 'a', newline='') as output_csv_file:
writer = csv.writer(output_csv_file)
with open(input_path) as input_csv_file:
reader = csv.DictReader(input_csv_file)
for row in reader:
# Skip empty row
diff = float(row['diff'])
if diff == 0:
continue
# Get timestamp
ts = int(float(row['timestamp']))
# Init current timestamp at beginning
if counter == 0:
current_ts = ts
# Increase counter
counter += 1
# process = str(counter) + '/' + str(total) + ' (' + str(round(counter/total*100, 2)) + '%)'
# print(process, end='\r')
if ts != current_ts:
for bidx, borrower in enumerate(borrowers):
if len(lenders) <= 0:
log.print('No lender is available.')
break
# 1st. Check if can use own battery power first
for idx, lender in enumerate(lenders):
if lender['house_id'] == borrower['house_id']:
if discharge_rates[lender['house_id']] <= 0:
continue
# Borrow amount greater than own discharge rate
if borrower['diff'] >= discharge_rates[lender['house_id']]:
# Power provided by own battery is greater than discharge rate, then use discharge rate amount, keep rest for sharing
if lender['diff'] > discharge_rates[lender['house_id']]:
log.print('Use own: b>=d, l>d')
borrower['diff'] -= discharge_rates[lender['house_id']]
borrowers[bidx] = borrower
lender['diff'] -= discharge_rates[lender['house_id']]
# lenders[idx] = lender
lenders.remove(lender)
lenders_len = len(lenders)
if lenders_len < 1:
lenders.append(lender)
else:
for i, l in enumerate(lenders):
if l['diff'] < lender['diff']:
lenders.insert(i, lender)
break
else:
if i == (lenders_len - 1):
lenders.insert(i+1, lender)
break
writer.writerow([current_ts, borrower['datetime'], lender['house_id'], borrower['house_id'], discharge_rates[lender['house_id']], 'own'])
discharge_rates[lender['house_id']] = 0
break
# Own battery cannot provide power greater than discharge rate, use up all and withdraw sharing
else:
log.print('Use own: b>=d, l=<d')
borrower['diff'] -= lender['diff']
borrowers[bidx] = borrower
lenders.remove(lender)
discharge_rates[lender['house_id']] -= lender['diff']
writer.writerow([current_ts, borrower['datetime'], lender['house_id'], borrower['house_id'], lender['diff'], 'own'])
# Borrow amount less than own discharge rate
else:
if borrower['diff'] >= lender['diff']:
log.print('own: b<d, b>=l')
borrower['diff'] -= lender['diff']
borrowers[bidx] = borrower
lenders.remove(lender)
discharge_rates[lender['house_id']] -= lender['diff']
writer.writerow([current_ts, borrower['datetime'], lender['house_id'], borrower['house_id'], lender['diff'], 'own'])
else:
log.print('own: b<d, b<l')
lender['diff'] -= borrower['diff']
# lenders[idx] = lender
lenders.remove(lender)
lenders_len = len(lenders)
if lenders_len < 1:
lenders.append(lender)
else:
for i, l in enumerate(lenders):
if l['diff'] < lender['diff']:
lenders.insert(i, lender)
break
else:
if i == (lenders_len - 1):
lenders.insert(i+1, lender)
break
discharge_rates[lender['house_id']] -= borrower['diff']
writer.writerow([current_ts, borrower['datetime'], lender['house_id'], borrower['house_id'], borrower['diff'], 'own'])
borrower['diff'] = 0
borrowers[bidx] = borrower
if borrower['diff'] == 0:
break
elif borrower['diff'] < 0:
log.print('Error: borrowing amount is negative!')
# 2. Borrow from other lenders
# if len(lenders) < 1:
# log('No lender is available.')
# break
lender_index = 0
while borrower['diff'] > 0:
# log(borrower['diff'])
# log(str(lender_index) + '/' + str(len(lenders)))
if lender_index < len(lenders):
lender = lenders[lender_index]
if discharge_rates[lender['house_id']] <= 0:
lender_index += 1
continue
if lender['house_id'] == borrower['house_id']:
lender_index += 1
continue
# lend_amount = abs(float(lenders[lender_index]['diff']))
# log(lender['diff'])
# Borrow amount greater than own discharge rate
if borrower['diff'] >= discharge_rates[lender['house_id']]:
# Power provided by lender's battery is greater than discharge rate, then use discharge rate amount, keep rest for sharing
if lender['diff'] > discharge_rates[lender['house_id']]:
log.print('Share: b>=d, l>d')
borrower['diff'] -= discharge_rates[lender['house_id']]
borrowers[bidx] = borrower
lender['diff'] -= discharge_rates[lender['house_id']]
# lenders[lender_index] = lender
lenders.remove(lender)
lenders_len = len(lenders)
if lenders_len < 1:
lenders.append(lender)
else:
for i, l in enumerate(lenders):
if l['diff'] < lender['diff']:
lenders.insert(i, lender)
break
else:
if i == (lenders_len - 1):
lenders.insert(i+1, lender)
break
writer.writerow([current_ts, borrower['datetime'], lender['house_id'], borrower['house_id'], discharge_rates[lender['house_id']], 'share'])
discharge_rates[lender['house_id']] = 0
# Own battery cannot provide power greater than discharge rate, use up all and withdraw sharing
else:
log.print('Share: b>=d, l=<d')
borrower['diff'] -= lender['diff']
borrowers[bidx] = borrower
lenders.remove(lender)
discharge_rates[lender['house_id']] -= lender['diff']
writer.writerow([current_ts, borrower['datetime'], lender['house_id'], borrower['house_id'], lender['diff'], 'share'])
# Borrow amount less than lender's discharge rate
else:
if borrower['diff'] >= lender['diff']:
log.print('Share: b<d, b>=l')
borrower['diff'] -= lender['diff']
borrowers[bidx] = borrower
lenders.remove(lender)
discharge_rates[lender['house_id']] -= lender['diff']
writer.writerow([current_ts, borrower['datetime'], lender['house_id'], borrower['house_id'], lender['diff'], 'share'])
else:
log.print('Share: b<d, b<l')
lender['diff'] -= borrower['diff']
# lenders[lender_index] = lender
lenders.remove(lender)
lenders_len = len(lenders)
if lenders_len < 1:
lenders.append(lender)
else:
for i, l in enumerate(lenders):
if l['diff'] < lender['diff']:
lenders.insert(i, lender)
break
else:
if i == (lenders_len - 1):
lenders.insert(i+1, lender)
break
discharge_rates[lender['house_id']] -= borrower['diff']
writer.writerow([current_ts, borrower['datetime'], lender['house_id'], borrower['house_id'], borrower['diff'], 'share'])
borrower['diff'] = 0
borrowers[bidx] = borrower
lender_index += 1
# No lenders available, get from grid
else:
log.print('grid')
writer.writerow([current_ts, borrower['datetime'], 0, borrower['house_id'], borrower['diff'], 'grid'])
borrower['diff'] = 0
break
if borrower['diff'] == 0:
break
elif borrower['diff'] < 0:
log.print('Error: borrowing amount is negative!')
# Reset dicharge rate list
for dr in discharge_rates:
discharge_rates[dr] = discharge_rate
# Reset borrowers list
borrowers = []
# Sum up power left in batteries
battery_remain = 0
for l in lenders:
battery_remain += l['diff']
if battery_remain > 0:
dt = datetime.fromtimestamp(current_ts)
writer.writerow([current_ts, dt, '', '', battery_remain, 'battery_remain'])
current_ts = ts
row['diff'] = abs(float(row['diff']))
if diff < 0:
borrowers.append(row)
else:
lenders_len = len(lenders)
if lenders_len < 1:
lenders.append(row)
else:
for i, l in enumerate(lenders):
if l['diff'] < row['diff']:
lenders.insert(i, row)
break
else:
if i == (lenders_len - 1):
lenders.insert(i+1, row)
break
# log(lenders)
# log('-----------------------')
# log(str(counter) + ':' + str(ts))
input_csv_file.close()
output_csv_file.close()
def __init__(self, notebook):
self.logger = Logger(self, Logger.DEBUG)
self.logger.debug(sys._getframe().f_code.co_name)
super().__init__(notebook, notebook.SETUP_FRAME, 'Business')
self.add_notebook(SetupNotebook)
class LineWidgit(tkinter.Frame):
'''
This is the GUI widget that represents a single line in the output
data. It uses the data_store to communicate values into and out of
itself.
'''
def __init__(self, parent,lineno):
self.logger = Logger(self, Logger.INFO)
self.logger.debug("constructor")
tkinter.Frame.__init__(self, parent)
self.data_store = DataStore.get_instance()
self.index = lineno
self.name = "Hole %d" % (lineno+1)
self.line_name = tkinter.Label(self, text=self.name, width=12)
self.line_name.grid(row=lineno+1, column=0, sticky=tkinter.W)
self.inter_ctl = tkinter.Entry(self, width=5, validate="focusout", validatecommand=self.change_interval)
self.inter_ctl.bind('<Return>', self.change_interval)
self.inter_ctl.bind('<Tab>', self.change_interval)
self.inter_ctl.grid(row=lineno+1, column=1)
self.note_ctl_txt = tkinter.StringVar()
self.note_ctl = tkinter.Label(self, textvariable=self.note_ctl_txt, width=12)
self.note_ctl.grid(row=lineno+1, column=2)
self.freq_ctl_txt = tkinter.StringVar()
self.freq_ctl = tkinter.Label(self, textvariable=self.freq_ctl_txt, width=12)
self.freq_ctl.grid(row=lineno+1, column=3)
self.hole_ctl = HoleSizeWidgit(self, lineno)
self.hole_ctl.config(padx=25)
self.hole_ctl.grid(row=lineno+1, column=4)
self.locat_ctl_txt = tkinter.StringVar()
self.locat_ctl = tkinter.Label(self, textvariable=self.locat_ctl_txt, width=12)
self.locat_ctl.grid(row=lineno+1, column=5)
self.diff_ctl_txt = tkinter.StringVar()
self.diff_ctl = tkinter.Label(self, textvariable=self.diff_ctl_txt, width=12)
self.diff_ctl.grid(row=lineno+1, column=6)
self.cutoff_ctl_txt = tkinter.StringVar()
self.cutoff_ctl = tkinter.Label(self, textvariable=self.cutoff_ctl_txt, width=12)
self.cutoff_ctl.grid(row=lineno+1, column=7)
self.set_state()
self.logger.debug("end constructor")
@debugger
def set_state(self):
'''
Place the data from the data_store into the GUI.
'''
self.inter_ctl.delete(0, tkinter.END)
self.inter_ctl.insert(0, str(self.data_store.get_hole_interval(self.index)))
self.note_ctl_txt.set(str(self.data_store.get_hole_note(self.index))) # Label
self.freq_ctl_txt.set("%s Hz"%(str(self.data_store.get_hole_freq(self.index))))
self.locat_ctl_txt.set("%0.4f"%self.data_store.get_hole_xloc(self.index)) # Label
self.diff_ctl_txt.set("%0.4f"%self.data_store.get_hole_diff(self.index)) # Label
self.cutoff_ctl_txt.set("%0.4f"%self.data_store.get_hole_cutoff(self.index)) # Label
self.hole_ctl.set_state()
@debugger
def get_state(self):
'''
Get the data out of the display and place it in the data_store.
'''
self.data_store.set_hole_interval(self.index, int(self.inter_ctl.get()))
self.data_store.set_hole_note(self.index, self.note_ctl_txt.get()) # str
self.data_store.set_hole_freq(self.index, float(self.freq_ctl_txt.get().split()[0]))
self.data_store.set_hole_location(self.index, float(self.locat_ctl_txt.get()))
self.data_store.set_hole_diff(self.index, float(self.diff_ctl_txt.get()))
self.data_store.set_hole_cutoff(self.index, float(self.cutoff_ctl_txt.get()))
self.hole_ctl.get_state()
@debugger
def print_state(self):
self.logger.msg(str(self.get_state()))
@debugger
def change_units(self):
'''
When this is called, it is assumed that the datastore and the GUI need to have
the vaules updated to reflect the new units.
'''
if self.data_store.get_units():
self.data_store.set_hole_size(self.index, utility.in_to_mm(self.data_store.get_hole_size(self.index)))
self.data_store.set_hole_location(self.index, utility.in_to_mm(self.data_store.get_hole_location(self.index)))
self.data_store.set_hole_diff(self.index, utility.in_to_mm(self.data_store.get_hole_diff(self.index)))
self.data_store.set_hole_cutoff(self.index, utility.in_to_mm(self.data_store.get_hole_cutoff(self.index)))
else:
self.data_store.set_hole_size(self.index, utility.mm_to_in(self.data_store.get_hole_size(self.index)))
self.data_store.set_hole_location(self.index, utility.mm_to_in(self.data_store.get_hole_location(self.index)))
self.data_store.set_hole_diff(self.index, utility.mm_to_in(self.data_store.get_hole_diff(self.index)))
self.data_store.set_hole_cutoff(self.index, utility.mm_to_in(self.data_store.get_hole_cutoff(self.index)))
self.set_state()
self.data_store.set_change_flag()
@debugger
def change_interval(self, event=None):
try:
val = int(self.inter_ctl.get())
oldval = self.data_store.get_hole_interval(self.index)
if val != oldval:
if val > 0 and val < 5:
self.data_store.set_hole_interval(self.index, val)
self.logger.debug("change interval from %d to %d"%(oldval, val))
raise_event("UPDATE_NOTES_EVENT")
self.data_store.set_change_flag()
else:
self.logger.error("invalid value for interval: %s"%(str(self.inter_ctl.get())))
messagebox.showerror("ERROR", "Intervals must be an integer between 1 and 4")
self.inter_ctl.delete(0, tkinter.END)
self.inter_ctl.insert(0, str(self.data_store.get_hole_interval(self.index)))
return False
else:
self.logger.debug("ignore")
except ValueError:
self.logger.error("invalid integer for interval: %s"%(str(self.inter_ctl.get())))
messagebox.showerror("ERROR", "Cannot convert the string \"%s\" to an integer between 1 1nd 4"%(self.inter_ctl.get()))
self.inter_ctl.delete(0, tkinter.END)
self.inter_ctl.insert(0, str(self.data_store.get_hole_interval(self.index)))
return False
except IndexError:
pass # always ignore
return True
class DependencyScriptConfiguration(object):
@staticmethod
def GetConfigurationFilename(scriptFilename):
path, ext = os.path.splitext(argv[0])
path, name = os.path.split(path)
return '{0}.ini'.format(name)
def __init__(self, argv=None, argparser=None):
# Make an argparser if we were not given one already.
if argparser is None:
self.argparser = argparse.ArgumentParser(description='Generates an SQLite3 database containing all of the #include dependencies in your project.')
else:
self.argparser = argparser
# Add our arguments to the argparser.
self.argparser.add_argument('-C', '--print-example-config', action='store_true', default=False, dest='printExampleConfig', help='The script requires a configuration file in the current directory or the script directory in order to run. This option generates an example config file that lists all the available options. Note: the databases must have the same filename in the config file. You will also want to make sure that the default filename, :memory:, is no longer set. If it was then you won\'t have a file to work with. Read the comments in the example config for details.')
self.argparser.add_argument('--debug', action='store_true', default=False, dest='debugMessages', help='Print the debug messages. Very verbose, use only in development.')
self.argparser.add_argument('--verbose', action='store_true', default=False, dest='verbose', help='Print the info messages about progress and status.')
self.argparser.add_argument('--silence-errors', action='store_true', default=False, dest='silenceErrors', help='Don\'t print any errors.')
self.argparser.add_argument('-c', '--config-filename', dest='scriptIni', metavar='<config-filename>', help='Specifies the path to the configuration file to use.')
self.argparser.add_argument('-f', '--database-filename', dest='databaseFilename', metavar='<db-filename>', help='Specifies the filename of the database. Note: specifying this option overrides the filename in the configuration file.')
self.argparser.add_argument('-s', '--source-path', dest='sourcePath', metavar='<source-path>', help='Specifies the path to the root of the source code.')
# Configure the rest of our class. We need to initialize unused variables if we want to use
# them later in our class.
if argv is not None:
self.Configure(argv)
else:
self.ClearConfiguration()
def Configure(self, argv):
self.args = argv
self.messagePrinter = Logger()
self.scriptPath, self.scriptExtension = os.path.splitext(argv[0])
self.scriptPath, self.scriptName = os.path.split(self.scriptPath)
self.argparser.parse_args(args=argv[1:], namespace=self)
self.messagePrinter.isDbgEnabled = self.debugMessages
self.messagePrinter.isInfoEnabled = self.verbose
self.messagePrinter.isErrEnabled = not self.silenceErrors
if self.scriptIni is None:
iniPath = ''
if self.scriptPath is not None:
iniPath = self.scriptPath
self.scriptIni = toPosixPath(os.path.join(iniPath, '{0}.ini'.format(self.scriptName)))
self.parser = configparser.ConfigParser(allow_no_value = True) # see https://docs.python.org/2/library/configparser.html
self.parser.optionxform = str # make case-sensitive as per https://docs.python.org/2/library/configparser.html
self.isConfigured = True
if os.path.exists(self.scriptIni):
self.parser.read(self.scriptIni)
elif os.path.exists(os.path.join(self.scriptPath, self.scriptIni)):
self.parser.read(os.path.join(self.scriptPath, self.scriptIni))
else:
msg = "No configuration file found. Searched, current directory and script directory directory for {0}.".format(self.scriptIni)
self.messagePrinter.error(msg)
self.isConfigured = False
self.databaseFilename = ':memory:'
self.sourcePath = './'
return
try:
if self.databaseFilename is None:
self.databaseFilename = self.parser.get("Output", "DatabaseFilename")
except:
self.databaseFilename = ':memory:'
try:
if self.sourcePath is None:
self.sourcePath = self.parser.get("Paths","SourceRoot")
# Make the read SourceRoot path relative to the INI file's path.
if self.isConfigured:
iniPath, iniFilename = os.path.split(self.scriptIni)
self.sourcePath = toPosixPath(os.path.normpath(os.path.join(iniPath, self.sourcePath)))
print('source-path: {0}'.format(self.sourcePath))
except:
self.sourcePath = './'
def ClearConfiguration(self):
self.args = None
self.parser = None
self.scriptName = None
self.scriptPath = None
self.scriptExtension = None
self.scriptIni = None
def main(svc_input, configs):
logger = Logger("查询日志", verbose=True)
log_file_name = "log%s_%s.txt" % (svc_input.replace(
"?", "#"), DateTimeUtil.get_current_datetime(is_date=True))
log_file_path = WindowsUtil.convert_win_path(
os.path.join(temp_dir, log_file_name))
logger.info("[开始查询] %s" % svc_input)
try:
# 找到本地匹配的保修历史记录
history_zip = ZipFileSVC(zip_file_path=history_zipfile, mode='a')
start_time = DateTimeUtil.get_current_datetime()
# 创建出所有可能查询码
svc_generator = SVCGenerator(svc_input, logger)
logger.info("创建出所有可能查询码：%s" % len(svc_generator.target_svc_set))
# 根据本地匹配的非法查询码历史，筛选出目标查询码，以及非法查询码
existed_svc = history_zip.find_file_regex(svc_generator.regex)
svc_generator.generate_target_svc_batch(existed_svc,
invalid_history_file_path)
# 调用戴尔查询API，并将API数据转化为实体类数据
output_dell_asset_list = list([])
if svc_generator.target_svc_set:
batch = Batch(logger, configs)
api_dell_asset_list = batch.begin(svc_generator.target_svc_set)
output_dell_asset_list = api_dell_asset_list
logger.info("从API中总共得到%s个结果" % (len(api_dell_asset_list)))
logger.info("将实体类序列化到本地临时TXT文件")
temp_text_files_path = DellAsset.serialize_txt_batch(
api_dell_asset_list, temp_dir)
logger.info("将序列化临时文件存到本地zip历史记录，总数：%s" %
len(temp_text_files_path))
history_zip.add_new_file_batch(temp_text_files_path)
logger.info("删除临时 %s 个TXT文件" % len(temp_text_files_path))
for file_path in temp_text_files_path:
FileUtil.delete_file(file_path)
logger.info("将API得到的实体类和历史记录实体类合并")
else:
logger.warn("目标查询码为空，仅从从历史记录中导出结果")
for svc in svc_generator.existed_svc_set:
dell_asset_content = history_zip.get_member_content(
file_name="%s.txt" % svc)
output_dell_asset_list.append(
DellAsset.deserialize_txt(dell_asset_content))
logger.info("添加历史记录，总共得到%s个结果" % (len(output_dell_asset_list)))
excel_output_path = WindowsUtil.convert_win_path(
os.path.join(excel_dir, "%s.xlsx" % svc_generator.get_file_name()))
DellAsset.save_as_excel_batch(output_dell_asset_list,
excel_output_path)
if FileUtil.is_path_existed(excel_output_path):
logger.info("存为Excel文档成功")
end_time = DateTimeUtil.get_current_datetime()
logger.info("总用时 %s " %
DateTimeUtil.datetime_diff(start_time, end_time))
logger.info("[查询结束] 总共%s个结果 保存在：%s" %
(len(output_dell_asset_list), excel_output_path))
else:
logger.error("[保存结果失败] %s" % excel_output_path)
except Exception as e:
# 若程序出现错误失败，发送邮件
logger.error("[查询失败] 已发送报告 请等待解决")
logger.error("%s\n%s" % (e, traceback.format_exc()))
logger.save(log_file_path)
email_api_key = configs["email_api_key"]
email = Email(
email_api_key,
subject="[查询失败] %s %s" %
(DateTimeUtil.get_current_datetime(is_date=True), svc_input))
email.add_attachment(log_file_path)
email.send(cc_mode=logger.has_error)
class UpperFrame(tkinter.Frame):
'''
This class manages the upper frame of the display.
'''
def __init__(self, master):
self.logger = Logger(self, Logger.INFO)
self.logger.debug("constructor")
self.master = master
self.data_store = DataStore.get_instance()
register_event("UPDATE_UPPER_EVENT", self.set_state)
@debugger
def create_frame(self):
# build the screen
# Fill in the upper frame
tkinter.Label(self.master, text="Title").grid(row=0, column=0, sticky=tkinter.E)
self.titleEntry = tkinter.Entry(self.master, width=40, validate="focusout", validatecommand=self.setTitleCommand)
self.titleEntry.bind('<Return>', self.setTitleCommand)
self.titleEntry.bind('<Tab>', self.setTitleCommand)
self.titleEntry.grid(row=0, column=1, columnspan=3, padx=9, pady=4)
tkinter.Label(self.master, text="Inside Diameter").grid(row=1, column=0, sticky=tkinter.E, pady=4)
self.insideDiaEntry = tkinter.Entry(self.master, validate="focusout", validatecommand=self.insideDiaCommand)
self.insideDiaEntry.bind('<Return>', self.insideDiaCommand)
self.insideDiaEntry.bind('<Tab>', self.insideDiaCommand)
self.insideDiaEntry.grid(row=1, column=1, pady=4)
tkinter.Label(self.master, text="Wall Thickness").grid(row=1, column=2, sticky=tkinter.E, pady=4)
self.wallThicknessEntry = tkinter.Entry(self.master, validate="focusout", validatecommand=self.wallThicknessCommand)
self.wallThicknessEntry.bind('<Return>', self.wallThicknessCommand)
self.wallThicknessEntry.bind('<Tab>', self.wallThicknessCommand)
self.wallThicknessEntry.grid(row=1, column=3, pady=4)
tkinter.Label(self.master, text="Number of Holes").grid(row=2, column=0, sticky=tkinter.E, pady=4)
self.numHolesEntry = tkinter.Entry(self.master, validate="focusout", validatecommand=self.numHolesCommand)
self.numHolesEntry.bind('<Return>', self.numHolesCommand)
self.numHolesEntry.bind('<Tab>', self.numHolesCommand)
self.numHolesEntry.grid(row=2, column=1, pady=4)
tkinter.Label(self.master, text="Select Bell Note").grid(row=2, column=2, sticky=tkinter.E, pady=4)
self.bellNoteCombo = ttk.Combobox(self.master, state="readonly", values=self.data_store.bellNoteArray)
self.bellNoteCombo.config(width=17)
self.bellNoteCombo.grid(row=2, column=3, pady=4)
self.bellNoteCombo.bind("<<ComboboxSelected>>", self.bellSelectCallback)
tkinter.Label(self.master, text="Embouchure Area").grid(row=4, column=0, sticky=tkinter.E, pady=4)
self.embouchureAreaEntry = tkinter.Entry(self.master)
self.embouchureAreaEntry.grid(row=4, column=1, pady=4)
tkinter.Label(self.master, text="Units of Measure").grid(row=3, column=0, sticky=tkinter.E, pady=4)
self.measureUnitsOpt = ttk.Combobox(self.master, state="readonly", values=["inch", "mm"])
self.measureUnitsOpt.config(width=17)
self.measureUnitsOpt.grid(row=3, column=1, pady=4)
self.measureUnitsOpt.bind("<<ComboboxSelected>>", self.measureUnitsCallback)
tkinter.Label(self.master, text="Display Format").grid(row=3, column=2, sticky=tkinter.E, pady=4)
self.displayFormatOpt = ttk.Combobox(self.master, state="readonly", values=["decimal", "fraction"])
self.displayFormatOpt.current(1)
self.displayFormatOpt.config(width=17)
self.displayFormatOpt.grid(row=3, column=3, pady=4)
self.displayFormatOpt.bind("<<ComboboxSelected>>", self.displayFormatCallback)
tkinter.Label(self.master, text="Length").grid(row=4, column=2, sticky=tkinter.E, pady=4)
self.lengthEntry = tkinter.Entry(self.master)
self.lengthEntry.grid(row=4, column=3, pady=4)
if self.measureUnitsOpt.get() == 'mm':
self.displayFormatOpt.config(state="readonly")
self.refreshButton = tkinter.Button(
self.master, text="Refresh", width=14, command=self.refreshButtonCommand)
self.refreshButton.grid(row=5, column=0, columnspan=4, pady=4)
self.set_state() # write what's in the data_store to the GUI
@debugger
def get_state(self):
'''
Return the state of the controls in the upper half into the data store.
'''
if self.displayFormatOpt.current() == 0:
self.data_store.set_disp_frac(False)
else:
self.data_store.set_disp_frac(True)
if self.measureUnitsOpt.current() == 0:
self.data_store.set_units(False)
else:
self.data_store.set_units(True)
self.data_store.set_title(self.titleEntry.get())
self.data_store.set_inside_dia(float(self.insideDiaEntry.get()))
self.data_store.set_wall_thickness(float(self.wallThicknessEntry.get()))
self.data_store.set_number_holes(int(self.numHolesEntry.get()))
self.data_store.set_bell_note_select(self.bellNoteCombo.current())
#self.data_store.set_embouchure_area(float(self.embouchureAreaEntry.get()))
self.data_store.set_bell_freq(
self.data_store.note_table[self.data_store.get_bell_note_select()]['frequency'])
@debugger
def set_state(self):
'''
Take the state from the data store and put in the GUI.
'''
self.titleEntry.delete(0, tkinter.END)
self.titleEntry.insert(0, self.data_store.get_title())
self.bellNoteCombo.current(self.data_store.get_bell_note_select())
self.measureUnitsOpt.current(int(self.data_store.get_units())) # it's a bool in the data_store
self.displayFormatOpt.current(int(self.data_store.get_disp_frac())) # it's a bool in the data_store
self.insideDiaEntry.delete(0, tkinter.END)
self.insideDiaEntry.insert(0, str(self.data_store.get_inside_dia()))
self.wallThicknessEntry.delete(0, tkinter.END)
self.wallThicknessEntry.insert(0, str(self.data_store.get_wall_thickness()))
self.numHolesEntry.delete(0, tkinter.END)
self.numHolesEntry.insert(0, str(self.data_store.get_number_holes()))
self.embouchureAreaEntry.config(state=tkinter.NORMAL)
self.embouchureAreaEntry.delete(0, tkinter.END)
self.embouchureAreaEntry.insert(0, "%0.4f"%(self.data_store.get_embouchure_area()))
self.embouchureAreaEntry.config(state="readonly")
self.lengthEntry.config(state=tkinter.NORMAL)
self.lengthEntry.delete(0, tkinter.END)
self.lengthEntry.insert(0, "%0.4f"%(self.data_store.get_length()))
self.lengthEntry.config(state="readonly")
@debugger
def insideDiaCommand(self, event=None):
try:
v = self.insideDiaEntry.get()
n = float(v)
if self.data_store.get_inside_dia() != n:
self.logger.debug("change wall from %f to %f"%(self.data_store.get_inside_dia(), n))
self.data_store.set_inside_dia(n)
self.insideDiaEntry.delete(0, tkinter.END)
self.insideDiaEntry.insert(0, str(n))
raise_event("CALCULATE_EVENT")
self.data_store.set_change_flag()
else:
self.logger.debug("ignore")
return True
except ValueError as e:
self.logger.error(str(e))
messagebox.showerror("Error", "Could not convert inside diameter to a floating point number.\nRead value was \"%s\"." % (v))
self.insideDiaEntry.delete(0, tkinter.END)
self.insideDiaEntry.insert(0, str(self.data_store.get_inside_dia()))
return False
except IndexError:
self.logger.error(str(e))
self.wallThicknessEntry.delete(0, tkinter.END)
self.wallThicknessEntry.insert(0, str(self.data_store.get_wall_thickness()))
except Exception as e:
self.logger.error(str(e))
messagebox.showerror("Unknown Error", "Unknown exception trying to convert inside diameter to a floating point number.\nRead value was \"%s\".\nException: %s" % (v, str(e)))
self.wallThicknessEntry.delete(0, tkinter.END)
self.wallThicknessEntry.insert(0, str(self.data_store.get_wall_thickness()))
@debugger
def wallThicknessCommand(self, event=None):
try:
v = self.wallThicknessEntry.get()
n = float(v)
if n != self.data_store.get_wall_thickness():
self.logger.debug("change wall from %f to %f"%(self.data_store.get_wall_thickness(), n))
self.data_store.set_wall_thickness(n)
self.wallThicknessEntry.delete(0, tkinter.END)
self.wallThicknessEntry.insert(0, str(n))
raise_event("CALCULATE_EVENT")
self.data_store.set_change_flag()
else:
self.logger.debug("ignore")
return True
except ValueError as e:
self.logger.error(str(e))
messagebox.showerror("Error", "Could not convert wall thickness to a floating point number.\nRead value was \"%s\"." % (v))
self.wallThicknessEntry.delete(0, tkinter.END)
self.wallThicknessEntry.insert(0, str(self.data_store.get_wall_thickness()))
return False
except IndexError:
self.logger.error(str(e))
self.wallThicknessEntry.delete(0, tkinter.END)
self.wallThicknessEntry.insert(0, str(self.data_store.get_wall_thickness()))
except Exception as e:
self.logger.error(str(e))
messagebox.showerror("Unknown Error", "Unknown exception trying convert wall thickness to a floating point number.\nRead value was \"%s\".\nException %s" % (v, str(e)))
self.wallThicknessEntry.delete(0, tkinter.END)
self.wallThicknessEntry.insert(0, str(self.data_store.get_wall_thickness()))
@debugger
def numHolesCommand(self, event=None):
n = 0
try:
v = self.numHolesEntry.get()
n = int(v)
if n >= 1 and n <= 12:
# only raise the event if the number of holes is different from
# what is in the data_store
if n != self.data_store.get_number_holes():
self.logger.debug("change number of holes from %d to %d"%(self.data_store.get_number_holes(), n))
self.data_store.set_number_holes(n)
raise_event('UPDATE_LOWER_FRAME_EVENT')
self.data_store.set_change_flag()
else:
self.logger.debug("ignore")
return True
else:
self.logger.error("range error on number of holes: %s"%(str(n)))
messagebox.showerror("Error", message="Number of holes must be an integer between 1 and 12.\nRead value was \"%s\"." % (v))
self.numHolesEntry.delete(0, tkinter.END)
self.numHolesEntry.insert(0, str(self.data_store.get_number_holes()))
return False
except ValueError as e:
self.logger.error(str(e))
messagebox.showerror("Error", message="Could not convert number of holes to an integer.\nRead value was \"%s\"." % (v))
self.numHolesEntry.delete(0, tkinter.END)
self.numHolesEntry.insert(0, str(self.data_store.get_number_holes()))
return False
except IndexError as e:
self.logger.error(str(e))
self.numHolesEntry.delete(0, tkinter.END)
self.numHolesEntry.insert(0, str(self.data_store.get_number_holes()))
except Exception as e:
self.logger.error(str(e))
messagebox.showerror("Unknown Error", message="Unknown exception trying to convert number of holes to an integer.\nRead value was \"%s\".\nException: %s" % (v, str(e)))
self.wallThicknessEntry.delete(0, tkinter.END)
self.wallThicknessEntry.insert(0, str(self.data_store.get_wall_thickness()))
@debugger
def displayFormatCallback(self, event):
if self.displayFormatOpt.current() == 0:
val = False
else:
val = True
if val != self.data_store.get_disp_frac():
self.data_store.set_disp_frac(val)
raise_event("UPDATE_HOLE_EVENT")
self.logger.debug("current format set to: %s"%(str(self.data_store.get_disp_frac())))
self.data_store.set_change_flag()
else:
self.logger.debug("ignore")
@debugger
def measureUnitsCallback(self, event):
if self.measureUnitsOpt.current() == 0:
val = False
else:
val = True
if self.data_store.get_units() != val:
if self.measureUnitsOpt.current() == 1:
self.displayFormatOpt.config(state=tkinter.DISABLED)
else:
self.displayFormatOpt.config(state="readonly")
self.data_store.set_units(val)
self.change_units()
self.logger.debug("current units set to: %s"%(str(self.data_store.get_units())))
self.data_store.set_change_flag()
else:
self.logger.debug("ignore")
@debugger
def bellSelectCallback(self, event):
'''
Change the data_store to match the new bell selection
'''
val = self.bellNoteCombo.current()
if val != self.data_store.get_bell_note_select():
self.data_store.set_bell_note_select(val)
self.data_store.set_bell_freq(self.data_store.note_table[val]['frequency'])
self.logger.debug("current bell selection set to: %d: %f"%(self.data_store.get_bell_note_select(), self.data_store.get_bell_freq()))
raise_event("UPDATE_NOTES_EVENT")
self.data_store.set_change_flag()
else:
self.logger.debug("ignore")
self.set_state()
@debugger
def refreshButtonCommand(self):
self.refreshButton.focus_set()
self.get_state()
#raise_event('UPDATE_LINES_EVENT')
raise_event('UPDATE_LOWER_FRAME_EVENT')
self.set_state()
@debugger
def change_units(self):
'''
When this is called, the assumption is that the GUI and the
data_store have the wrong units. This function takes what ever
is in the data_sore and converts if to the units that it finds
there. Then it updates the GUI.
'''
if self.data_store.get_units(): # true of units are mm
self.data_store.set_inside_dia(utility.in_to_mm(self.data_store.get_inside_dia()))
self.data_store.set_wall_thickness(utility.in_to_mm(self.data_store.get_wall_thickness()))
else:
self.data_store.set_inside_dia(utility.mm_to_in(self.data_store.get_inside_dia()))
self.data_store.set_wall_thickness(utility.mm_to_in(self.data_store.get_wall_thickness()))
self.set_state()
# Cause the other frames to update
raise_event("CHANGE_UNITS_EVENT")
raise_event('CALCULATE_EVENT')
self.data_store.set_change_flag()
@debugger
def setTitleCommand(self, event=None):
try:
title = self.titleEntry.get()
old_title = self.data_store.get_title()
if title != old_title:
self.data_store.set_title(title)
self.logger.debug("title set to: \"%s\""%(str(self.data_store.get_title())))
self.data_store.set_change_flag()
else:
self.logger.debug("ignore")
except IndexError:
pass # always ignore
class SetupFormBase(object):
'''
This class provides common services for forms in the setup notebook.
'''
def __init__(self, master, table, empty_ok=False):
self.logger = Logger(self, level=Logger.DEBUG)
self.logger.debug("Setup Dialog start constructor")
self.master = master
self.table = table
self.empty_ok = empty_ok
self.data = Database.get_instance()
self.events = EventHandler.get_instance()
self.id_list = self.get_id_list()
self.crnt_index = 0
@debugger
def get_id_list(self):
'''
This method exists so that a form can manage a smaller set of
records than every single one. To do that, override this default
method in the form class.
'''
return self.data.get_id_list(self.table)
@debugger
def get_id(self):
'''
Returns the id of the record in the current form.
'''
return self.id_list[self.crnt_index]
@debugger
def select_button_command(self):
self.id_list = self.get_id_list()
sel = SelectItem(self.master, self.table)
#if not hasattr(sel, 'item_id'):
if sel.item_id == -1:
self.logger.debug('Select dialog was canceled')
elif sel.item_id == 0:
self.logger.debug('Select dialog item was not found')
else:
try:
self.logger.debug('Select dialog item selected = %d' %
(sel.item_id))
self.crnt_index = self.id_list.index(sel.item_id)
self.set_form()
except TypeError:
mb.showerror(
'ERROR',
'No record was selected. (no records are available?)')
self.events.raise_event('select_button')
@debugger
def new_button_command(self):
''' Clear the form '''
self.clear_form()
self.events.raise_event('new_button')
@debugger
def save_button_command(self):
''' Save the form to the database '''
if not self.id_list is None:
self.get_form()
self.events.raise_event('save_button')
@debugger
def del_button_command(self):
''' Delete the item given in the form from the database '''
if not self.id_list is None:
val = mb.askokcancel(
"Sure?",
"Are you sure you want to delete item from %s?" % (self.table))
if val:
self.logger.info("Deleting item %d from %s" %
(self.id_list[self.crnt_index], self.table))
self.data.delete_row(self.table, self.id_list[self.crnt_index])
self.data.commit()
self.id_list = self.get_id_list()
if self.crnt_index >= len(self.id_list):
self.crnt_index -= 1
self.set_form()
self.events.raise_event('del_button')
@debugger
def next_btn_command(self):
''' Go to the next item in the form table '''
if not self.id_list is None:
self.crnt_index += 1
if self.crnt_index >= len(self.id_list):
self.crnt_index = len(self.id_list) - 1
self.set_form()
self.events.raise_event('next_button')
@debugger
def prev_btn_command(self):
''' Go to the previous item in the table '''
if not self.id_list is None:
self.crnt_index -= 1
if self.crnt_index < 0:
self.crnt_index = 0
self.set_form()
self.events.raise_event('prev_button')
@debugger
def clear_form(self):
'''
Clear the form.
'''
for item in self.form_contents:
print(item)
item['self'].clear()
self.id_list = None
self.events.raise_event('clear_form')
@debugger
def set_form(self): #, row_id):
'''
Read the database and place the data in the form.
'''
if self.id_list is None:
return
try:
self.id_list = self.get_id_list()
row_id = self.id_list[self.crnt_index]
except IndexError:
if not self.empty_ok:
self.logger.info('No records defined for table \'%s\'' %
(self.table))
mb.showinfo(
'Records',
'There are no records available for this form: \'%s\".' %
(self.table))
self.clear_form()
return
row = self.data.get_row_by_id(self.table, row_id)
if row is None:
if not self.empty_ok:
self.logger.info('No records defined for table \'%s\'' %
(self.table))
mb.showinfo(
'Records',
'There are no records available for this table: \'%s\'.' %
(self.table))
self.clear_form()
return
print(self.form_contents)
for item in self.form_contents:
if not item['hasid'] is None:
# swap in the value that the ID points to rather than the actual ID
item['hasid']['id'] = int(row[item['column']])
tmp_row = self.data.get_row_by_id(item['hasid']['table'],
item['hasid']['id'])
item['self'].write(tmp_row[item['hasid']['column']])
else:
item['self'].write(row[item['column']])
self.events.raise_event('set_form')
@debugger
def get_form(self):
'''
Read the form and place the data in the database.
'''
if self.id_list is None:
return
row = {}
for item in self.form_contents:
if not item['hasid'] is None:
# If in the future, forms that require a writable ID in the
# form is implemented, then this line will have to change.
row[item['column']] = item['hasid']['id']
else:
row[item['column']] = item['self'].read()
if self.id_list is None:
self.data.insert_row(self.table, row)
else:
row_id = self.id_list[self.crnt_index]
self.data.update_row_by_id(self.table, row, row_id)
self.id_list = self.get_id_list()
self.events.raise_event('get_form')
def move_to_point_and_extract(
coords_from_to: list, gps: adapters.GPSUbloxAdapter,
vesc_engine: adapters.VescAdapter, smoothie: adapters.SmoothieAdapter,
camera: adapters.CameraAdapterIMX219_170,
periphery_det: detection.YoloOpenCVDetection,
precise_det: detection.YoloOpenCVDetection, client,
logger_full: utility.Logger, logger_table: utility.Logger,
report_field_names, used_points_history: list, undistorted_zone_radius,
working_zone_polygon, working_zone_points_cv, view_zone_polygon,
view_zone_points_cv, img_output_dir, nav: navigation.GPSComputing):
"""
Moves to the given target point and extracts all weeds on the way.
:param coords_from_to:
:param gps:
:param vesc_engine:
:param smoothie:
:param camera:
:param periphery_det:
:param precise_det:
:param client:
:param logger_full:
:param logger_table:
:param report_field_names:
:param used_points_history:
:param nav:
:param working_zone_polygon:
:param undistorted_zone_radius:
:param working_zone_points_cv:
:param img_output_dir:
:return:
"""
vesc_engine.apply_rpm(config.VESC_RPM_SLOW)
slow_mode_time = -float("inf")
current_working_mode = working_mode_slow = 1
working_mode_switching = 2
working_mode_fast = 3
close_to_end = True # True if robot is close to one of current movement vector points, False otherwise
raw_angles_history = []
stop_helping_point = nav.get_coordinate(coords_from_to[1],
coords_from_to[0], 90, 1000)
prev_maneuver_time = time.time()
prev_pos = gps.get_last_position()
# set camera to the Y min
res = smoothie.custom_move_to(config.XY_F_MAX,
X=config.X_MAX / 2 /
config.XY_COEFFICIENT_TO_MM,
Y=config.Y_MIN)
if res != smoothie.RESPONSE_OK:
msg = "INIT: Failed to move camera to Y min, smoothie response:\n" + res
logger_full.write(msg + "\n")
smoothie.wait_for_all_actions_done()
# main navigation control loop
while True:
# EXTRACTION CONTROL
start_t = time.time()
frame = camera.get_image()
frame_t = time.time()
plants_boxes = periphery_det.detect(frame)
per_det_t = time.time()
debug_save_image(
img_output_dir,
"(periphery view scan M=" + str(current_working_mode) + ")", frame,
plants_boxes, undistorted_zone_radius, working_zone_points_cv
if current_working_mode == 1 else view_zone_points_cv)
msg = "View frame time: " + str(
frame_t - start_t) + "\t\tPer. det. time: " + str(per_det_t -
frame_t)
logger_full.write(msg + "\n")
# slow mode
if current_working_mode == working_mode_slow:
if any_plant_in_zone(plants_boxes, working_zone_polygon):
vesc_engine.stop_moving()
time.sleep(0.2)
start_work_t = time.time()
frame = camera.get_image()
frame_t = time.time()
plants_boxes = precise_det.detect(frame)
pre_det_t = time.time()
debug_save_image(img_output_dir, "(precise view scan M=1)",
frame, plants_boxes, undistorted_zone_radius,
working_zone_points_cv)
msg = "Work frame time: " + str(
frame_t -
start_work_t) + "\t\tPrec. det. time: " + str(pre_det_t -
frame_t)
logger_full.write(msg + "\n")
if any_plant_in_zone(plants_boxes, working_zone_polygon):
extract_all_plants(smoothie, camera, precise_det,
working_zone_polygon, frame,
plants_boxes, undistorted_zone_radius,
working_zone_points_cv, img_output_dir)
elif not any_plant_in_zone(plants_boxes, view_zone_polygon) and \
time.time() - slow_mode_time > config.SLOW_MODE_MIN_TIME:
# set camera to the Y max
res = smoothie.custom_move_to(
config.XY_F_MAX,
X=config.X_MAX / 2 / config.XY_COEFFICIENT_TO_MM,
Y=config.Y_MAX / config.XY_COEFFICIENT_TO_MM)
if res != smoothie.RESPONSE_OK:
msg = "M=" + str(
current_working_mode
) + ": " + "Failed to move to Y max, smoothie response:\n" + res
logger_full.write(msg + "\n")
smoothie.wait_for_all_actions_done()
current_working_mode = working_mode_switching
vesc_engine.start_moving()
# switching to fast mode
elif current_working_mode == working_mode_switching:
if any_plant_in_zone(plants_boxes, view_zone_polygon):
vesc_engine.stop_moving()
# set camera to the Y min
res = smoothie.custom_move_to(config.XY_F_MAX,
X=config.X_MAX / 2 /
config.XY_COEFFICIENT_TO_MM,
Y=config.Y_MIN)
if res != smoothie.RESPONSE_OK:
msg = "M=" + str(
current_working_mode
) + ": " + "Failed to move to Y min, smoothie response:\n" + res
logger_full.write(msg + "\n")
smoothie.wait_for_all_actions_done()
current_working_mode = working_mode_slow
slow_mode_time = time.time()
elif smoothie.get_smoothie_current_coordinates(False)[
"Y"] + config.XY_COEFFICIENT_TO_MM * 20 > config.Y_MAX:
current_working_mode = working_mode_fast
if not close_to_end:
vesc_engine.apply_rpm(config.VESC_RPM_FAST)
# fast mode
else:
if any_plant_in_zone(plants_boxes, view_zone_polygon):
vesc_engine.stop_moving()
# set camera to the Y min
res = smoothie.custom_move_to(config.XY_F_MAX,
X=config.X_MAX / 2 /
config.XY_COEFFICIENT_TO_MM,
Y=config.Y_MIN)
if res != smoothie.RESPONSE_OK:
msg = "M=" + str(
current_working_mode
) + ": " + "Failed to move to Y min, smoothie response:\n" + res
logger_full.write(msg + "\n")
smoothie.wait_for_all_actions_done()
current_working_mode = working_mode_slow
slow_mode_time = time.time()
vesc_engine.apply_rpm(config.VESC_RPM_SLOW)
elif close_to_end:
vesc_engine.apply_rpm(config.VESC_RPM_SLOW)
else:
vesc_engine.apply_rpm(config.VESC_RPM_FAST)
# NAVIGATION CONTROL
nav_start_t = time.time()
cur_pos = gps.get_last_position()
if str(cur_pos) == str(prev_pos):
# msg = "Got the same position, added to history, calculations skipped. Am I stuck?"
# print(msg)
# logger_full.write(msg + "\n")
continue
used_points_history.append(cur_pos.copy())
if not client.sendData("{};{}".format(cur_pos[0], cur_pos[1])):
msg = "[Client] Connection closed !"
print(msg)
logger_full.write(msg + "\n")
distance = nav.get_distance(cur_pos, coords_from_to[1])
msg = "Distance to B: " + str(distance)
# print(msg)
logger_full.write(msg + "\n")
# check if arrived
_, side = nav.get_deviation(coords_from_to[1], stop_helping_point,
cur_pos)
# if distance <= config.COURSE_DESTINATION_DIFF: # old way
if side != 1: # TODO: maybe should use both side and distance checking methods at once
vesc_engine.stop_moving()
# msg = "Arrived (allowed destination distance difference " + str(config.COURSE_DESTINATION_DIFF) + " mm)"
msg = "Arrived to " + str(coords_from_to[1])
# print(msg)
logger_full.write(msg + "\n")
break
# pass by cur points which are very close to prev point to prevent angle errors when robot is staying
# (too close points in the same position can produce false huge angles)
if nav.get_distance(prev_pos,
cur_pos) < config.PREV_CUR_POINT_MIN_DIST:
continue
# reduce speed if near the target point
if config.USE_SPEED_LIMIT:
distance_from_start = nav.get_distance(coords_from_to[0], cur_pos)
close_to_end = distance < config.DECREASE_SPEED_TRESHOLD or distance_from_start < config.DECREASE_SPEED_TRESHOLD
# do maneuvers not more often than specified value
cur_time = time.time()
if cur_time - prev_maneuver_time < config.MANEUVERS_FREQUENCY:
continue
prev_maneuver_time = cur_time
msg = "Prev: " + str(prev_pos) + " Cur: " + str(cur_pos) + " A: " + str(coords_from_to[0]) \
+ " B: " + str(coords_from_to[1])
# print(msg)
logger_full.write(msg + "\n")
raw_angle = nav.get_angle(prev_pos, cur_pos, cur_pos,
coords_from_to[1])
# sum(e)
if len(raw_angles_history) >= config.WINDOW:
raw_angles_history.pop(0)
raw_angles_history.append(raw_angle)
sum_angles = sum(raw_angles_history)
if sum_angles > config.SUM_ANGLES_HISTORY_MAX:
msg = "Sum angles " + str(sum_angles) + " is bigger than max allowed value " + \
str(config.SUM_ANGLES_HISTORY_MAX) + ", setting to " + str(config.SUM_ANGLES_HISTORY_MAX)
# print(msg)
logger_full.write(msg + "\n")
sum_angles = config.SUM_ANGLES_HISTORY_MAX
elif sum_angles < -config.SUM_ANGLES_HISTORY_MAX:
msg = "Sum angles " + str(sum_angles) + " is less than min allowed value " + \
str(-config.SUM_ANGLES_HISTORY_MAX) + ", setting to " + str(-config.SUM_ANGLES_HISTORY_MAX)
# print(msg)
logger_full.write(msg + "\n")
sum_angles = -config.SUM_ANGLES_HISTORY_MAX
angle_kp_ki = raw_angle * config.KP + sum_angles * config.KI
target_angle_sm = angle_kp_ki * -config.A_ONE_DEGREE_IN_SMOOTHIE # smoothie -Value == left, Value == right
ad_wheels_pos = smoothie.get_adapter_current_coordinates()["A"]
sm_wheels_pos = smoothie.get_smoothie_current_coordinates()["A"]
# compute order angle (smoothie can't turn for huge values immediately also as cancel movement,
# so we need to do nav. actions in steps)
order_angle_sm = target_angle_sm - ad_wheels_pos
# check for out of update frequency and smoothie execution speed range (for nav wheels)
if order_angle_sm > config.MANEUVERS_FREQUENCY * config.A_DEGREES_PER_SECOND * \
config.A_ONE_DEGREE_IN_SMOOTHIE:
msg = "Order angle changed from " + str(
order_angle_sm) + " to " + str(
config.MANEUVERS_FREQUENCY * config.A_DEGREES_PER_SECOND +
config.A_ONE_DEGREE_IN_SMOOTHIE
) + " due to exceeding degrees per tick allowed range."
# print(msg)
logger_full.write(msg + "\n")
order_angle_sm = config.MANEUVERS_FREQUENCY * config.A_DEGREES_PER_SECOND * \
config.A_ONE_DEGREE_IN_SMOOTHIE
elif order_angle_sm < -(config.MANEUVERS_FREQUENCY *
config.A_DEGREES_PER_SECOND *
config.A_ONE_DEGREE_IN_SMOOTHIE):
msg = "Order angle changed from " + str(
order_angle_sm) + " to " + str(-(
config.MANEUVERS_FREQUENCY * config.A_DEGREES_PER_SECOND *
config.A_ONE_DEGREE_IN_SMOOTHIE
)) + " due to exceeding degrees per tick allowed range."
# print(msg)
logger_full.write(msg + "\n")
order_angle_sm = -(config.MANEUVERS_FREQUENCY *
config.A_DEGREES_PER_SECOND *
config.A_ONE_DEGREE_IN_SMOOTHIE)
# convert to global smoothie coordinates
order_angle_sm += ad_wheels_pos
# checking for out of smoothie supported range
if order_angle_sm > config.A_MAX:
msg = "Global order angle changed from " + str(order_angle_sm) + " to config.A_MAX = " + \
str(config.A_MAX) + " due to exceeding smoothie allowed values range."
# print(msg)
logger_full.write(msg + "\n")
order_angle_sm = config.A_MAX
elif order_angle_sm < config.A_MIN:
msg = "Global order angle changed from " + str(order_angle_sm) + " to config.A_MIN = " + \
str(config.A_MIN) + " due to exceeding smoothie allowed values range."
# print(msg)
logger_full.write(msg + "\n")
order_angle_sm = config.A_MIN
raw_angle = round(raw_angle, 2)
angle_kp_ki = round(angle_kp_ki, 2)
order_angle_sm = round(order_angle_sm, 2)
sum_angles = round(sum_angles, 2)
distance = round(distance, 2)
ad_wheels_pos = round(ad_wheels_pos, 2)
sm_wheels_pos = round(sm_wheels_pos, 2)
gps_quality = cur_pos[2]
msg = str(gps_quality).ljust(5) + str(raw_angle).ljust(8) + str(
angle_kp_ki).ljust(8) + str(order_angle_sm).ljust(8) + str(
sum_angles).ljust(8) + str(distance).ljust(13) + str(
ad_wheels_pos).ljust(8) + str(sm_wheels_pos).ljust(9)
print(msg)
logger_full.write(msg + "\n")
# load sensors data to csv
s = ","
msg = str(gps_quality) + s + str(raw_angle) + s + str(angle_kp_ki) + s + str(order_angle_sm) + s + \
str(sum_angles) + s + str(distance) + s + str(ad_wheels_pos) + s + str(sm_wheels_pos)
vesc_data = vesc_engine.get_sensors_data(report_field_names)
if vesc_data is not None:
msg += s
for key in vesc_data:
msg += str(vesc_data[key]) + s
msg = msg[:-1]
logger_table.write(msg + "\n")
prev_pos = cur_pos
response = smoothie.nav_turn_wheels_to(order_angle_sm, config.A_F_MAX)
if response != smoothie.RESPONSE_OK: # TODO: what if response is not ok?
msg = "Smoothie response is not ok: " + response
print(msg)
logger_full.write(msg + "\n")
msg = "Nav calc time: " + str(time.time() - nav_start_t)
logger_full.write(msg + "\n\n")
class NotesBox(tk.Frame):
'''
Implement a notes widget and provide a regular interface to it, same as the
other form widgets.
'''
def __init__(self,
master,
table,
column,
height=20,
width=60,
*args,
**kargs):
'''
master = the frame to bind this frame to
name = the text of the label
table = the name of the database table that is associated with this widget
column = the name of the column this widget associates with
lw = label width
cw = control width
ch = control height
'''
self.logger = Logger(self, level=Logger.INFO)
self.logger.debug("NotesBox enter constructor")
super().__init__(master, *args, **kargs)
self.column = column
self.table = table
frame2 = tk.Frame(self, bd=1, relief=tk.RIDGE)
frame2.grid(row=0, column=1)
self.content = tk.Text(frame2,
height=height,
width=width,
wrap=tk.NONE)
self.content.insert(tk.END, '')
# see https://www.homeandlearn.uk/tkinter-scrollbars.html
self.vsb = tk.Scrollbar(frame2, orient=tk.VERTICAL)
self.vsb.config(command=self.content.yview)
self.content.config(yscrollcommand=self.vsb.set)
self.vsb.pack(side=tk.RIGHT, fill=tk.Y)
self.hsb = tk.Scrollbar(frame2, orient=tk.HORIZONTAL)
self.hsb.config(command=self.content.xview)
self.content.config(xscrollcommand=self.hsb.set)
self.hsb.pack(side=tk.BOTTOM, fill=tk.X)
self.content.pack(side=tk.LEFT)
self.row = {
'table': self.table,
'column': self.column,
'self': self,
'hasid': None
}
self.logger.debug("NotesBox leave constructor")
@debugger
def read(self):
return self.content.get(1.0, tk.END)
@debugger
def write(self, val):
self.content.delete('1.0', tk.END)
if not val is None:
self.content.insert(tk.END, val)
@debugger
def clear(self):
self.content.delete('1.0', tk.END)
@debugger
def get_line(self):
'''
Return the form entry to update the form.
'''
return self.row
def move_to_point(coords_from_to: list, used_points_history: list,
gps: adapters.GPSUbloxAdapter,
vesc_engine: adapters.VescAdapter,
smoothie: adapters.SmoothieAdapter, logger: utility.Logger,
client, nav: navigation.GPSComputing,
raw_angles_history: list):
"""
Moves to given point.
:param coords_from_to:
:param used_points_history:
:param gps:
:param vesc_engine:
:param smoothie:
:param logger:
:param client:
:param nav:
:return:
:param raw_angles_history:
"""
raw_angles_history = []
stop_helping_point = nav.get_coordinate(coords_from_to[1],
coords_from_to[0], 90, 1000)
prev_maneuver_time = time.time()
prev_point = gps.get_fresh_position(
) # TODO: maybe it's ok to get last position instead of waiting for fresh
vesc_engine.set_rpm(int(config.VESC_RPM / 2))
vesc_engine.start_moving()
# main navigation control loop
while True:
cur_pos = gps.get_fresh_position()
used_points_history.append(cur_pos.copy())
if not client.sendData("{};{}".format(cur_pos[0], cur_pos[1])):
msg = "[Client] Connection closed !"
print(msg)
logger.write(msg + "\n")
if str(cur_pos) == str(prev_point):
msg = "Got the same position, added to history, calculations skipped"
print(msg)
logger.write(msg + "\n")
continue
distance = nav.get_distance(cur_pos, coords_from_to[1])
msg = "Distance to B: " + str(distance)
print(msg)
logger.write(msg + "\n")
# check if arrived
_, side = nav.get_deviation(coords_from_to[1], stop_helping_point,
cur_pos)
# if distance <= config.COURSE_DESTINATION_DIFF: # old way
if side != 1: # TODO: maybe should use both side and distance checking methods at once
vesc_engine.stop_moving()
# msg = "Arrived (allowed destination distance difference " + str(config.COURSE_DESTINATION_DIFF) + " mm)"
msg = "Arrived."
print(msg)
logger.write(msg + "\n")
break
# reduce speed if near the target point
if config.USE_SPEED_LIMIT:
distance_from_start = nav.get_distance(coords_from_to[0], cur_pos)
if distance < config.DECREASE_SPEED_TRESHOLD or distance_from_start < config.DECREASE_SPEED_TRESHOLD:
vesc_engine.apply_rpm(int(config.VESC_RPM / 2))
else:
vesc_engine.apply_rpm(config.VESC_RPM)
# do maneuvers not more often than specified value
cur_time = time.time()
if cur_time - prev_maneuver_time < config.MANEUVERS_FREQUENCY:
continue
prev_maneuver_time = cur_time
msg = "Timestamp: " + str(cur_time)
print(msg)
logger.write(msg + "\n")
msg = "Prev: " + str(prev_point) + " Cur: " + str(cur_pos) + " A: " + str(coords_from_to[0]) \
+ " B: " + str(coords_from_to[1])
print(msg)
logger.write(msg + "\n")
raw_angle = nav.get_angle(prev_point, cur_pos, cur_pos,
coords_from_to[1])
# sum(e)
if len(raw_angles_history) >= config.WINDOW:
raw_angles_history.pop(0)
raw_angles_history.append(raw_angle)
sum_angles = sum(raw_angles_history)
if sum_angles > config.SUM_ANGLES_HISTORY_MAX:
msg = "Sum angles " + str(sum_angles) + " is bigger than max allowed value " + \
str(config.SUM_ANGLES_HISTORY_MAX) + ", setting to " + str(config.SUM_ANGLES_HISTORY_MAX)
print(msg)
logger.write(msg)
sum_angles = config.SUM_ANGLES_HISTORY_MAX
elif sum_angles < -config.SUM_ANGLES_HISTORY_MAX:
msg = "Sum angles " + str(sum_angles) + " is less than min allowed value " + \
str(-config.SUM_ANGLES_HISTORY_MAX) + ", setting to " + str(-config.SUM_ANGLES_HISTORY_MAX)
print(msg)
logger.write(msg)
sum_angles = -config.SUM_ANGLES_HISTORY_MAX
angle_kp_ki = raw_angle * config.KP + sum_angles * config.KI
target_angle_sm = angle_kp_ki * -config.A_ONE_DEGREE_IN_SMOOTHIE # smoothie -Value == left, Value == right
ad_wheels_pos = smoothie.get_adapter_current_coordinates()["A"]
sm_wheels_pos = smoothie.get_smoothie_current_coordinates()["A"]
# compute order angle (smoothie can't turn for huge values immediately also as cancel movement,
# so we need to do nav. actions in steps)
order_angle_sm = target_angle_sm - ad_wheels_pos
# check for out of update frequency and smoothie execution speed (for nav wheels)
if order_angle_sm > config.MANEUVERS_FREQUENCY * config.A_DEGREES_PER_SECOND * \
config.A_ONE_DEGREE_IN_SMOOTHIE:
msg = "Order angle changed from " + str(
order_angle_sm) + " to " + str(
config.MANEUVERS_FREQUENCY * config.A_DEGREES_PER_SECOND +
config.A_ONE_DEGREE_IN_SMOOTHIE
) + " due to exceeding degrees per tick allowed range."
print(msg)
logger.write(msg + "\n")
order_angle_sm = config.MANEUVERS_FREQUENCY * config.A_DEGREES_PER_SECOND * \
config.A_ONE_DEGREE_IN_SMOOTHIE
elif order_angle_sm < -(config.MANEUVERS_FREQUENCY *
config.A_DEGREES_PER_SECOND *
config.A_ONE_DEGREE_IN_SMOOTHIE):
msg = "Order angle changed from " + str(
order_angle_sm) + " to " + str(-(
config.MANEUVERS_FREQUENCY * config.A_DEGREES_PER_SECOND *
config.A_ONE_DEGREE_IN_SMOOTHIE
)) + " due to exceeding degrees per tick allowed range."
print(msg)
logger.write(msg + "\n")
order_angle_sm = -(config.MANEUVERS_FREQUENCY *
config.A_DEGREES_PER_SECOND *
config.A_ONE_DEGREE_IN_SMOOTHIE)
# convert to global coordinates
order_angle_sm += ad_wheels_pos
# checking for out of smoothie supported range
if order_angle_sm > config.A_MAX:
msg = "Global order angle changed from " + str(order_angle_sm) + " to config.A_MAX = " + \
str(config.A_MAX) + " due to exceeding smoothie allowed values range."
print(msg)
logger.write(msg + "\n")
order_angle_sm = config.A_MAX
elif order_angle_sm < config.A_MIN:
msg = "Global order angle changed from " + str(order_angle_sm) + " to config.A_MIN = " + \
str(config.A_MIN) + " due to exceeding smoothie allowed values range."
print(msg)
logger.write(msg + "\n")
order_angle_sm = config.A_MIN
msg = "Adapter wheels pos (target): " + str(ad_wheels_pos) + " Smoothie wheels pos (current): " \
+ str(sm_wheels_pos)
print(msg)
logger.write(msg + "\n")
msg = "KI: " + str(config.KI) + " Sum angles: " + str(sum_angles) + " Sum angles history: " + \
str(raw_angles_history)
print(msg)
logger.write(msg + "\n")
msg = "KP: " + str(config.KP) + " Raw angle: " + str(raw_angle) + " Angle * KP + sum(angles) * KI: " + \
str(angle_kp_ki) + " Smoothie target angle: " + str(target_angle_sm) + \
" Smoothie absolute order angle: " + str(order_angle_sm)
print(msg)
logger.write(msg + "\n")
prev_point = cur_pos
response = smoothie.nav_turn_wheels_to(order_angle_sm, config.A_F_MAX)
msg = "Smoothie response: " + response
print(msg)
logger.write(msg)
# next tick indent
print()
logger.write("\n")
class LineBox(tk.Frame):
'''
This is a container for the line widgets. It holds a variable number of
line boxes and has the ability to write them all to the database when the
write method is called.
'''
def __init__(self, master, form=None, *args, **kargs):
'''
master = The frame to bind the widgets to.
name_id = The id of the line containing the customer to associate
form = Name of the form to bind the events to.
'''
self.logger = Logger(self, level=Logger.DEBUG)
self.logger.debug("Line Widget enter constructor")
super().__init__(master, bd=1, relief=tk.RIDGE, *args, **kargs)
#self.name_id = int(name_id)
self.form = form
self.events = EventHandler.get_instance()
self.data = Database.get_instance()
self.line_list = []
self.crnt_index = 0
self.events.register_event('next_button', self.clear)
self.events.register_event('prev_button', self.clear)
# add button
tk.Button(self, text="Add", command=self.add).grid(row=0, column=0)
# reset button
tk.Button(self, text="Reset", command=self.clear).grid(row=0, column=1)
# add one line widget
self.add()
# self.row = {'table': None, 'column':None, 'self':self, 'hasid':None}
self.logger.debug("Line Widget leave constructor")
@debugger
def add(self):
'''
Method that actually adds the line to the widget.
'''
line = LineWidget(self, None, self.crnt_index + 1)
line.grid(row=self.crnt_index + 1, column=0, columnspan=2, padx=5)
self.line_list.append(line)
self.crnt_index += 1
# @debugger
# def get_line(self):
# return self.row
@debugger
def clear(self):
'''
Reset the widget to having one blank line.
'''
for item in self.line_list:
item.grid_forget()
self.line_list = []
self.crnt_index = 0
self.add()
@debugger
def read(self, sale_id):
'''
Read method saves the contects to the database
'''
for item in self.line_list:
(quan, name) = item.read()
if name != '':
row = {
'sale_record_ID': sale_id,
'quantity': quan,
'inventory_ID':
self.data.get_id_by_name('InventoryItem', name)
}
self.data.insert_row('ProductList', row)
self.data.commit()
@debugger
def write(self):
'''
This function does nothing for this widget
'''
self.clear()
def configure(filename="tuf.interposition.json",
parent_repository_directory=None,
parent_ssl_certificates_directory=None):
"""
The optional parent_repository_directory parameter is used to specify the
containing parent directory of the "repository_directory" specified in a
configuration for *all* network locations, because sometimes the absolute
location of the "repository_directory" is only known at runtime. If you
need to specify a different parent_repository_directory for other
network locations, simply call this method again with different parameters.
Ditto for the optional parent_ssl_certificates_directory parameter.
Example of a TUF interposition configuration JSON object:
{
"configurations": {
"seattle.cs.washington.edu": {
"repository_directory": "client/",
"repository_mirrors" : {
"mirror1": {
"url_prefix": "http://seattle-tuf.cs.washington.edu",
"metadata_path": "metadata",
"targets_path": "targets",
"confined_target_dirs": [ "" ]
}
},
("target_paths": [
{ ".*/(simple/\\w+)/$": "{0}/index.html" },
{ ".*/(packages/.+)$": "{0}" }
],
"ssl_certificates": "cacert.pem")
}
}
}
"target_paths" is optional: If you do not tell TUF to selectively match
paths with regular expressions, TUF will work over any path under the given
network location. However, if you do specify it, you are then telling TUF
how to transform a specified path into another one, and TUF will *not*
recognize any unspecified path for the given network location.
Unless any "url_prefix" begins with "https://", "ssl_certificates" is
optional; it must specify certificates bundled as PEM (RFC 1422).
"""
INVALID_TUF_CONFIGURATION = "Invalid configuration for {network_location}!"
INVALID_TUF_INTERPOSITION_JSON = "Invalid configuration in {filename}!"
NO_CONFIGURATIONS = "No configurations found in configuration in {filename}!"
try:
with open(filename) as tuf_interposition_json:
tuf_interpositions = json.load(tuf_interposition_json)
configurations = tuf_interpositions.get("configurations", {})
if len(configurations) == 0:
raise InvalidConfiguration(NO_CONFIGURATIONS.format(filename=filename))
else:
for network_location, configuration in configurations.iteritems():
try:
configuration_parser = ConfigurationParser(network_location,
configuration, parent_repository_directory=parent_repository_directory,
parent_ssl_certificates_directory=parent_ssl_certificates_directory)
configuration = configuration_parser.parse()
__updater_controller.add(configuration)
except:
Logger.exception(INVALID_TUF_CONFIGURATION.format(network_location=network_location))
raise
except:
Logger.exception(INVALID_TUF_INTERPOSITION_JSON.format(filename=filename))
raise
'sgd_huber_loss_combine_sampled',
'sgd_huber_loss_over_sampled',
'sgd_huber_loss_under_sampled',
'sgd_log_loss_basic',
'sgd_log_loss_combine_sampled',
'sgd_log_loss_over_sampled',
'sgd_log_loss_under_sampled',
'xgboost_basic',
'xgboost_combine_sampled',
'xgboost_over_sampled',
'xgboost_under_sampled',
]
if __name__ == '__main__':
use_project_path()
logger = Logger('model/experiment/output/model_scoring_search.txt')
scores = []
for predictions in input_predictions:
with open('model/experiment/output/%s_predict_proba.p' % predictions,
'rb') as file:
frame = pickle.load(file)
optimal_threshold, optimal_f1_score = find_optimal_f1_threshold(frame)
y_actual = frame.y_actual
y_predict_proba = frame.y_predict
y_predict = (y_predict_proba[:, 1] >= optimal_threshold).astype(bool)
score = [
predictions, optimal_threshold, optimal_f1_score,
accuracy_score(y_actual, y_predict),
class BaseDialog(tk.Toplevel):
'''
This class provides common services to simple data dialogs.
'''
def __init__(self, parent):# , title = None):
#init the logger
self.logger = Logger(self, level=Logger.DEBUG)
self.logger.debug("Base Dialog start constructor")
tk.Toplevel.__init__(self, parent)
self.transient(parent)
self.parent = parent
self.result = None
# get a copy of the data_store for the children
body = tk.Frame(self)
self.initial_focus = self.body(body)
body.grid(padx=5, pady=5)
self.buttonbox()
self.grab_set()
if not self.initial_focus:
self.initial_focus = self
self.protocol("WM_DELETE_WINDOW", self.cancel)
self.initial_focus.focus_set()
#self.wait_window(self)
self.logger.debug("Base Dialog leave constructor")
#
# construction hooks
def body(self, master):
# create dialog body. return widget that should have
# initial focus. this method should be overridden
return self
def buttonbox(self):
# add standard button box. override if you don't want the
# standard buttons
box = tk.Frame(self)
w = tk.Button(box, text="OK", width=10, command=self.ok, default=tk.ACTIVE)
w.pack(side=tk.LEFT, padx=5, pady=5)
w = tk.Button(box, text="Cancel", width=10, command=self.cancel)
w.pack(side=tk.LEFT, padx=5, pady=5)
box.grid()
#
# standard button semantics
@debugger
def ok(self, event=None):
if not self.validate():
self.initial_focus.focus_set() # put focus back
return
self.withdraw()
self.update_idletasks()
self.apply()
self.cancel()
@debugger
def cancel(self, event=None):
# put focus back to the parent window
self.parent.focus_set()
self.destroy()
#
# command hooks
def validate(self):
return True # override
def apply(self):
pass # override
class HoleSizeWidgit(tkinter.Frame):
def __init__(self, parent, line):
'''
This is a specialized widget to track the hole diameter. It displays
the value according to the state. It has up and down buttons used to
increment or decrement the value.
'''
self.logger = Logger(self, Logger.INFO)
self.logger.debug("constructor")
self.index = line
tkinter.Frame.__init__(self, parent)
self.data_store = DataStore.get_instance()
BITMAP_UP = """
#define up_width 9
#define up_height 5
static unsigned char up_bits[] = {
0x10, 0x00, 0x38, 0x00, 0x7c, 0x00, 0xfe, 0x00, 0xff, 0x01 };
"""
BITMAP_DOWN = """
#define down_width 9
#define down_height 5
static unsigned char down_bits[] = {
0xff, 0x01, 0xfe, 0x00, 0x7c, 0x00, 0x38, 0x00, 0x10, 0x00 };
"""
self.entry = tkinter.Entry(self, width=25-18)
self.entry.grid(row=0, column=1, rowspan=2)
self.bitmap_up = tkinter.BitmapImage(data=BITMAP_UP)
self.b1 = tkinter.Button(self, text="up", image=self.bitmap_up, command=self.incr_command)
self.b1.grid(row=0, column=2)
self.b1.config(width=9, height=5)
self.bitmap_down = tkinter.BitmapImage(data=BITMAP_DOWN)
self.b2 = tkinter.Button(self, image=self.bitmap_down, command=self.decr_command)
self.b2.grid(row=1, column=2)
self.b2.config(width=9, height=5)
register_event("UPDATE_HOLE_EVENT", self.update_val)
self.set_state()
self.logger.debug("constructor return")
@debugger
def get_state(self):
'''
Put the current value into the data_store.
'''
try:
if self.data_store.get_disp_frac():
val = utility.fractof(self.entry.get())
else:
val = float(self.entry.get())
except ValueError as e:
mbox.showerror('ERROR', 'Cannot convert hole to float: \"%s\": %s'%(self.entry.get(), e))
self.data_store.set_hole_size(self.index, val)
@debugger
def set_state(self):
'''
Get the current state from the data_store and place it in the GUI
'''
self.update_val() # display the new state
@debugger
def update_val(self):
siz = self.data_store.get_hole_size(self.index)
self.entry.delete(0, tkinter.END)
if not self.data_store.get_units():
self.logger.debug("updating inch value to %s"%(str(siz)))
if self.data_store.get_disp_frac():
self.entry.insert(0, "%s"%(utility.reduce(siz)))
else:
self.entry.insert(0, "%0.4f"%(siz))
else:
self.logger.debug("updating mm value to %s"%(str(siz)))
self.entry.insert(0, "%0.4f"%(siz))
# event handlers
@debugger
def incr_command(self):
siz = self.data_store.get_hole_size(self.index)
siz = siz + self.data_store.get_hole_inc()
if siz > self.data_store.get_hole_max():
siz = self.data_store.get_hole_max()
elif siz < self.data_store.get_hole_min():
siz = self.data_store.get_hole_min()
self.data_store.set_hole_size(self.index, siz)
self.update_val() # update the GUI
raise_event("CALCULATE_EVENT")
self.data_store.set_change_flag()
@debugger
def decr_command(self):
siz = self.data_store.get_hole_size(self.index)
siz = siz - self.data_store.get_hole_inc()
if siz < self.data_store.get_hole_min():
siz = self.data_store.get_hole_min()
elif siz > self.data_store.get_hole_max():
siz = self.data_store.get_hole_max()
self.data_store.set_hole_size(self.index, siz)
self.update_val() # update the GUI
raise_event("CALCULATE_EVENT")
self.data_store.set_change_flag()
def print_state(self):
self.logger.msg(str(self.get_state()))
def __init__(self, notebook):
self.logger = Logger(self, Logger.DEBUG)
self.logger.debug(sys._getframe().f_code.co_name)
super().__init__(notebook, notebook.PURCHASE_FRAME, 'Business')
self.add_title('Purchases Form')
def __init__(self, notebook):
self.logger = Logger(self, Logger.DEBUG)
self.logger.debug(sys._getframe().f_code.co_name)
super().__init__(notebook, notebook.REPORTS_FRAME, 'Business')
self.add_title('Reports Form')
def __init__(self, master):
self.logger = Logger(self, Logger.INFO)
self.logger.debug("constructor")
self.master = master
self.data_store = DataStore.get_instance()
register_event("UPDATE_UPPER_EVENT", self.set_state)
'0110', '0130', '0150', '0141', '0261', '0262', '0271', '0272', '031A',
'031B', '0326', '033A', '033B', '041A', '041B', '0450', '0451', '0480',
'0481', '051A', '051B', '0550', '0551', '141A', '141B'
]
public_violence_excluded_locations = [
'APARTMENT', 'CHA APARTMENT', 'BUSINESS OFFICE', 'COIN OPERATED MACHINE',
'COLLEGE/UNIVERSITY RESIDENCE HALL', 'FACTORY/MANUFACTURING BUILDING',
'RESIDENCE-GARAGE', 'NURSING HOME/RETIREMENT HOME', 'RESIDENCE',
'WAREHOUSE', 'OTHER'
]
if __name__ == '__main__':
use_project_path()
logger = Logger('Data_Scratch/preprocess.txt')
iucr_codes = load_codes('data_scratch/iucr_codes.json')
type_codes = load_codes('data_scratch/type_codes.json')
description_codes = load_codes('data_scratch/description_codes.json')
location_codes = load_codes('data_scratch/location_codes.json')
fbi_codes = load_codes('data_scratch/fbi_codes.json')
missing_columns = {}
for column in column_names:
missing_columns[column] = 0
raw_record_count = -1
processed_records = 0
logger.time_log('Starting Pre-Processing.')
with open('data_scratch/Crimes_-_2001_to_present.csv') as input_file:
def cleanup(self):
"""Clean up after certain side effects, such as temporary directories."""
DELETED_TEMPDIR_MESSAGE = "Deleted temporary directory at {tempdir}"
shutil.rmtree(self.tempdir)
Logger.debug(DELETED_TEMPDIR_MESSAGE.format(tempdir=self.tempdir))
