def get_target_paths(self): """ Within a network_location, we match URLs with this list of regular expressions, which tell us to map from a source URL to a target URL. If there are multiple regular expressions which match a source URL, the order of appearance will be used to resolve ambiguity. """ INVALID_TARGET_PATH = "Invalid target path in {network_location}!" # An "identity" capture from source URL to target URL. WILD_TARGET_PATH = { "(.*)": "{0}" } target_paths = self.configuration.get("target_paths", [WILD_TARGET_PATH]) # target_paths: [ target_path, ... ] assert isinstance(target_paths, types.ListType) for target_path in target_paths: try: # target_path: { "regex_with_groups", "target_with_group_captures" } # e.g. { ".*(/some/directory)/$", "{0}/index.html" } assert isinstance(target_path, types.DictType) assert len(target_path) == 1 except: error_message = \ INVALID_TARGET_PATH.format(network_location=self.network_location) Logger.exception(error_message) raise InvalidConfiguration(error_message) return target_paths
def retrieve(self, url, filename=None, reporthook=None, data=None): INTERPOSITION_MESSAGE = "Interposing for {url}" Logger.info(INTERPOSITION_MESSAGE.format(url=url)) # What is the actual target to download given the URL? Sometimes we would # like to transform the given URL to the intended target; e.g. "/simple/" # => "/simple/index.html". target_filepath = self.get_target_filepath(url) # TODO: Set valid headers fetched from the actual download. # NOTE: Important to guess the mime type from the target_filepath, not the # unmodified URL. content_type, content_encoding = mimetypes.guess_type(target_filepath) headers = { # NOTE: pip refers to this same header in at least these two duplicate # ways. "content-type": content_type, "Content-Type": content_type, } # Download the target filepath determined by the original URL. temporary_directory, temporary_filename = self.download_target(target_filepath) if filename is None: # If no filename is given, use the temporary file. filename = temporary_filename else: # Otherwise, copy TUF-downloaded file in its own directory # to the location user specified. shutil.copy2(temporary_filename, filename) return filename, headers
def wrapper(self, *args, **kwargs): # We assume that the first argument to instancemethod is a URL-like object; # that is, either a string or a urllib2.Request. url_object = args[0] data = kwargs.get("data") # If this is a urllib2.Request... if isinstance(url_object, urllib2.Request): # If this is a GET HTTP method... if url_object.get_method() == "GET": # ...then you should check with TUF. url = url_object.get_full_url() else: # ...otherwise, revert to default behaviour. Logger.warn(NON_GET_HTTP_METHOD_MESSAGE.format(method=url_object.get_method(), url=url_object.get_full_url())) return instancemethod(self, *args, **kwargs) # ...otherwise, we assume this is a string. else: url = url_object updater = __updater_controller.get(url) # If TUF has not been configured for this URL... if updater is None: # ...then revert to default behaviour. return instancemethod(self, *args, **kwargs) else: # ...otherwise, use TUF to get this document. return updater.open(url, data=data)
def __urllib2_urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT): """Create a file-like object for the specified URL to read from.""" # We assume that the first argument to instancemethod is a URL-like object; # that is, either a string or a urllib2.Request. updater = None # If this is a urllib2.Request... if isinstance(url, urllib2.Request): # If this is a GET HTTP method... if url.get_method() == "GET": # ...then you should check with TUF. updater = __updater_controller.get(url.get_full_url()) else: # ...otherwise, revert to default behaviour. Logger.warn(NON_GET_HTTP_METHOD_MESSAGE.format(method=url.get_method(), url=url.get_full_url())) return urllib2.urlopen(url, data=data, timeout=timeout) else: # ...otherwise, we assume this is a string. updater = __updater_controller.get(url) if updater is None: return urllib2.urlopen(url, data=data, timeout=timeout) else: response = updater.open(url, data=data) # See urllib2.AbstractHTTPHandler.do_open # TODO: let Updater handle this response.msg = "" return response
def get(self, url): """Get an Updater, if any, for this URL. Assumptions: - @url is a string.""" GENERIC_WARNING_MESSAGE = "No updater or interposition for url={url}" DIFFERENT_NETLOC_MESSAGE = "We have an updater for netloc={netloc1} but not for netlocs={netloc2}" HOSTNAME_FOUND_MESSAGE = "Found updater for hostname={hostname}" HOSTNAME_NOT_FOUND_MESSAGE = "No updater for hostname={hostname}" updater = None try: parsed_url = urlparse.urlparse(url) hostname = parsed_url.hostname port = parsed_url.port or 80 netloc = parsed_url.netloc network_location = "{hostname}:{port}".format(hostname=hostname, port=port) # Sometimes parsed_url.netloc does not have a port (e.g. 80), # so we do a double check. network_locations = set((netloc, network_location)) updater = self.__updaters.get(hostname) if updater is None: Logger.warn(HOSTNAME_NOT_FOUND_MESSAGE.format(hostname=hostname)) else: # Ensure that the updater is meant for this (hostname, port). if updater.configuration.network_location in network_locations: Logger.info(HOSTNAME_FOUND_MESSAGE.format(hostname=hostname)) # Raises an exception in case we do not recognize how to # transform this URL for TUF. In that case, there will be no # updater for this URL. target_filepath = updater.get_target_filepath(url) else: # Same hostname, but different (not user-specified) port. Logger.warn(DIFFERENT_NETLOC_MESSAGE.format( netloc1=updater.configuration.network_location, netloc2=network_locations)) updater = None except: Logger.exception(GENERIC_WARNING_MESSAGE.format(url=url)) updater = None finally: if updater is None: Logger.warn(GENERIC_WARNING_MESSAGE.format(url=url)) return updater
def add(self, configuration): """Add an Updater based on the given Configuration.""" UPDATER_ADDED_MESSAGE = "Updater added for {configuration}." repository_mirror_hostnames = self.__check_configuration(configuration) # If all is well, build and store an Updater, and remember hostnames. self.__updaters[configuration.hostname] = Updater(configuration) self.__repository_mirror_hostnames.update(repository_mirror_hostnames) Logger.info(UPDATER_ADDED_MESSAGE.format(configuration=configuration))
def __init__(self, configuration): CREATED_TEMPDIR_MESSAGE = "Created temporary directory at {tempdir}" self.configuration = configuration # A temporary directory used for this updater over runtime. self.tempdir = tempfile.mkdtemp() Logger.debug(CREATED_TEMPDIR_MESSAGE.format(tempdir=self.tempdir)) # must switch context before instantiating updater # because updater depends on some module (tuf.conf) variables self.switch_context() self.updater = tuf.client.updater.Updater(self.configuration.hostname, self.configuration.repository_mirrors)
def __check_configuration(self, configuration): """ If the given Configuration is invalid, I raise an exception. Otherwise, I return some information about the Configuration, such as repository mirror hostnames. """ INVALID_REPOSITORY_MIRROR = "Invalid repository mirror {repository_mirror}!" # Updater has a "global" view of configurations, so it performs # additional checks after Configuration's own local checks. assert isinstance(configuration, Configuration) # Restrict each (incoming, outgoing) hostname pair to be unique across # configurations; this prevents interposition cycles, amongst other # things. # GOOD: A -> { A:X, A:Y, B, ... }, C -> { D }, ... # BAD: A -> { B }, B -> { C }, C -> { A }, ... assert configuration.hostname not in self.__updaters assert configuration.hostname not in self.__repository_mirror_hostnames # Parse TUF server repository mirrors. repository_mirrors = configuration.repository_mirrors repository_mirror_hostnames = set() for repository_mirror in repository_mirrors: mirror_configuration = repository_mirrors[repository_mirror] try: url_prefix = mirror_configuration["url_prefix"] parsed_url = urlparse.urlparse(url_prefix) mirror_hostname = parsed_url.hostname # Restrict each (incoming, outgoing) hostname pair to be unique # across configurations; this prevents interposition cycles, # amongst other things. assert mirror_hostname not in self.__updaters assert mirror_hostname not in self.__repository_mirror_hostnames # Remember this mirror's hostname for the next network_location. repository_mirror_hostnames.add(mirror_hostname) except: error_message = \ INVALID_REPOSITORY_MIRROR.format(repository_mirror=repository_mirror) Logger.exception(error_message) raise InvalidConfiguration(error_message) return repository_mirror_hostnames
def __read_configuration(configuration_handler, filename="tuf.interposition.json", parent_repository_directory=None, parent_ssl_certificates_directory=None): """ A generic function to read TUF interposition configurations off a file, and then handle those configurations with a given function. configuration_handler must be a function which accepts a tuf.interposition.Configuration instance. Returns the parsed configurations as a dictionary of configurations indexed by hostnames.""" INVALID_TUF_CONFIGURATION = "Invalid configuration for {network_location}!" INVALID_TUF_INTERPOSITION_JSON = "Invalid configuration in {filename}!" NO_CONFIGURATIONS = "No configurations found in configuration in {filename}!" # Configurations indexed by hostnames. parsed_configurations = {} try: with open(filename) as tuf_interposition_json: tuf_interpositions = json.load(tuf_interposition_json) configurations = tuf_interpositions.get("configurations", {}) if len(configurations) == 0: raise InvalidConfiguration(NO_CONFIGURATIONS.format(filename=filename)) else: for network_location, configuration in configurations.iteritems(): try: configuration_parser = ConfigurationParser(network_location, configuration, parent_repository_directory=parent_repository_directory, parent_ssl_certificates_directory=parent_ssl_certificates_directory) configuration = configuration_parser.parse() configuration_handler(configuration) parsed_configurations[configuration.hostname] = configuration except: Logger.exception(INVALID_TUF_CONFIGURATION.format(network_location=network_location)) raise except: Logger.exception(INVALID_TUF_INTERPOSITION_JSON.format(filename=filename)) raise else: return parsed_configurations
def get_repository_mirrors(self, hostname, port, ssl_certificates): """Parse TUF server repository mirrors.""" INVALID_REPOSITORY_MIRROR = "Invalid repository mirror {repository_mirror}!" repository_mirrors = self.configuration["repository_mirrors"] repository_mirror_network_locations = set() for repository_mirror in repository_mirrors: mirror_configuration = repository_mirrors[repository_mirror] try: url_prefix = mirror_configuration["url_prefix"] parsed_url = urlparse.urlparse(url_prefix) mirror_hostname = parsed_url.hostname mirror_port = parsed_url.port or 80 mirror_scheme = parsed_url.scheme mirror_netloc = "{hostname}:{port}".format(hostname = mirror_hostname, port = mirror_port) # TODO: warn is ssl_certificates is specified, # but there is no mirror_scheme == "https" if mirror_scheme == "https": assert os.path.isfile(ssl_certificates) # No single-edge cycle in interposition. # GOOD: A -> { A:XYZ, ... } # BAD: A -> { A, ... } assert not (mirror_hostname == hostname and mirror_port == port) # Unique network location over repository mirrors. # GOOD: A -> { A:X, A:Y, ... } # BAD: A -> { A:X, A:X, ... } assert mirror_netloc not in repository_mirror_network_locations # Remember this mirror's network location to check the rest of the mirrors. repository_mirror_network_locations.add(mirror_netloc) except: error_message = \ INVALID_REPOSITORY_MIRROR.format(repository_mirror=repository_mirror) Logger.exception(error_message) raise InvalidConfiguration(error_message) return repository_mirrors
def remove(self, configuration): """Remove an Updater matching the given Configuration.""" UPDATER_REMOVED_MESSAGE = "Updater removed for {configuration}." assert isinstance(configuration, Configuration) repository_mirror_hostnames = configuration.get_repository_mirror_hostnames() assert configuration.hostname in self.__updaters assert repository_mirror_hostnames.issubset(self.__repository_mirror_hostnames) # If all is well, remove the stored Updater as well as its associated # repository mirror hostnames. del self.__updaters[configuration.hostname] self.__repository_mirror_hostnames.difference_update(repository_mirror_hostnames) Logger.info(UPDATER_REMOVED_MESSAGE.format(configuration=configuration))
def get_target_filepath(self, source_url): """Given source->target map, figure out what TUF *should* download given a URL.""" WARNING_MESSAGE = "Possibly invalid target_paths for " + \ "{network_location}! No TUF interposition for {url}" parsed_source_url = urlparse.urlparse(source_url) target_filepath = None try: # Does this source URL match any regular expression which tells us # how to map the source URL to a target URL understood by TUF? for target_path in self.configuration.target_paths: # target_path: { "regex_with_groups", "target_with_group_captures" } # e.g. { ".*(/some/directory)/$", "{0}/index.html" } source_path_pattern, target_path_pattern = target_path.items()[0] source_path_match = re.match(source_path_pattern, parsed_source_url.path) # TODO: A failure in string formatting is *critical*. if source_path_match is not None: target_filepath = target_path_pattern.format(*source_path_match.groups()) # If there is more than one regular expression which # matches source_url, we resolve ambiguity by order of # appearance. break # If source_url does not match any regular expression... if target_filepath is None: # ...then we raise a predictable exception. raise URLMatchesNoPattern(source_url) except: Logger.exception(WARNING_MESSAGE.format( network_location=self.configuration.network_location, url=source_url)) raise else: # TUF assumes that target_filepath does not begin with a '/'. target_filepath = target_filepath.lstrip('/') return target_filepath
def retrieve(self, url, filename=None, reporthook=None, data=None): INTERPOSITION_MESSAGE = "Interposing for {url}" # TODO: set valid headers content_type, content_encoding = mimetypes.guess_type(url) headers = {"content-type": content_type} Logger.info(INTERPOSITION_MESSAGE.format(url=url)) target_filepath = self.get_target_filepath(url) temporary_directory, temporary_filename = self.download_target(target_filepath) if filename is None: # If no filename is given, use the temporary file. filename = temporary_filename else: # Otherwise, copy TUF-downloaded file in its own directory # to the location user specified. shutil.copy2(temporary_filename, filename) return filename, headers
def Configure(self, argv): self.args = argv self.messagePrinter = Logger() self.scriptPath, self.scriptExtension = os.path.splitext(argv[0]) self.scriptPath, self.scriptName = os.path.split(self.scriptPath) self.argparser.parse_args(args=argv[1:], namespace=self) self.messagePrinter.isDbgEnabled = self.debugMessages self.messagePrinter.isInfoEnabled = self.verbose self.messagePrinter.isErrEnabled = not self.silenceErrors if self.scriptIni is None: iniPath = '' if self.scriptPath is not None: iniPath = self.scriptPath self.scriptIni = toPosixPath(os.path.join(iniPath, '{0}.ini'.format(self.scriptName))) self.parser = configparser.ConfigParser(allow_no_value = True) # see https://docs.python.org/2/library/configparser.html self.parser.optionxform = str # make case-sensitive as per https://docs.python.org/2/library/configparser.html self.isConfigured = True if os.path.exists(self.scriptIni): self.parser.read(self.scriptIni) elif os.path.exists(os.path.join(self.scriptPath, self.scriptIni)): self.parser.read(os.path.join(self.scriptPath, self.scriptIni)) else: msg = "No configuration file found. Searched, current directory and script directory directory for {0}.".format(self.scriptIni) self.messagePrinter.error(msg) self.isConfigured = False self.databaseFilename = ':memory:' self.sourcePath = './' return try: if self.databaseFilename is None: self.databaseFilename = self.parser.get("Output", "DatabaseFilename") except: self.databaseFilename = ':memory:' try: if self.sourcePath is None: self.sourcePath = self.parser.get("Paths","SourceRoot") # Make the read SourceRoot path relative to the INI file's path. if self.isConfigured: iniPath, iniFilename = os.path.split(self.scriptIni) self.sourcePath = toPosixPath(os.path.normpath(os.path.join(iniPath, self.sourcePath))) print('source-path: {0}'.format(self.sourcePath)) except: self.sourcePath = './'
def __check_configuration_on_add(self, configuration): """ If the given Configuration is invalid, I raise an exception. Otherwise, I return some information about the Configuration, such as repository mirror hostnames. """ INVALID_REPOSITORY_MIRROR = "Invalid repository mirror {repository_mirror}!" # Updater has a "global" view of configurations, so it performs # additional checks after Configuration's own local checks. assert isinstance(configuration, Configuration) # Restrict each (incoming, outgoing) hostname pair to be unique across # configurations; this prevents interposition cycles, amongst other # things. # GOOD: A -> { A:X, A:Y, B, ... }, C -> { D }, ... # BAD: A -> { B }, B -> { C }, C -> { A }, ... assert configuration.hostname not in self.__updaters assert configuration.hostname not in self.__repository_mirror_hostnames # Check for redundancy in server repository mirrors. repository_mirror_hostnames = configuration.get_repository_mirror_hostnames() for mirror_hostname in repository_mirror_hostnames: try: # Restrict each hostname in every (incoming, outgoing) pair to be # unique across configurations; this prevents interposition cycles, # amongst other things. assert mirror_hostname not in self.__updaters assert mirror_hostname not in self.__repository_mirror_hostnames except: error_message = \ INVALID_REPOSITORY_MIRROR.format(repository_mirror=mirror_hostname) Logger.exception(error_message) raise InvalidConfiguration(error_message) return repository_mirror_hostnames
class ImportPayPal(object): ''' This class imports a PayPal CSV file into the database. ''' def __init__(self, fname): self.logger = Logger(self, Logger.DEBUG) self.logger.debug(sys._getframe().f_code.co_name) self.fname = fname self.data = Database.get_instance() self.accepted = 0 self.rejected = 0 self.legend = [ 'Date', 'Time', 'TimeZone', 'Name', 'Type', 'Status', 'Currency', 'Gross', 'Fee', 'Net', 'FromEmail', 'ToEmail', 'TransactionID', 'ShippingAddress', 'AddressStatus', 'ItemTitle', 'ItemID', 'Shipping', 'InsuranceAmount', 'SalesTax', 'Option1Name', 'Option1Value', 'Option2Name', 'Option2Value', 'ReferenceTxnID', 'InvoiceNumber', 'CustomNumber', 'Quantity', 'ReceiptID', 'Balance', 'AddressLine1', 'AddressLine2', 'City', 'State', 'PostalCode', 'Country', 'Phone', 'Subject', 'Note', 'CountryCode', 'BalanceImpact' ] @debugger def import_all(self): ''' This is the top level interface for the importer. All of the other methods are private. ''' self.logger.debug("import all records from %s" % (self.fname)) #try: # import the CSV file into the database. self._read_file() # get the various tables set up codes = self._countries() cust = self._customers() vend = self._vendors() sales = self._sales() purch = self._purchases() text = 'Imported records:\n' text += ' %d country codes\n' % (codes) text += ' %d unique customer entries\n' % (cust) text += ' %d unique vendor entries\n' % (vend) text += ' %d sale entries\n' % (sales) text += ' %d purchase entries\n' % (purch) text += ' %d CSV lines accepted\n' % (self.accepted) text += ' %d CSV lines rejected\n' % (self.rejected) showinfo('Import', text) # except Exception as e: # showerror('ERROR', str(e)) @debugger def _read_file(self): ''' Read the CSV file into an array of lines where each one is a dictionary with the column names as keys. ''' with open(self.fname, "r") as fh: reader = csv.reader(fh) line = [] for line in reader: break if not line[1] == 'Time': raise ('File selected is not a PayPal CSV import file.') for line in reader: rec = {} for idx, item in enumerate(line): #if item == '': # rec[self.legend[idx]] = None #else: rec[self.legend[idx]] = item rec['imported_country'] = False rec['imported_customer'] = False rec['imported_vendor'] = False rec['imported_sale'] = False rec['imported_purchase'] = False if not self.data.if_rec_exists('RawImport', 'TransactionID', rec['TransactionID']): self.data.insert_row('RawImport', rec) self.accepted += 1 else: self.rejected += 1 self.data.commit() @debugger def _countries(self): ''' Read the new import and copy new country codes into the country codes table. ''' data = self.data.get_row_list('RawImport', 'imported_country = false') if data is None: return 0 count = 0 for item in data: if item['CountryCode'] != '' and not self.data.if_rec_exists( 'Country', 'abbreviation', item['CountryCode']): rec = { 'name': item['Country'], 'abbreviation': item['CountryCode'] } self.data.insert_row('Country', rec) count += 1 self.data.update_row_by_id('RawImport', {'imported_country': True}, item['ID']) self.data.commit() return count @debugger def _customers(self): ''' Find all of the new customer records and copy the data into the customers table. ''' data = self.data.get_row_list( 'RawImport', 'imported_customer = false and BalanceImpact = \'Credit\'') if data is None: showinfo('INFO', 'There are no customer contacts to import.') return 0 count = 0 for item in data: if item['Type'] == 'Website Payment' or item[ 'Type'] == 'General Payment': # Yes it's a customer if not self.data.if_rec_exists('Customer', 'name', item['Name']): rec = { 'date_created': item['Date'], 'name': item['Name'], 'address1': item['AddressLine1'], 'address2': item['AddressLine2'], 'state': item['State'], 'city': item['City'], 'zip': item['PostalCode'], 'email_address': item['FromEmail'], 'email_status_ID': self.data.get_id_by_row('EmailStatus', 'name', 'primary'), 'phone_number': item['Phone'], 'phone_status_ID': self.data.get_id_by_row('PhoneStatus', 'name', 'primary'), 'description': 'Imported from PayPal', 'notes': item['Subject'], 'country_ID': self.data.get_id_by_row('Country', 'abbreviation', item['CountryCode']), 'class_ID': self.data.get_id_by_row('ContactClass', 'name', 'retail') } self.data.insert_row('Customer', rec) count += 1 # BUG: (fixed) When there are multiple instances of a name, the sale or purch record does not get imported # because the imported_customer field does not get updated due to the duplicate name interlock. self.data.update_row_by_id('RawImport', {'imported_customer': True}, item['ID']) self.data.commit() return count @debugger def _vendors(self): ''' Find all of the new vendor records and copy the data into the vendor table. ''' data = self.data.get_row_list( 'RawImport', 'imported_vendor = false and BalanceImpact = \'Debit\'') if data is None: showinfo('INFO', 'There are no customer contacts to import.') return 0 count = 0 for item in data: if item['Name'] != '' and item['Name'] != 'PayPal': if not self.data.if_rec_exists('Vendor', 'name', item['Name']): rec = { 'date_created': item['Date'], 'name': item['Name'], 'contact_name': '', 'email_address': item['ToEmail'], 'email_status_ID': self.data.get_id_by_row('EmailStatus', 'name', 'primary'), 'phone_number': '', 'phone_status_ID': self.data.get_id_by_row('PhoneStatus', 'name', 'primary'), 'description': item['ItemTitle'], 'notes': item['Subject'], 'type_ID': self.data.get_id_by_row('VendorType', 'name', 'unknown'), } self.data.insert_row('Vendor', rec) self.data.update_row_by_id('RawImport', {'imported_vendor': True}, item['ID']) count += 1 self.data.commit() return count @debugger def _sales(self): ''' Find the sales records and copy the data into the sales database table. ''' data = self.data.get_row_list( 'RawImport', 'imported_sale = false and imported_customer = true and BalanceImpact = \'Credit\'' ) if data is None: showinfo('INFO', 'There are no sales transcations to import.') return 0 count = 0 for item in data: if item['Name'] != '' and item['Name'] != 'PayPal': rec = { 'date': item['Date'], 'customer_ID': self.data.get_id_by_row('Customer', 'name', item['Name']), 'raw_import_ID': int(item['ID']), 'status_ID': self.data.get_id_by_row('SaleStatus', 'name', 'complete'), 'transaction_uuid': item['TransactionID'], 'gross': self.data.convert_value(item['Gross'], float), 'fees': self.data.convert_value(item['Fee'], float), 'shipping': self.data.convert_value(item['Shipping'], float), 'notes': item['Subject'] + '\n' + item['ItemTitle'], 'committed': False } self.data.insert_row('SaleRecord', rec) count += 1 self.data.update_row_by_id('RawImport', {'imported_sale': True}, item['ID']) self.data.commit() return count @debugger def _purchases(self): ''' Find all of the purchase records and copy the data into the purchase database table. ''' data = self.data.get_row_list( 'RawImport', 'imported_purchase = false and imported_vendor = true and BalanceImpact = \'Debit\'' ) if data is None: showinfo('INFO', 'There are no purchase transcations to import.') return 0 count = 0 for item in data: if item['Name'] != '' and item['Name'] != 'PayPal': gross = item['Gross'] tax = item['SalesTax'] shipping = item['Shipping'] rec = { 'date': item['Date'], 'raw_import_ID': int(item['ID']), 'vendor_ID': self.data.get_id_by_row('Vendor', 'name', item['Name']), 'status_ID': self.data.get_id_by_row('PurchaseStatus', 'name', 'complete'), 'type_ID': self.data.get_id_by_row('PurchaseType', 'name', 'unknown'), 'transaction_uuid': item['TransactionID'], 'gross': self.data.convert_value(item['Gross'], float), 'tax': self.data.convert_value(item['SalesTax'], float), 'shipping': self.data.convert_value(item['Shipping'], float), 'notes': item['Subject'] + '\n' + item['ItemTitle'], 'committed': False } self.data.insert_row('PurchaseRecord', rec) self.data.update_row_by_id('RawImport', {'imported_purchase': True}, item['ID']) count += 1 self.data.commit() return count
current_ts = ts row['diff'] = abs(float(row['diff'])) if diff < 0: borrowers.append(row) else: lenders_len = len(lenders) if lenders_len < 1: lenders.append(row) else: for i, l in enumerate(lenders): if l['diff'] < row['diff']: lenders.insert(i, row) break else: if i == (lenders_len - 1): lenders.insert(i+1, row) break # log(lenders) # log('-----------------------') # log(str(counter) + ':' + str(ts)) input_csv_file.close() output_csv_file.close() if __name__ == "__main__": log.debug_off() start = datetime.now() main(sys.argv[1:]) print("Finished in ", datetime.now()-start, '.')
class EntryBox(tk.Frame): ''' Implement a consistent interface to the entry widget ''' def __init__(self, master, table, column, width=None, readonly=False, *args, **kargs): ''' master = the frame to bind this frame to name = the text of the label table = the name of the database table that is associated with this widget column = the name of the column this widget associates with lw = label width cw = control width ''' self.logger = Logger(self, level=Logger.INFO) self.logger.debug("EntryBox enter constructor") super().__init__(master, *args, **kargs) self.table = table self.column = column self.readonly = readonly self.content = tk.StringVar(master) self.entry = tk.Entry(self, textvariable=self.content, width=width) self.entry.grid(row=0, column=1, padx=5, pady=5, sticky=tk.W) if self.readonly: self.entry.configure(state='readonly') self.row = { 'table': self.table, 'column': self.column, 'self': self, 'hasid': None } self.logger.debug("EntryBox leave constructor") @debugger def read(self): return self.content.get() @debugger def write(self, val): if self.readonly: self.entry.configure(state='normal') self.content.set(val) if self.readonly: self.entry.configure(state='readonly') @debugger def clear(self): self.content.set('') @debugger def get_line(self): ''' Return the form entry to update the form. ''' return self.row
class ButtonBox(tk.Frame): ''' Make the button box and register the events. ''' def __init__(self, master, form, disable_select=False, disable_new=False, *args, **kargs): ''' master = The frame to bind the widgets to. form = Name of the form to bind the events to. ''' self.logger = Logger(self, level=Logger.INFO) self.logger.debug("NotesBox enter constructor") super().__init__(master, *args, **kargs) row = 0 col = 0 self.form = form self.events = EventHandler.get_instance() tk.Button(self, text='Prev', command=self.prev_btn).grid(row=row, column=col, padx=5, pady=5) col += 1 tk.Button(self, text='Next', command=self.next_btn).grid(row=row, column=col, padx=5, pady=5) if not disable_select: col += 1 tk.Button(self, text='Select', command=self.select_btn).grid(row=row, column=col, padx=5, pady=5) if not disable_new: col += 1 tk.Button(self, text='New', command=self.new_btn).grid(row=row, column=col, padx=5, pady=5) col += 1 tk.Button(self, text='Save', command=self.save_btn).grid(row=row, column=col, padx=5, pady=5) col += 1 tk.Button(self, text='Delete', command=self.delete_btn).grid(row=row, column=col, padx=5, pady=5) @debugger def register_events(self, next, prev, select, new, save, delete): ''' next = callback for the next button prev = callback for the prev button select = callback for the select button new = callback for the new button save = callback for the save button delete = callback for the delete button ''' self.events.register_event('next_btn_%s' % (self.form), next) self.events.register_event('prev_btn_%s' % (self.form), prev) self.events.register_event('select_btn_%s' % (self.form), select) self.events.register_event('new_btn_%s' % (self.form), new) self.events.register_event('save_btn_%s' % (self.form), save) self.events.register_event('delete_btn_%s' % (self.form), delete) @debugger def next_btn(self): self.events.raise_event('next_btn_%s' % (self.form)) @debugger def prev_btn(self): self.events.raise_event('prev_btn_%s' % (self.form)) @debugger def select_btn(self): self.events.raise_event('select_btn_%s' % (self.form)) @debugger def new_btn(self): self.events.raise_event('new_btn_%s' % (self.form)) @debugger def save_btn(self): self.events.raise_event('save_btn_%s' % (self.form)) @debugger def delete_btn(self): self.events.raise_event('delete_btn_%s' % (self.form))
scores_template = '../../out/scores/crowdai/scores_{}_{}_{}_{}_{}.out' indices_template = '../../out/models/crowdai/indices_{}_{}_{}_{}_{}.out' batch_size = 3 img_height = 96 img_width = 1366 channels = 1 num_epochs = 500 # Decaying by factor of ~0.91 after each epoch (for batch_size 6) lr_starting = 9e-4 lr_decay = 0.9999714 start_time = time.time() logger = Logger(batch_size, num_epochs, lr_starting) score_data = dict() score_data['train_loss'] = [] score_data['valid_loss'] = [] score_data['f1_score'] = [] score_data['lr'] = [] optimizers = { 'sgd': SGD(lr=0.001, momentum=0.9, nesterov=True), 'adam': Adam(lr=lr_starting, decay=lr_decay) } def build_model(output_size): channel_axis = 3 freq_axis = 1
class SelectItem(BaseDialog): ''' Create a list of items called 'name' from a table and return the database ID of the item in item_id. ''' def __init__(self, master, table, thing=None): self.logger = Logger(self, level=Logger.DEBUG) self.logger.debug('SelectItem enter constructor') self.table = table if thing is None: self.thing = 'Item' else: self.thing = thing self.item_id = -1 super().__init__(master) self.wait_window(self) self.logger.debug('SelectItem leave constructor') @debugger def body(self, master): self.title("Select %s" % (self.thing)) self.data = Database.get_instance() padx = 6 pady = 2 frame = tk.Frame(master, bd=1, relief=tk.RIDGE) frame.grid(row=0, column=0, padx=4, pady=7) tk.Label(frame, text="Select %s" % (self.thing), font=("Helvetica", 14)).grid(row=0, column=0, columnspan=2) ###################### # Populate the combo boxes lst = self.data.populate_list(self.table, 'name') lst.sort() ###################### # Show the boxes tk.Label(frame, text='Name:').grid(row=1, column=0) self.cbb = ttk.Combobox(frame, values=lst) self.cbb.grid(row=1, column=1, padx=padx, pady=pady) try: self.cbb.current(0) except tk.TclError: mb.showerror("TCL ERROR", "No records are available to select for this table.") @debugger def validate(self): # Since the name was selected from the list, there is no need to # validate. return True @debugger def apply(self): ''' Populate the form with the selected data. ''' id = self.data.get_id_by_name(self.table, self.cbb.get()) self.item_id = id
def main(svc_input, configs): logger = Logger("查询日志", verbose=True) log_file_name = "log%s_%s.txt" % (svc_input.replace("?", "#"), DateTimeUtil.get_current_datetime(is_date=True)) log_file_path = WindowsUtil.convert_win_path(os.path.join(temp_dir, log_file_name)) logger.info("[开始查询] %s" % svc_input) try: # 找到本地匹配的保修历史记录 history_zip = ZipFileSVC(zip_file_path=history_zipfile, mode='a') start_time = DateTimeUtil.get_current_datetime() # 创建出所有可能查询码 svc_generator = SVCGenerator(svc_input, logger) logger.info("创建出所有可能查询码:%s" % len(svc_generator.target_svc_set)) # 根据本地匹配的非法查询码历史,筛选出目标查询码,以及非法查询码 existed_svc = history_zip.find_file_regex(svc_generator.regex) svc_generator.generate_target_svc_batch(existed_svc, invalid_history_file_path) # 调用戴尔查询API,并将API数据转化为实体类数据 output_dell_asset_list = list([]) if svc_generator.target_svc_set: batch = Batch(logger, configs) api_dell_asset_list = batch.begin(svc_generator.target_svc_set) output_dell_asset_list = api_dell_asset_list logger.info("从API中总共得到%s个结果" % (len(api_dell_asset_list))) logger.info("将实体类序列化到本地临时TXT文件") temp_text_files_path = DellAsset.serialize_txt_batch(api_dell_asset_list, temp_dir) logger.info("将序列化临时文件存到本地zip历史记录,总数:%s" % len(temp_text_files_path)) history_zip.add_new_file_batch(temp_text_files_path) logger.info("删除临时 %s 个TXT文件" % len(temp_text_files_path)) for file_path in temp_text_files_path: FileUtil.delete_file(file_path) logger.info("将API得到的实体类和历史记录实体类合并") else: logger.warn("目标查询码为空,仅从从历史记录中导出结果") for svc in svc_generator.existed_svc_set: dell_asset_content = history_zip.get_member_content(file_name="%s.txt" % svc) output_dell_asset_list.append(DellAsset.deserialize_txt(dell_asset_content)) logger.info("添加历史记录,总共得到%s个结果" % (len(output_dell_asset_list))) excel_output_path = WindowsUtil.convert_win_path(os.path.join(excel_dir, "%s.xlsx" % svc_generator.get_file_name())) DellAsset.save_as_excel_batch(output_dell_asset_list, excel_output_path) if FileUtil.is_path_existed(excel_output_path): logger.info("存为Excel文档成功") end_time = DateTimeUtil.get_current_datetime() logger.info("总用时 %s " % DateTimeUtil.datetime_diff(start_time, end_time)) logger.info("[查询结束] 总共%s个结果 保存在:%s" % (len(output_dell_asset_list), excel_output_path)) else: logger.error("[保存结果失败] %s" % excel_output_path) except Exception as e: # 若程序出现错误失败,发送邮件 logger.error("[查询失败] 已发送报告 请等待解决") logger.error("%s\n%s" % (e, traceback.format_exc())) logger.save(log_file_path) email_api_key = configs["email_api_key"] email = Email(email_api_key, subject="[查询失败] %s %s" % (DateTimeUtil.get_current_datetime(is_date=True), svc_input)) email.add_attachment(log_file_path) email.send(cc_mode=logger.has_error)
class Importer(object): def __init__(self): #, master): self.logger = Logger(self, Logger.DEBUG) self.logger.debug(sys._getframe().f_code.co_name) #self.master = master self.legend = [ 'Date', 'Time', 'TimeZone', 'Name', 'Type', 'Status', 'Currency', 'Gross', 'Fee', 'Net', 'FromEmail', 'ToEmail', 'TransactionID', 'ShippingAddress', 'AddressStatus', 'ItemTitle', 'ItemID', 'Shipping', 'InsuranceAmount', 'SalesTax', 'Option1Name', 'Option1Value', 'Option2Name', 'Option2Value', 'ReferenceTxnID', 'InvoiceNumber', 'CustomNumber', 'Quantity', 'ReceiptID', 'Balance', 'AddressLine1', 'AddressLine2', 'City', 'State', 'PostalCode', 'Country', 'Phone', 'Subject', 'Note', 'CountryCode', 'BalanceImpact' ] self.data = Database.get_instance() @debugger def import_all(self): ''' Perform all of the steps to import the entire CSV file ''' total = self.read_all() if total > 0: #data = self.get_sales() codes = self.import_country_codes() cust = self.import_customer_contacts() sales = self.do_sales_transactions() #data = self.get_purchases() #codes += self.import_country_codes() vend = self.import_vendor_contacts() purch = self.do_purchase_transactions() text = 'Imported records:\n' text += ' %d total lines in import\n' % (total) text += ' %d country codes\n' % (codes) text += ' %d unique customer entries\n' % (cust) text += ' %d unique vendor entries\n' % (vend) text += ' %d sale entries\n' % (sales) text += ' %d purchase entries' % (purch) mb.showinfo('Import', text) else: mb.showinfo("Import", 'There were no records found to import.') @debugger def validate_email(self, email): if bool(re.search(r"^[\w\.\+\-]+\@[\w]+\.[a-z]{2,3}$", email)): return email else: return '' @debugger def read_file(self): ''' Handle the file IO for the CSV file ''' #master = tkinter.Tk() fh = askopenfile(mode='r', filetypes=[('Spread Sheets', '*.csv *.CSV'), ('All Files', '*')]) if fh is None: return None # error or cancel raw = [] reader = csv.reader(fh) for line in reader: raw.append(line) fh.close() #master.destroy() #print(raw) return (raw, fh.name) @debugger def read_all(self): ''' Read all of the lines in the CSV into the database ''' raw_data = self.read_file() if raw_data is None: return 0 raw = raw_data[0] name = raw_data[1] if self.data.if_rec_exists('ImportedFileNames', 'name', name): mb.showwarning( 'WARNING', 'File name \"%s\" has already been imported. Rename file and try again.' % (name)) return 0 else: tmp = { 'name': name, 'date': time.strftime('%m/%d/%Y', time.localtime()) } self.data.insert_row('ImportedFileNames', tmp) data = [] count = 0 for line in raw: tmp = {} for idx, item in enumerate(line): tmp[self.legend[idx]] = item data.append(tmp) if tmp['Status'] == 'Completed': # and tmp['Name'] != '' and tmp['Name'] != 'PayPal': tmp['imported_country'] = False tmp['imported_customer'] = False tmp['imported_vendor'] = False tmp['imported_sale'] = False tmp['imported_purchase'] = False if not self.data.if_rec_exists('RawImport', 'TransactionID', tmp['TransactionID']): self.data.insert_row('RawImport', tmp) count += 1 self.data.commit() return count @debugger def import_country_codes(self): ''' Find the new country codes and store them ''' data = self.data.get_row_list('RawImport', 'imported_country = false') if data is None: return count = 0 for item in data: if item['CountryCode'] != '' and not self.data.if_rec_exists( 'Country', 'abbreviation', item['CountryCode']): rec = { 'name': item['Country'], 'abbreviation': item['CountryCode'] } self.data.insert_row('Country', rec) count += 1 self.data.update_row_by_id('RawImport', {'imported_country': True}, item['ID']) self.data.commit() return count @debugger def import_customer_contacts(self): ''' Store contact information for customers ''' data = self.data.get_row_list( 'RawImport', 'imported_customer = false and BalanceImpact = \'Credit\'') if data is None: mb.showinfo('INFO', 'There are no customer contacts to import.') return count = 0 for item in data: if item['Type'] == 'Website Payment' or item[ 'Type'] == 'General Payment': # Yes it's a customer if not self.data.if_rec_exists('Customer', 'name', item['Name']): rec = { 'date_created': item['Date'], 'name': item['Name'], 'address1': item['AddressLine1'], 'address2': item['AddressLine2'], 'state': item['State'], 'city': item['City'], 'zip': item['PostalCode'], 'email_address': item['FromEmail'], 'email_status_ID': self.data.get_id_by_row('EmailStatus', 'name', 'primary'), 'phone_number': item['Phone'], 'phone_status_ID': self.data.get_id_by_row('PhoneStatus', 'name', 'primary'), 'description': 'Imported from PayPal', 'notes': item['Subject'], 'country_ID': self.data.get_id_by_row('Country', 'abbreviation', item['CountryCode']), 'class_ID': self.data.get_id_by_row('ContactClass', 'name', 'retail') } self.data.insert_row('Customer', rec) count += 1 # BUG: (fixed) When there are multiple instances of a name, the sale or purch record does not get imported # because the imported_customer field does not get updated due to the duplicate name interlock. self.data.update_row_by_id('RawImport', {'imported_customer': True}, item['ID']) self.data.commit() #mb.showinfo('INFO', 'Imported %d customer contacts.'%(count)) return count @debugger def import_vendor_contacts(self): ''' Store contact information for vendors ''' data = self.data.get_row_list( 'RawImport', 'imported_vendor = false and BalanceImpact = \'Debit\'') if data is None: mb.showinfo('INFO', 'There are no customer contacts to import.') return count = 0 for item in data: if item['Name'] != '' and item['Name'] != 'PayPal': if not self.data.if_rec_exists('Vendor', 'name', item['Name']): rec = { 'date_created': item['Date'], 'name': item['Name'], 'contact_name': '', 'email_address': self.validate_email(item['ToEmail']), 'email_status_ID': self.data.get_id_by_row('EmailStatus', 'name', 'primary'), 'phone_number': '', 'phone_status_ID': self.data.get_id_by_row('PhoneStatus', 'name', 'primary'), 'description': item['ItemTitle'], 'notes': item['Subject'], 'type_ID': self.data.get_id_by_row('VendorType', 'name', 'unknown'), } self.data.insert_row('Vendor', rec) self.data.update_row_by_id('RawImport', {'imported_vendor': True}, item['ID']) count += 1 self.data.commit() #mb.showinfo('INFO', 'Imported %d vendor contacts.'%(count)) return count @debugger def do_purchase_transactions(self): ''' Handle all of the transaction details for purchases ''' data = self.data.get_row_list( 'RawImport', 'imported_purchase = false and imported_vendor = true and BalanceImpact = \'Debit\'' ) if data is None: mb.showinfo('INFO', 'There are no purchase transcations to import.') return count = 0 for item in data: if item['Name'] != '' and item['Name'] != 'PayPal': gross = item['Gross'] tax = item['SalesTax'] shipping = item['Shipping'] rec = { 'date': item['Date'], 'raw_import_ID': int(item['ID']), 'vendor_ID': self.data.get_id_by_row('Vendor', 'name', item['Name']), 'status_ID': self.data.get_id_by_row('PurchaseStatus', 'name', 'complete'), 'type_ID': self.data.get_id_by_row('PurchaseType', 'name', 'unknown'), 'transaction_uuid': item['TransactionID'], 'gross': self.data.convert_value(item['Gross'], float), 'tax': self.data.convert_value(item['SalesTax'], float), 'shipping': self.data.convert_value(item['Shipping'], float), 'notes': item['Subject'] + '\n' + item['ItemTitle'], 'committed': False } self.data.insert_row('PurchaseRecord', rec) self.data.update_row_by_id('RawImport', {'imported_purchase': True}, item['ID']) count += 1 self.data.commit() #mb.showinfo('INFO', 'Imported %d purchase transactions.'%(count)) return count @debugger def do_sales_transactions(self): ''' Do all sales transactions ''' data = self.data.get_row_list( 'RawImport', 'imported_sale = false and imported_customer = true and BalanceImpact = \'Credit\'' ) if data is None: mb.showinfo('INFO', 'There are no sales transcations to import.') return count = 0 for item in data: if item['Name'] != '' and item['Name'] != 'PayPal': rec = { 'date': item['Date'], 'customer_ID': self.data.get_id_by_row('Customer', 'name', item['Name']), 'raw_import_ID': int(item['ID']), 'status_ID': self.data.get_id_by_row('SaleStatus', 'name', 'complete'), 'transaction_uuid': item['TransactionID'], 'gross': self.data.convert_value(item['Gross'], float), 'fees': self.data.convert_value(item['Fee'], float), 'shipping': self.data.convert_value(item['Shipping'], float), 'notes': item['Subject'] + '\n' + item['ItemTitle'], 'committed': False } self.data.insert_row('SaleRecord', rec) count += 1 self.data.update_row_by_id('RawImport', {'imported_sale': True}, item['ID']) self.data.commit() #mb.showinfo('INFO', 'Imported %d sale transactions.'%(count)) return count
#!/usr/bin/python3 import tkinter import traceback, sys from main_frame import MainFrame from utility import Logger #from data_store import DataStore #from configuration import Configuration if __name__ == "__main__": try: logger = Logger(__name__, Logger.DEBUG) logger.debug(sys._getframe().f_code.co_name) top = tkinter.Tk() app = MainFrame(top) top.wm_title("Shop Timer") logger.debug("start main loop") top.mainloop() logger.debug("end main loop") except Exception as e: traceback.print_exception(*sys.exc_info())
class LowerFrame(tkinter.Frame): ''' This class manages the lower from of the display. ''' def __init__(self, master): self.logger = Logger(self, Logger.INFO) self.logger.debug("constructor") self.data_store = DataStore.get_instance() self.master = master #self.line_data = [] self.line_widgits = [] register_event('UPDATE_LOWER_FRAME_EVENT', self.update_frame) register_event('UPDATE_LINES_EVENT', self.set_state) register_event('UPDATE_NOTES_EVENT', self.update_notes) register_event('CHANGE_UNITS_EVENT', self.change_units) tkinter.Label(self.master, width=12, text="Hole").grid(row=0, column=0, sticky=tkinter.W) tkinter.Label(self.master, width=5, text="Interval").grid(row=0, column=1, sticky=tkinter.W) tkinter.Label(self.master, width=10, text="Note").grid(row=0, column=2, sticky=tkinter.W) tkinter.Label(self.master, width=10, text="Frequency").grid(row=0, column=3, sticky=tkinter.W) tkinter.Label(self.master, width=18, text="Hole Size").grid(row=0, column=4, sticky=tkinter.W) tkinter.Label(self.master, width=11, text="Hole Location").grid(row=0, column=5, sticky=tkinter.W) tkinter.Label(self.master, width=12, text="Hole Diff").grid(row=0, column=6, sticky=tkinter.W) tkinter.Label(self.master, width=12, text="Cutoff Freq").grid(row=0, column=7, sticky=tkinter.W) # create all of the lines for n in range(12): lw = LineWidgit(self.master, n) self.line_widgits.append(lw) for idx in range(self.data_store.get_number_holes()): self.line_widgits[idx].grid(row=idx + 1, column=0, columnspan=8, sticky=tkinter.W) self.line_widgits[idx].set_state() self.logger.debug("end constructor") @debugger def update_frame(self): # hide all of the lines for idx in range(12): self.line_widgits[idx].grid_forget() # expose the correct number of lines for idx in range(self.data_store.get_number_holes()): self.line_widgits[idx].grid(row=idx + 1, column=0, columnspan=8, sticky=tkinter.W) self.line_widgits[idx].set_state() self.update_notes() @debugger def set_state(self): ''' Copy data from the datastore into all lines in GUI ''' for line in self.line_widgits: line.set_state() @debugger def get_state(self): ''' Copy data from the line widgits into the data_store ''' for line in self.line_widgits: line.get_state() @debugger def update_notes(self): sel = self.data_store.get_bell_note_select() for index, line in enumerate(self.line_widgits): sel += self.data_store.get_hole_interval(index) self.data_store.set_hole_freq( index, self.data_store.note_table[sel]["frequency"]) self.data_store.set_hole_note( index, self.data_store.note_table[sel]["note"]) line.set_state() raise_event("CALCULATE_EVENT") @debugger def change_units(self): ''' When this is called, it is assumed that the data_store has the wrong units. The values are converted to the current units and the GUI is updated. ''' for line in self.line_widgits: line.change_units()
class Form: #(tk.Frame): ''' Base class for a form. This contains everything needed to display a form and associate the elements of a form with a database record. This allows a complete form to be added to a notebook panel. A single generic driver routine is used to get or set the form with the database. The idea is to capture all of the calls to tkinter having to do with the forms. ''' # widgets that may be added. Used internally to identify them. TITLE = 0 ENTRY = 1 TEXT = 2 COMBO = 3 LABEL = 4 BUTTON = 5 # used by the supplemental_form class in setup_forms.py INDIRECT_LABEL = 6 COMMIT_BTN = 7 PRODUCT = 8 DIR_BROWSER = 9 IMPORT_BTN = 10 def __init__(self, notebook, index, table=None, height=700, width=1000, span=4): ''' Create a form object. A form must be associated with exactly one database table. The form is filled in with exactly one row from the database table. If the form does not use all of the row, that is handled transparently. This class contains frames. It is not, itself, a frame. notebook = The notebook container index = The index of the tab to bind the form to table = The database table where the data from the form is located height = The height of the form in pixels width = the width of the form in pixels span = The number of columns in the form ''' self.logger = Logger(self, level=Logger.DEBUG) self.logger.debug("enter constructor") self.notebook = notebook # container for the form. self.nb_index = index # index of the tab to bind the form to self.table = table # database table to save and retrieve data # frame to bind the widgets to self.owner = self.notebook.get_frame(self.nb_index) # load_form should be over ridden to handle special cases. self.notebook.set_callback(self.nb_index, self.load_form) # database singleton object self.data = Database.get_instance() # the rows may not be sequentially numbered. if not table is None: self.row_list = self.data.get_id_list(self.table) self.row_index = 0 else: self.row_list = None self.row_index = 0 # dict of dicts that lists all of the controls by name. self.controls = {} # actual height and width of the form frame. self.height = height self.width = width self.logger.debug("window size = %d, %d" % (self.height, self.width)) # keep track of the current layout position. self.row = 0 self.col = 0 self.btn_row = 0 self.btn_width = 10 self.padx = 5 self.pady = 5 self.btn_padx = 1 self.btn_pady = 1 self.span = span # number of columns for controls self.ctrl_width = 60 # default control width in chars self.text_height = 20 # default text height # create the frames #self.base_frame = tk.Frame(self.owner) #self.ctl_frame = ScrollableFrame(self.base_frame, height, width).get_frame() self.ctl_frame = ScrollableFrame(self.owner, height, width).get_frame() #self.ctl_frame = ScrollableFrame(self.base_frame).get_frame() #self.ctl_frame = tk.Frame(self.base_frame) #self.btn_frame = tk.Frame(self.base_frame) self.btn_frame = tk.Frame(self.owner) self.btn_frame.grid(row=0, column=0, sticky='es') self.ctl_frame.grid(row=0, column=1, sticky='wn') #self.base_frame.grid(row=0, column=0) self.logger.debug("leave constructor") @debugger def add_title(self, name): ''' Add the form title. This is not a tracked item. ''' lab = tk.Label(self.ctl_frame, text=name, font=("Helvetica", 14)) lab.grid(row=0, column=0, padx=self.padx, pady=self.pady, columnspan=self.span, sticky=(tk.E, tk.W)) self.row += 1 @debugger def add_entry(self, name, column, _type, inc_row=True, readonly=False, span=None, **kargs): ''' Add an entry line to the form. The name is displayed as the label and is also used to access the control when reading it or writing to it. Kargs are passed directly to the widget constructor. name = The name to use to access the control object. column = Name of the database column. _type = The type of data stored in the database. inc_row = whether to increment the row after adding this line. ''' if not 'width' in kargs: kargs['width'] = self.ctrl_width self.logger.debug("kargs = %s" % (str(kargs))) lab = tk.Label(self.ctl_frame, text=name + ':') strvar = tk.StringVar(self.ctl_frame) item = tk.Entry(self.ctl_frame, textvariable=strvar, **kargs) if readonly: item.configure(state='readonly') lab.grid(row=self.row, column=self.col, padx=self.padx, pady=self.pady, sticky=tk.E) self.col += 1 if span is None: span = self.span - 1 item.grid(row=self.row, column=self.col, padx=self.padx, pady=self.pady, columnspan=span, sticky=tk.W) if inc_row: self.row += 1 self.col = 0 else: self.col += 1 def getter(): return _type(strvar.get()) def setter(s): state = item.configure()['state'] if state == 'readonly': item.configure(state='normal') strvar.set(str(s)) if state == 'readonly': item.configure(state='readonly') def clear(): state = item.configure()['state'] if state == 'readonly': item.configure(state='normal') strvar.set('') if state == 'readonly': item.configure(state='readonly') self.controls[name] = { 'column': column, 'obj': item, 'get': getter, 'set': lambda s: setter(s), 'clear': clear, 'type': _type, 'kind': Form.ENTRY } @debugger def add_text(self, name, column, inc_row=True, **kargs): ''' Add a multi-line text entry box with scroll bars to the form. The name is displayed as the label and is also used to access the control when reading it or writing to it. Kargs are passed directly to the widget constructor. name = The name of the widgit and the label text column = the column in the database to retrieve inc_row = whether to increment the row after adding the widget ''' lab = tk.Label(self.ctl_frame, text=name + ':') if not 'width' in kargs: kargs['width'] = self.ctrl_width if not 'height' in kargs: kargs['height'] = self.text_height self.logger.debug("kargs = %s" % (str(kargs))) frame = tk.Frame(self.ctl_frame, bd=1, relief=tk.RIDGE) text = tk.Text(frame, wrap=tk.NONE, **kargs) text.insert(tk.END, '') # see https://www.homeandlearn.uk/tkinter-scrollbars.html self.vsb = tk.Scrollbar(frame, orient=tk.VERTICAL) self.vsb.config(command=text.yview) text.config(yscrollcommand=self.vsb.set) self.vsb.pack(side=tk.RIGHT, fill=tk.Y) self.hsb = tk.Scrollbar(frame, orient=tk.HORIZONTAL) self.hsb.config(command=text.xview) text.config(xscrollcommand=self.hsb.set) self.hsb.pack(side=tk.BOTTOM, fill=tk.X) text.pack(side=tk.LEFT) lab.grid(row=self.row, column=self.col, padx=self.padx, pady=self.pady, sticky=tk.E) self.col += 1 frame.grid(row=self.row, column=self.col, padx=self.padx, pady=self.pady, columnspan=self.span - 1, sticky=tk.W) if inc_row: self.row += 1 self.col = 0 else: self.col += 1 def getter(): return text.get(1.0, tk.END) def setter(s): text.delete('1.0', tk.END) if not s is None: text.insert(tk.END, str(s)) def clear(): text.delete('1.0', tk.END) self.controls[name] = { 'column': column, 'obj': text, 'get': getter, 'set': lambda s: setter(s), 'clear': clear, 'kind': Form.TEXT } @debugger def add_combo(self, name, table, column, inc_row=True, **kargs): ''' Add a combo box line to the form. The name is displayed as the label and is also used to access the control when reading it or writing to it. Combo boxes use a different table to store their data in. This widgit automatically retrieves that data when it is displayed. The type is always strings. Kargs are passed directly to the widget constructor. name = the name of the row and the label text table = the name of the table to retrieve the data for the combo box column = the name of the column to retrieve the value to set the box to inc_row = whether to increment the row after adding the widget ''' lab = tk.Label(self.ctl_frame, text=name + ':') if not 'width' in kargs: kargs['width'] = self.ctrl_width # Note that the method that displays the frame has to populate this. combo = ttk.Combobox(self.ctl_frame, state='readonly', **kargs) lab.grid(row=self.row, column=self.col, padx=self.padx, pady=self.pady, sticky=tk.E) self.col += 1 combo.grid(row=self.row, column=self.col, padx=self.padx, pady=self.pady, sticky=tk.W) if inc_row: self.row += 1 self.col = 0 else: self.col += 1 def getter(): return combo.current() + 1 def setter(s): combo.current(int(s) - 1) def clear(): try: combo.current(0) except tk.TclError: pass # empty content is not an error def populate(): combo['values'] = self.data.populate_list(table, 'name') self.controls[name] = { 'column': column, 'table': table, 'obj': combo, 'get': getter, 'set': lambda s: setter(s), 'clear': clear, 'populate': populate, 'kind': Form.COMBO } @debugger def add_dynamic_label(self, name, column, inc_row=True, **kargs): ''' Add a label that can be changed according to a database column. ''' lab = tk.Label(self.ctl_frame, text=name + ':') value = tk.StringVar(self.ctl_frame) val = tk.Label(self.ctl_frame, textvariable=value, **kargs) lab.grid(row=self.row, column=self.col, padx=self.padx, pady=self.pady, sticky=tk.E) self.col += 1 val.grid(row=self.row, column=self.col, padx=self.padx, pady=self.pady, sticky=tk.W) if inc_row: self.row += 1 self.col = 0 else: self.col += 1 def getter(): return value.get() def setter(s): value.set(str(s)) def clear(): value.set('') self.controls[name] = { 'column': column, 'obj': lab, 'get': getter, 'set': lambda s: setter(s), 'clear': clear, 'kind': Form.LABEL } @debugger def add_notebook(self, nb_class): ''' Add a notebook to the form. ''' #nb_class(self.base_frame) nb_class(self.owner) @debugger def add_button(self, name, command=None, **kargs): ''' Add a button to the button line on the form. The name is used as the label on the button. If a callback is not provided, then a default callback will be run. Buttons are added in the order they are created and arranged accroding default patterns. Args and kargs are passed directly to the widget constructor. ''' if command is None: if name == "Next": command = self.next_callback elif name == "Prev": command = self.prev_callback elif name == "Select": command = self.select_callback elif name == "New": command = self.new_callback elif name == "Save": command = self.save_callback elif name == "Delete": command = self.delete_callback else: raise "unknown name and no command to exec" btn = tk.Button(self.btn_frame, text=name, command=command, width=self.btn_width, **kargs) #btn.pack(padx=self.btn_padx, pady=self.btn_pady, side=tk.TOP) btn.grid(row=self.btn_row, column=0, padx=self.btn_padx, pady=self.btn_pady) self.btn_row += 1 @debugger def get_form(self): ''' Return a dictionary of all of the values that are currently in the form. Form fields are returned in the type that will be stored in the database. ''' values = {} for item in self.controls: values[item] = self.get_form_field(item) return values @debugger def set_form(self, values): ''' Accept a dictionary of all of the values for the form and make them visible. If a form field is different from the type that will be stored in the database, then the value will be changed to the correct type. ''' for item in values: self.set_form_field(item, values[item]) @debugger def clear_form(self): ''' Clear the form with blank/default values. ''' for item in self.controls: self.clear_form_field(item) @debugger def get_form_field(self, name): ''' Return the contents of a form field in the type that the form stores. If the database has stored an int, then an int is returned, rather than a string. If the field is a combo box, then setting the values is expected to be an array of text to populate the control. ''' return self.controls[name]['get']() @debugger def set_form_field(self, name, value): ''' Set the value of the field according to the type that the field stores. The type will be automatically converted from a string, if the type to be stored in the is different. ''' self.controls[name]['set'](value) @debugger def clear_form_field(self, name): ''' Clear the form field to the default value. ''' self.controls[name]['clear']() @debugger def commit_form(self): ''' Write the contents of the form to the database. The form does not need to be visible for this to take place. Nothing is written to the database until this method is called. This assumes that all of the data is in the same table. ''' if not self.row_list is None: vals = {} row_id = self.row_list[self.row_index] for item in self.controls: cval = self.controls[item]['get']() col = self.controls[item]['column'] if not cval is None and col != '': vals[col] = cval if self.data.if_rec_exists(self.table, 'ID', row_id): self.data.update_row(self.table, vals, "ID=%d" % (row_id)) else: self.data.insert_row(self.table, vals) self.data.commit() self.row_list = self.data.get_id_list(self.table) else: self.logger.debug("Form has no table") @debugger def load_form(self): ''' Read the contents of the form from the database and place the values in the form controls. If the form is currently visible, then display the values. ''' if not self.row_list is None: try: row_id = self.row_list[self.row_index] row = self.data.get_row_by_id(self.table, row_id) for item in self.controls: if self.controls[item]['kind'] == Form.COMBO: self.controls[item]['populate']() self.controls[item]['set']( row[self.controls[item]['column']]) elif self.controls[item]['kind'] == Form.PRODUCT: self.controls[item]['set'](row_id) elif self.controls[item]['kind'] == Form.DIR_BROWSER: self.controls[item]['set']() elif self.controls[item]['kind'] == Form.IMPORT_BTN: self.controls[item]['set']() else: self.controls[item]['set']( row[self.controls[item]['column']]) except IndexError as e: showerror( 'No Records', 'No records exist for this form.\n\nThere are %d records in the table.' % (len(self.row_list))) @debugger def configure_obj(self, name, **kargs): ''' Call "configure" on an underlying object with the parameters indicated. ''' self.controls[name]['obj'].configure(**kargs) @debugger def prev_callback(self): ''' Default callback for the Prev button. ''' if not self.row_list is None: self.row_index -= 1 if self.row_index < 0: self.row_index = 0 showinfo('First Record', 'This is the first record.') else: self.load_form() @debugger def next_callback(self): ''' Default callback for the next button. ''' if not self.row_list is None: self.row_index += 1 if self.row_index > len(self.row_list) - 1: self.row_index = len(self.row_list) - 1 showinfo('Last Record', 'This is the last record.') else: self.load_form() @debugger def select_callback(self): ''' Default callback for the select button. This displays a dialog of the "Name" field of all of the rows that are defined in the table. If the name field does not exist, throw an exception. ''' if not self.row_list is None: item = SelectItem(self.owner, self.table, 'name') self.row_index = self.row_list.index(item.item_id) self.load_form() @debugger def new_callback(self): ''' Default callback for the "New" button. This clears the form to default values. ''' for item in self.controls: self.controls[item]['clear']() @debugger def save_callback(self): ''' Default callback for the "Save" button. This writes the for to the database. ''' if askyesno('Save record?', 'Are you sure you want to save this?'): self.commit_form() @debugger def delete_callback(self): ''' Default callback for the delete button. This deletes the current row from the database and displays the next one. If deleting the last row, then the first row is displayed. ''' if askyesno('Delete record?', 'Are you sure you want to delete this?'): self.data.delete_row(self.table, self.row_list[self.row_index]) self.data.commit() @debugger def set_layout_row(self, num): ''' Set the layout row to the given number. ''' self.row = num @debugger def set_layout_col(self, num): ''' Set the layout column to the given number. ''' self.col = num @debugger def get_layout_row(self): ''' Returnthe layout row. ''' return self.row @debugger def get_layout_col(self): ''' Return the layout column. ''' return self.col
class ScrollableFrame: ''' This is the scrollable frame class that is used by the notebook class. It could be used generically, but it was designed for use here. The scrolling functionality supports both a scroll bar as well as the mouse wheel. ''' def __init__(self, container, height=800, width=1000, *args, **kwargs): self.logger = Logger(self, level=Logger.INFO) self.logger.debug("enter constructor") self.canvas = tk.Canvas(container, height=height, width=width) self.canvas.grid(row=0, column=0, sticky='news') self.scrollbar = tk.Scrollbar(container, orient="vertical", command=self.canvas.yview) self.scrollbar.grid(row=0, column=1, sticky='nsw') self.canvas.configure(yscrollcommand=self.scrollbar.set) self.canvas.configure(yscrollincrement='20') self.canvas.configure(scrollregion=self.canvas.bbox("all")) self.scrollwindow = tk.Frame(self.canvas) self.canvas.create_window((0, 0), window=self.scrollwindow, anchor="nw") self.scrollwindow.bind("<Configure>", self.configure_window) self.scrollwindow.bind("<Enter>", self.enter_handler) self.scrollwindow.bind("<Leave>", self.leave_handler) self.scrollwindow.bind('<Button-4>', self.mouse_wheel) self.scrollwindow.bind('<Button-5>', self.mouse_wheel) self.canvas.focus_set() self.logger.debug("leave constructor") def get_frame(self): ''' Get the master frame for embedded widgets. ''' #print('here') return self.scrollwindow # PORTABILITY! This may not be the same on every platform. It works under # linux using xfce. Other must be tested. The problem that this solves is # that mousewheel events are only routed to the widget that the mouse is # pointing to. If the mouse is hovering over a label or something, the # canvas never gets the event. Surprisingly, the enter and leave events # are both sent when the mousewheel is rolled. I am sure that this is an # "unsupported" feature. If the mousewheel stops working, this is where # to look for a fix. def enter_handler(self, event): #print('enter', event.state) state = (event.state & 0x1800) >> 11 direction = 0 if state == 2: direction = 1 if state == 1: direction = -1 #print(direction) self.canvas.yview_scroll(direction, tk.UNITS) def leave_handler(self, event): #print('leave', event.state) pass @debugger def configure_window(self, event): #print('here', self.mark) self.canvas.configure(scrollregion=self.canvas.bbox("all")) def mouse_wheel(self, event): direction = 0 if event.num == 5: direction = 1 if event.num == 4: direction = -1 #print(direction) self.canvas.yview_scroll(direction, tk.UNITS)
def __init__(self, notebook, index, table=None, height=700, width=1000, span=4): ''' Create a form object. A form must be associated with exactly one database table. The form is filled in with exactly one row from the database table. If the form does not use all of the row, that is handled transparently. This class contains frames. It is not, itself, a frame. notebook = The notebook container index = The index of the tab to bind the form to table = The database table where the data from the form is located height = The height of the form in pixels width = the width of the form in pixels span = The number of columns in the form ''' self.logger = Logger(self, level=Logger.DEBUG) self.logger.debug("enter constructor") self.notebook = notebook # container for the form. self.nb_index = index # index of the tab to bind the form to self.table = table # database table to save and retrieve data # frame to bind the widgets to self.owner = self.notebook.get_frame(self.nb_index) # load_form should be over ridden to handle special cases. self.notebook.set_callback(self.nb_index, self.load_form) # database singleton object self.data = Database.get_instance() # the rows may not be sequentially numbered. if not table is None: self.row_list = self.data.get_id_list(self.table) self.row_index = 0 else: self.row_list = None self.row_index = 0 # dict of dicts that lists all of the controls by name. self.controls = {} # actual height and width of the form frame. self.height = height self.width = width self.logger.debug("window size = %d, %d" % (self.height, self.width)) # keep track of the current layout position. self.row = 0 self.col = 0 self.btn_row = 0 self.btn_width = 10 self.padx = 5 self.pady = 5 self.btn_padx = 1 self.btn_pady = 1 self.span = span # number of columns for controls self.ctrl_width = 60 # default control width in chars self.text_height = 20 # default text height # create the frames #self.base_frame = tk.Frame(self.owner) #self.ctl_frame = ScrollableFrame(self.base_frame, height, width).get_frame() self.ctl_frame = ScrollableFrame(self.owner, height, width).get_frame() #self.ctl_frame = ScrollableFrame(self.base_frame).get_frame() #self.ctl_frame = tk.Frame(self.base_frame) #self.btn_frame = tk.Frame(self.base_frame) self.btn_frame = tk.Frame(self.owner) self.btn_frame.grid(row=0, column=0, sticky='es') self.ctl_frame.grid(row=0, column=1, sticky='wn') #self.base_frame.grid(row=0, column=0) self.logger.debug("leave constructor")
def main(argv): workspace = '' # datasets = { # 'pecanstreet': ['California', 'Austin', 'New York'], # 'eGauge':['Colorado'] # } datasets = { 'pecanstreet': ['California'], 'eGauge':['Colorado'] } data_years = { 'California': '2015', # 2015 full year for PecanStreet California dataset UTC 'Austin': '2018', # 2018 full year for PecanStreet Austin dataset UTC 'New York': '2019', # 2019/5/1-10/31 half year for PecanStreet New York dataset UTC 'Colorado': '2015' # 2015 full year for eGauge dataset } discharge_speed = '100' for dataset in datasets: for location in datasets[dataset]: print(f'Start simulate {location} by LRF...') input_path = workspace + 'data/' + dataset + '/' + location + '/' + data_years[location] + '.csv' output_path = workspace + 'data/' + dataset + '/' + location + '/logs/LRF.csv' # init csv file header output_csv_header = ['timestamp', 'datetime', 'from', 'to', 'amount', 'type'] # type: share, grid, own with open(output_path, 'w', newline='') as csvfile: writer = csv.DictWriter(csvfile, fieldnames=output_csv_header) writer.writeheader() csvfile.close() lenders = [] borrowers = [] discharge_rate = int(discharge_speed) df = pd.read_csv(input_path) total = len(df) counter = 0 current_ts = 0 # house_info_path = workspace + 'data/metadata.csv' house_info_path =workspace + 'data/' + dataset + '/' + location + '/metadata.csv' discharge_rates = {} # Init charge rate list with open(house_info_path) as house_info_csv_file: reader = csv.DictReader(house_info_csv_file) for row in reader: discharge_rates[row['house_id']] = discharge_rate house_info_csv_file.close() with open(output_path, 'a', newline='') as output_csv_file: writer = csv.writer(output_csv_file) with open(input_path) as input_csv_file: reader = csv.DictReader(input_csv_file) for row in reader: # Skip empty row diff = float(row['diff']) if diff == 0: continue # Get timestamp ts = int(float(row['timestamp'])) # Init current timestamp at beginning if counter == 0: current_ts = ts # Increase counter counter += 1 # process = str(counter) + '/' + str(total) + ' (' + str(round(counter/total*100, 2)) + '%)' # print(process, end='\r') if ts != current_ts: for bidx, borrower in enumerate(borrowers): if len(lenders) <= 0: log.print('No lender is available.') break # 1st. Check if can use own battery power first for idx, lender in enumerate(lenders): if lender['house_id'] == borrower['house_id']: if discharge_rates[lender['house_id']] <= 0: continue # Borrow amount greater than own discharge rate if borrower['diff'] >= discharge_rates[lender['house_id']]: # Power provided by own battery is greater than discharge rate, then use discharge rate amount, keep rest for sharing if lender['diff'] > discharge_rates[lender['house_id']]: log.print('Use own: b>=d, l>d') borrower['diff'] -= discharge_rates[lender['house_id']] borrowers[bidx] = borrower lender['diff'] -= discharge_rates[lender['house_id']] # lenders[idx] = lender lenders.remove(lender) lenders_len = len(lenders) if lenders_len < 1: lenders.append(lender) else: for i, l in enumerate(lenders): if l['diff'] < lender['diff']: lenders.insert(i, lender) break else: if i == (lenders_len - 1): lenders.insert(i+1, lender) break writer.writerow([current_ts, borrower['datetime'], lender['house_id'], borrower['house_id'], discharge_rates[lender['house_id']], 'own']) discharge_rates[lender['house_id']] = 0 break # Own battery cannot provide power greater than discharge rate, use up all and withdraw sharing else: log.print('Use own: b>=d, l=<d') borrower['diff'] -= lender['diff'] borrowers[bidx] = borrower lenders.remove(lender) discharge_rates[lender['house_id']] -= lender['diff'] writer.writerow([current_ts, borrower['datetime'], lender['house_id'], borrower['house_id'], lender['diff'], 'own']) # Borrow amount less than own discharge rate else: if borrower['diff'] >= lender['diff']: log.print('own: b<d, b>=l') borrower['diff'] -= lender['diff'] borrowers[bidx] = borrower lenders.remove(lender) discharge_rates[lender['house_id']] -= lender['diff'] writer.writerow([current_ts, borrower['datetime'], lender['house_id'], borrower['house_id'], lender['diff'], 'own']) else: log.print('own: b<d, b<l') lender['diff'] -= borrower['diff'] # lenders[idx] = lender lenders.remove(lender) lenders_len = len(lenders) if lenders_len < 1: lenders.append(lender) else: for i, l in enumerate(lenders): if l['diff'] < lender['diff']: lenders.insert(i, lender) break else: if i == (lenders_len - 1): lenders.insert(i+1, lender) break discharge_rates[lender['house_id']] -= borrower['diff'] writer.writerow([current_ts, borrower['datetime'], lender['house_id'], borrower['house_id'], borrower['diff'], 'own']) borrower['diff'] = 0 borrowers[bidx] = borrower if borrower['diff'] == 0: break elif borrower['diff'] < 0: log.print('Error: borrowing amount is negative!') # 2. Borrow from other lenders # if len(lenders) < 1: # log('No lender is available.') # break lender_index = 0 while borrower['diff'] > 0: # log(borrower['diff']) # log(str(lender_index) + '/' + str(len(lenders))) if lender_index < len(lenders): lender = lenders[lender_index] if discharge_rates[lender['house_id']] <= 0: lender_index += 1 continue if lender['house_id'] == borrower['house_id']: lender_index += 1 continue # lend_amount = abs(float(lenders[lender_index]['diff'])) # log(lender['diff']) # Borrow amount greater than own discharge rate if borrower['diff'] >= discharge_rates[lender['house_id']]: # Power provided by lender's battery is greater than discharge rate, then use discharge rate amount, keep rest for sharing if lender['diff'] > discharge_rates[lender['house_id']]: log.print('Share: b>=d, l>d') borrower['diff'] -= discharge_rates[lender['house_id']] borrowers[bidx] = borrower lender['diff'] -= discharge_rates[lender['house_id']] # lenders[lender_index] = lender lenders.remove(lender) lenders_len = len(lenders) if lenders_len < 1: lenders.append(lender) else: for i, l in enumerate(lenders): if l['diff'] < lender['diff']: lenders.insert(i, lender) break else: if i == (lenders_len - 1): lenders.insert(i+1, lender) break writer.writerow([current_ts, borrower['datetime'], lender['house_id'], borrower['house_id'], discharge_rates[lender['house_id']], 'share']) discharge_rates[lender['house_id']] = 0 # Own battery cannot provide power greater than discharge rate, use up all and withdraw sharing else: log.print('Share: b>=d, l=<d') borrower['diff'] -= lender['diff'] borrowers[bidx] = borrower lenders.remove(lender) discharge_rates[lender['house_id']] -= lender['diff'] writer.writerow([current_ts, borrower['datetime'], lender['house_id'], borrower['house_id'], lender['diff'], 'share']) # Borrow amount less than lender's discharge rate else: if borrower['diff'] >= lender['diff']: log.print('Share: b<d, b>=l') borrower['diff'] -= lender['diff'] borrowers[bidx] = borrower lenders.remove(lender) discharge_rates[lender['house_id']] -= lender['diff'] writer.writerow([current_ts, borrower['datetime'], lender['house_id'], borrower['house_id'], lender['diff'], 'share']) else: log.print('Share: b<d, b<l') lender['diff'] -= borrower['diff'] # lenders[lender_index] = lender lenders.remove(lender) lenders_len = len(lenders) if lenders_len < 1: lenders.append(lender) else: for i, l in enumerate(lenders): if l['diff'] < lender['diff']: lenders.insert(i, lender) break else: if i == (lenders_len - 1): lenders.insert(i+1, lender) break discharge_rates[lender['house_id']] -= borrower['diff'] writer.writerow([current_ts, borrower['datetime'], lender['house_id'], borrower['house_id'], borrower['diff'], 'share']) borrower['diff'] = 0 borrowers[bidx] = borrower lender_index += 1 # No lenders available, get from grid else: log.print('grid') writer.writerow([current_ts, borrower['datetime'], 0, borrower['house_id'], borrower['diff'], 'grid']) borrower['diff'] = 0 break if borrower['diff'] == 0: break elif borrower['diff'] < 0: log.print('Error: borrowing amount is negative!') # Reset dicharge rate list for dr in discharge_rates: discharge_rates[dr] = discharge_rate # Reset borrowers list borrowers = [] # Sum up power left in batteries battery_remain = 0 for l in lenders: battery_remain += l['diff'] if battery_remain > 0: dt = datetime.fromtimestamp(current_ts) writer.writerow([current_ts, dt, '', '', battery_remain, 'battery_remain']) current_ts = ts row['diff'] = abs(float(row['diff'])) if diff < 0: borrowers.append(row) else: lenders_len = len(lenders) if lenders_len < 1: lenders.append(row) else: for i, l in enumerate(lenders): if l['diff'] < row['diff']: lenders.insert(i, row) break else: if i == (lenders_len - 1): lenders.insert(i+1, row) break # log(lenders) # log('-----------------------') # log(str(counter) + ':' + str(ts)) input_csv_file.close() output_csv_file.close()
def __init__(self, notebook): self.logger = Logger(self, Logger.DEBUG) self.logger.debug(sys._getframe().f_code.co_name) super().__init__(notebook, notebook.SETUP_FRAME, 'Business') self.add_notebook(SetupNotebook)
class LineWidgit(tkinter.Frame): ''' This is the GUI widget that represents a single line in the output data. It uses the data_store to communicate values into and out of itself. ''' def __init__(self, parent,lineno): self.logger = Logger(self, Logger.INFO) self.logger.debug("constructor") tkinter.Frame.__init__(self, parent) self.data_store = DataStore.get_instance() self.index = lineno self.name = "Hole %d" % (lineno+1) self.line_name = tkinter.Label(self, text=self.name, width=12) self.line_name.grid(row=lineno+1, column=0, sticky=tkinter.W) self.inter_ctl = tkinter.Entry(self, width=5, validate="focusout", validatecommand=self.change_interval) self.inter_ctl.bind('<Return>', self.change_interval) self.inter_ctl.bind('<Tab>', self.change_interval) self.inter_ctl.grid(row=lineno+1, column=1) self.note_ctl_txt = tkinter.StringVar() self.note_ctl = tkinter.Label(self, textvariable=self.note_ctl_txt, width=12) self.note_ctl.grid(row=lineno+1, column=2) self.freq_ctl_txt = tkinter.StringVar() self.freq_ctl = tkinter.Label(self, textvariable=self.freq_ctl_txt, width=12) self.freq_ctl.grid(row=lineno+1, column=3) self.hole_ctl = HoleSizeWidgit(self, lineno) self.hole_ctl.config(padx=25) self.hole_ctl.grid(row=lineno+1, column=4) self.locat_ctl_txt = tkinter.StringVar() self.locat_ctl = tkinter.Label(self, textvariable=self.locat_ctl_txt, width=12) self.locat_ctl.grid(row=lineno+1, column=5) self.diff_ctl_txt = tkinter.StringVar() self.diff_ctl = tkinter.Label(self, textvariable=self.diff_ctl_txt, width=12) self.diff_ctl.grid(row=lineno+1, column=6) self.cutoff_ctl_txt = tkinter.StringVar() self.cutoff_ctl = tkinter.Label(self, textvariable=self.cutoff_ctl_txt, width=12) self.cutoff_ctl.grid(row=lineno+1, column=7) self.set_state() self.logger.debug("end constructor") @debugger def set_state(self): ''' Place the data from the data_store into the GUI. ''' self.inter_ctl.delete(0, tkinter.END) self.inter_ctl.insert(0, str(self.data_store.get_hole_interval(self.index))) self.note_ctl_txt.set(str(self.data_store.get_hole_note(self.index))) # Label self.freq_ctl_txt.set("%s Hz"%(str(self.data_store.get_hole_freq(self.index)))) self.locat_ctl_txt.set("%0.4f"%self.data_store.get_hole_xloc(self.index)) # Label self.diff_ctl_txt.set("%0.4f"%self.data_store.get_hole_diff(self.index)) # Label self.cutoff_ctl_txt.set("%0.4f"%self.data_store.get_hole_cutoff(self.index)) # Label self.hole_ctl.set_state() @debugger def get_state(self): ''' Get the data out of the display and place it in the data_store. ''' self.data_store.set_hole_interval(self.index, int(self.inter_ctl.get())) self.data_store.set_hole_note(self.index, self.note_ctl_txt.get()) # str self.data_store.set_hole_freq(self.index, float(self.freq_ctl_txt.get().split()[0])) self.data_store.set_hole_location(self.index, float(self.locat_ctl_txt.get())) self.data_store.set_hole_diff(self.index, float(self.diff_ctl_txt.get())) self.data_store.set_hole_cutoff(self.index, float(self.cutoff_ctl_txt.get())) self.hole_ctl.get_state() @debugger def print_state(self): self.logger.msg(str(self.get_state())) @debugger def change_units(self): ''' When this is called, it is assumed that the datastore and the GUI need to have the vaules updated to reflect the new units. ''' if self.data_store.get_units(): self.data_store.set_hole_size(self.index, utility.in_to_mm(self.data_store.get_hole_size(self.index))) self.data_store.set_hole_location(self.index, utility.in_to_mm(self.data_store.get_hole_location(self.index))) self.data_store.set_hole_diff(self.index, utility.in_to_mm(self.data_store.get_hole_diff(self.index))) self.data_store.set_hole_cutoff(self.index, utility.in_to_mm(self.data_store.get_hole_cutoff(self.index))) else: self.data_store.set_hole_size(self.index, utility.mm_to_in(self.data_store.get_hole_size(self.index))) self.data_store.set_hole_location(self.index, utility.mm_to_in(self.data_store.get_hole_location(self.index))) self.data_store.set_hole_diff(self.index, utility.mm_to_in(self.data_store.get_hole_diff(self.index))) self.data_store.set_hole_cutoff(self.index, utility.mm_to_in(self.data_store.get_hole_cutoff(self.index))) self.set_state() self.data_store.set_change_flag() @debugger def change_interval(self, event=None): try: val = int(self.inter_ctl.get()) oldval = self.data_store.get_hole_interval(self.index) if val != oldval: if val > 0 and val < 5: self.data_store.set_hole_interval(self.index, val) self.logger.debug("change interval from %d to %d"%(oldval, val)) raise_event("UPDATE_NOTES_EVENT") self.data_store.set_change_flag() else: self.logger.error("invalid value for interval: %s"%(str(self.inter_ctl.get()))) messagebox.showerror("ERROR", "Intervals must be an integer between 1 and 4") self.inter_ctl.delete(0, tkinter.END) self.inter_ctl.insert(0, str(self.data_store.get_hole_interval(self.index))) return False else: self.logger.debug("ignore") except ValueError: self.logger.error("invalid integer for interval: %s"%(str(self.inter_ctl.get()))) messagebox.showerror("ERROR", "Cannot convert the string \"%s\" to an integer between 1 1nd 4"%(self.inter_ctl.get())) self.inter_ctl.delete(0, tkinter.END) self.inter_ctl.insert(0, str(self.data_store.get_hole_interval(self.index))) return False except IndexError: pass # always ignore return True
class DependencyScriptConfiguration(object): @staticmethod def GetConfigurationFilename(scriptFilename): path, ext = os.path.splitext(argv[0]) path, name = os.path.split(path) return '{0}.ini'.format(name) def __init__(self, argv=None, argparser=None): # Make an argparser if we were not given one already. if argparser is None: self.argparser = argparse.ArgumentParser(description='Generates an SQLite3 database containing all of the #include dependencies in your project.') else: self.argparser = argparser # Add our arguments to the argparser. self.argparser.add_argument('-C', '--print-example-config', action='store_true', default=False, dest='printExampleConfig', help='The script requires a configuration file in the current directory or the script directory in order to run. This option generates an example config file that lists all the available options. Note: the databases must have the same filename in the config file. You will also want to make sure that the default filename, :memory:, is no longer set. If it was then you won\'t have a file to work with. Read the comments in the example config for details.') self.argparser.add_argument('--debug', action='store_true', default=False, dest='debugMessages', help='Print the debug messages. Very verbose, use only in development.') self.argparser.add_argument('--verbose', action='store_true', default=False, dest='verbose', help='Print the info messages about progress and status.') self.argparser.add_argument('--silence-errors', action='store_true', default=False, dest='silenceErrors', help='Don\'t print any errors.') self.argparser.add_argument('-c', '--config-filename', dest='scriptIni', metavar='<config-filename>', help='Specifies the path to the configuration file to use.') self.argparser.add_argument('-f', '--database-filename', dest='databaseFilename', metavar='<db-filename>', help='Specifies the filename of the database. Note: specifying this option overrides the filename in the configuration file.') self.argparser.add_argument('-s', '--source-path', dest='sourcePath', metavar='<source-path>', help='Specifies the path to the root of the source code.') # Configure the rest of our class. We need to initialize unused variables if we want to use # them later in our class. if argv is not None: self.Configure(argv) else: self.ClearConfiguration() def Configure(self, argv): self.args = argv self.messagePrinter = Logger() self.scriptPath, self.scriptExtension = os.path.splitext(argv[0]) self.scriptPath, self.scriptName = os.path.split(self.scriptPath) self.argparser.parse_args(args=argv[1:], namespace=self) self.messagePrinter.isDbgEnabled = self.debugMessages self.messagePrinter.isInfoEnabled = self.verbose self.messagePrinter.isErrEnabled = not self.silenceErrors if self.scriptIni is None: iniPath = '' if self.scriptPath is not None: iniPath = self.scriptPath self.scriptIni = toPosixPath(os.path.join(iniPath, '{0}.ini'.format(self.scriptName))) self.parser = configparser.ConfigParser(allow_no_value = True) # see https://docs.python.org/2/library/configparser.html self.parser.optionxform = str # make case-sensitive as per https://docs.python.org/2/library/configparser.html self.isConfigured = True if os.path.exists(self.scriptIni): self.parser.read(self.scriptIni) elif os.path.exists(os.path.join(self.scriptPath, self.scriptIni)): self.parser.read(os.path.join(self.scriptPath, self.scriptIni)) else: msg = "No configuration file found. Searched, current directory and script directory directory for {0}.".format(self.scriptIni) self.messagePrinter.error(msg) self.isConfigured = False self.databaseFilename = ':memory:' self.sourcePath = './' return try: if self.databaseFilename is None: self.databaseFilename = self.parser.get("Output", "DatabaseFilename") except: self.databaseFilename = ':memory:' try: if self.sourcePath is None: self.sourcePath = self.parser.get("Paths","SourceRoot") # Make the read SourceRoot path relative to the INI file's path. if self.isConfigured: iniPath, iniFilename = os.path.split(self.scriptIni) self.sourcePath = toPosixPath(os.path.normpath(os.path.join(iniPath, self.sourcePath))) print('source-path: {0}'.format(self.sourcePath)) except: self.sourcePath = './' def ClearConfiguration(self): self.args = None self.parser = None self.scriptName = None self.scriptPath = None self.scriptExtension = None self.scriptIni = None
def main(svc_input, configs): logger = Logger("查询日志", verbose=True) log_file_name = "log%s_%s.txt" % (svc_input.replace( "?", "#"), DateTimeUtil.get_current_datetime(is_date=True)) log_file_path = WindowsUtil.convert_win_path( os.path.join(temp_dir, log_file_name)) logger.info("[开始查询] %s" % svc_input) try: # 找到本地匹配的保修历史记录 history_zip = ZipFileSVC(zip_file_path=history_zipfile, mode='a') start_time = DateTimeUtil.get_current_datetime() # 创建出所有可能查询码 svc_generator = SVCGenerator(svc_input, logger) logger.info("创建出所有可能查询码:%s" % len(svc_generator.target_svc_set)) # 根据本地匹配的非法查询码历史,筛选出目标查询码,以及非法查询码 existed_svc = history_zip.find_file_regex(svc_generator.regex) svc_generator.generate_target_svc_batch(existed_svc, invalid_history_file_path) # 调用戴尔查询API,并将API数据转化为实体类数据 output_dell_asset_list = list([]) if svc_generator.target_svc_set: batch = Batch(logger, configs) api_dell_asset_list = batch.begin(svc_generator.target_svc_set) output_dell_asset_list = api_dell_asset_list logger.info("从API中总共得到%s个结果" % (len(api_dell_asset_list))) logger.info("将实体类序列化到本地临时TXT文件") temp_text_files_path = DellAsset.serialize_txt_batch( api_dell_asset_list, temp_dir) logger.info("将序列化临时文件存到本地zip历史记录,总数:%s" % len(temp_text_files_path)) history_zip.add_new_file_batch(temp_text_files_path) logger.info("删除临时 %s 个TXT文件" % len(temp_text_files_path)) for file_path in temp_text_files_path: FileUtil.delete_file(file_path) logger.info("将API得到的实体类和历史记录实体类合并") else: logger.warn("目标查询码为空,仅从从历史记录中导出结果") for svc in svc_generator.existed_svc_set: dell_asset_content = history_zip.get_member_content( file_name="%s.txt" % svc) output_dell_asset_list.append( DellAsset.deserialize_txt(dell_asset_content)) logger.info("添加历史记录,总共得到%s个结果" % (len(output_dell_asset_list))) excel_output_path = WindowsUtil.convert_win_path( os.path.join(excel_dir, "%s.xlsx" % svc_generator.get_file_name())) DellAsset.save_as_excel_batch(output_dell_asset_list, excel_output_path) if FileUtil.is_path_existed(excel_output_path): logger.info("存为Excel文档成功") end_time = DateTimeUtil.get_current_datetime() logger.info("总用时 %s " % DateTimeUtil.datetime_diff(start_time, end_time)) logger.info("[查询结束] 总共%s个结果 保存在:%s" % (len(output_dell_asset_list), excel_output_path)) else: logger.error("[保存结果失败] %s" % excel_output_path) except Exception as e: # 若程序出现错误失败,发送邮件 logger.error("[查询失败] 已发送报告 请等待解决") logger.error("%s\n%s" % (e, traceback.format_exc())) logger.save(log_file_path) email_api_key = configs["email_api_key"] email = Email( email_api_key, subject="[查询失败] %s %s" % (DateTimeUtil.get_current_datetime(is_date=True), svc_input)) email.add_attachment(log_file_path) email.send(cc_mode=logger.has_error)
class UpperFrame(tkinter.Frame): ''' This class manages the upper frame of the display. ''' def __init__(self, master): self.logger = Logger(self, Logger.INFO) self.logger.debug("constructor") self.master = master self.data_store = DataStore.get_instance() register_event("UPDATE_UPPER_EVENT", self.set_state) @debugger def create_frame(self): # build the screen # Fill in the upper frame tkinter.Label(self.master, text="Title").grid(row=0, column=0, sticky=tkinter.E) self.titleEntry = tkinter.Entry(self.master, width=40, validate="focusout", validatecommand=self.setTitleCommand) self.titleEntry.bind('<Return>', self.setTitleCommand) self.titleEntry.bind('<Tab>', self.setTitleCommand) self.titleEntry.grid(row=0, column=1, columnspan=3, padx=9, pady=4) tkinter.Label(self.master, text="Inside Diameter").grid(row=1, column=0, sticky=tkinter.E, pady=4) self.insideDiaEntry = tkinter.Entry(self.master, validate="focusout", validatecommand=self.insideDiaCommand) self.insideDiaEntry.bind('<Return>', self.insideDiaCommand) self.insideDiaEntry.bind('<Tab>', self.insideDiaCommand) self.insideDiaEntry.grid(row=1, column=1, pady=4) tkinter.Label(self.master, text="Wall Thickness").grid(row=1, column=2, sticky=tkinter.E, pady=4) self.wallThicknessEntry = tkinter.Entry(self.master, validate="focusout", validatecommand=self.wallThicknessCommand) self.wallThicknessEntry.bind('<Return>', self.wallThicknessCommand) self.wallThicknessEntry.bind('<Tab>', self.wallThicknessCommand) self.wallThicknessEntry.grid(row=1, column=3, pady=4) tkinter.Label(self.master, text="Number of Holes").grid(row=2, column=0, sticky=tkinter.E, pady=4) self.numHolesEntry = tkinter.Entry(self.master, validate="focusout", validatecommand=self.numHolesCommand) self.numHolesEntry.bind('<Return>', self.numHolesCommand) self.numHolesEntry.bind('<Tab>', self.numHolesCommand) self.numHolesEntry.grid(row=2, column=1, pady=4) tkinter.Label(self.master, text="Select Bell Note").grid(row=2, column=2, sticky=tkinter.E, pady=4) self.bellNoteCombo = ttk.Combobox(self.master, state="readonly", values=self.data_store.bellNoteArray) self.bellNoteCombo.config(width=17) self.bellNoteCombo.grid(row=2, column=3, pady=4) self.bellNoteCombo.bind("<<ComboboxSelected>>", self.bellSelectCallback) tkinter.Label(self.master, text="Embouchure Area").grid(row=4, column=0, sticky=tkinter.E, pady=4) self.embouchureAreaEntry = tkinter.Entry(self.master) self.embouchureAreaEntry.grid(row=4, column=1, pady=4) tkinter.Label(self.master, text="Units of Measure").grid(row=3, column=0, sticky=tkinter.E, pady=4) self.measureUnitsOpt = ttk.Combobox(self.master, state="readonly", values=["inch", "mm"]) self.measureUnitsOpt.config(width=17) self.measureUnitsOpt.grid(row=3, column=1, pady=4) self.measureUnitsOpt.bind("<<ComboboxSelected>>", self.measureUnitsCallback) tkinter.Label(self.master, text="Display Format").grid(row=3, column=2, sticky=tkinter.E, pady=4) self.displayFormatOpt = ttk.Combobox(self.master, state="readonly", values=["decimal", "fraction"]) self.displayFormatOpt.current(1) self.displayFormatOpt.config(width=17) self.displayFormatOpt.grid(row=3, column=3, pady=4) self.displayFormatOpt.bind("<<ComboboxSelected>>", self.displayFormatCallback) tkinter.Label(self.master, text="Length").grid(row=4, column=2, sticky=tkinter.E, pady=4) self.lengthEntry = tkinter.Entry(self.master) self.lengthEntry.grid(row=4, column=3, pady=4) if self.measureUnitsOpt.get() == 'mm': self.displayFormatOpt.config(state="readonly") self.refreshButton = tkinter.Button( self.master, text="Refresh", width=14, command=self.refreshButtonCommand) self.refreshButton.grid(row=5, column=0, columnspan=4, pady=4) self.set_state() # write what's in the data_store to the GUI @debugger def get_state(self): ''' Return the state of the controls in the upper half into the data store. ''' if self.displayFormatOpt.current() == 0: self.data_store.set_disp_frac(False) else: self.data_store.set_disp_frac(True) if self.measureUnitsOpt.current() == 0: self.data_store.set_units(False) else: self.data_store.set_units(True) self.data_store.set_title(self.titleEntry.get()) self.data_store.set_inside_dia(float(self.insideDiaEntry.get())) self.data_store.set_wall_thickness(float(self.wallThicknessEntry.get())) self.data_store.set_number_holes(int(self.numHolesEntry.get())) self.data_store.set_bell_note_select(self.bellNoteCombo.current()) #self.data_store.set_embouchure_area(float(self.embouchureAreaEntry.get())) self.data_store.set_bell_freq( self.data_store.note_table[self.data_store.get_bell_note_select()]['frequency']) @debugger def set_state(self): ''' Take the state from the data store and put in the GUI. ''' self.titleEntry.delete(0, tkinter.END) self.titleEntry.insert(0, self.data_store.get_title()) self.bellNoteCombo.current(self.data_store.get_bell_note_select()) self.measureUnitsOpt.current(int(self.data_store.get_units())) # it's a bool in the data_store self.displayFormatOpt.current(int(self.data_store.get_disp_frac())) # it's a bool in the data_store self.insideDiaEntry.delete(0, tkinter.END) self.insideDiaEntry.insert(0, str(self.data_store.get_inside_dia())) self.wallThicknessEntry.delete(0, tkinter.END) self.wallThicknessEntry.insert(0, str(self.data_store.get_wall_thickness())) self.numHolesEntry.delete(0, tkinter.END) self.numHolesEntry.insert(0, str(self.data_store.get_number_holes())) self.embouchureAreaEntry.config(state=tkinter.NORMAL) self.embouchureAreaEntry.delete(0, tkinter.END) self.embouchureAreaEntry.insert(0, "%0.4f"%(self.data_store.get_embouchure_area())) self.embouchureAreaEntry.config(state="readonly") self.lengthEntry.config(state=tkinter.NORMAL) self.lengthEntry.delete(0, tkinter.END) self.lengthEntry.insert(0, "%0.4f"%(self.data_store.get_length())) self.lengthEntry.config(state="readonly") @debugger def insideDiaCommand(self, event=None): try: v = self.insideDiaEntry.get() n = float(v) if self.data_store.get_inside_dia() != n: self.logger.debug("change wall from %f to %f"%(self.data_store.get_inside_dia(), n)) self.data_store.set_inside_dia(n) self.insideDiaEntry.delete(0, tkinter.END) self.insideDiaEntry.insert(0, str(n)) raise_event("CALCULATE_EVENT") self.data_store.set_change_flag() else: self.logger.debug("ignore") return True except ValueError as e: self.logger.error(str(e)) messagebox.showerror("Error", "Could not convert inside diameter to a floating point number.\nRead value was \"%s\"." % (v)) self.insideDiaEntry.delete(0, tkinter.END) self.insideDiaEntry.insert(0, str(self.data_store.get_inside_dia())) return False except IndexError: self.logger.error(str(e)) self.wallThicknessEntry.delete(0, tkinter.END) self.wallThicknessEntry.insert(0, str(self.data_store.get_wall_thickness())) except Exception as e: self.logger.error(str(e)) messagebox.showerror("Unknown Error", "Unknown exception trying to convert inside diameter to a floating point number.\nRead value was \"%s\".\nException: %s" % (v, str(e))) self.wallThicknessEntry.delete(0, tkinter.END) self.wallThicknessEntry.insert(0, str(self.data_store.get_wall_thickness())) @debugger def wallThicknessCommand(self, event=None): try: v = self.wallThicknessEntry.get() n = float(v) if n != self.data_store.get_wall_thickness(): self.logger.debug("change wall from %f to %f"%(self.data_store.get_wall_thickness(), n)) self.data_store.set_wall_thickness(n) self.wallThicknessEntry.delete(0, tkinter.END) self.wallThicknessEntry.insert(0, str(n)) raise_event("CALCULATE_EVENT") self.data_store.set_change_flag() else: self.logger.debug("ignore") return True except ValueError as e: self.logger.error(str(e)) messagebox.showerror("Error", "Could not convert wall thickness to a floating point number.\nRead value was \"%s\"." % (v)) self.wallThicknessEntry.delete(0, tkinter.END) self.wallThicknessEntry.insert(0, str(self.data_store.get_wall_thickness())) return False except IndexError: self.logger.error(str(e)) self.wallThicknessEntry.delete(0, tkinter.END) self.wallThicknessEntry.insert(0, str(self.data_store.get_wall_thickness())) except Exception as e: self.logger.error(str(e)) messagebox.showerror("Unknown Error", "Unknown exception trying convert wall thickness to a floating point number.\nRead value was \"%s\".\nException %s" % (v, str(e))) self.wallThicknessEntry.delete(0, tkinter.END) self.wallThicknessEntry.insert(0, str(self.data_store.get_wall_thickness())) @debugger def numHolesCommand(self, event=None): n = 0 try: v = self.numHolesEntry.get() n = int(v) if n >= 1 and n <= 12: # only raise the event if the number of holes is different from # what is in the data_store if n != self.data_store.get_number_holes(): self.logger.debug("change number of holes from %d to %d"%(self.data_store.get_number_holes(), n)) self.data_store.set_number_holes(n) raise_event('UPDATE_LOWER_FRAME_EVENT') self.data_store.set_change_flag() else: self.logger.debug("ignore") return True else: self.logger.error("range error on number of holes: %s"%(str(n))) messagebox.showerror("Error", message="Number of holes must be an integer between 1 and 12.\nRead value was \"%s\"." % (v)) self.numHolesEntry.delete(0, tkinter.END) self.numHolesEntry.insert(0, str(self.data_store.get_number_holes())) return False except ValueError as e: self.logger.error(str(e)) messagebox.showerror("Error", message="Could not convert number of holes to an integer.\nRead value was \"%s\"." % (v)) self.numHolesEntry.delete(0, tkinter.END) self.numHolesEntry.insert(0, str(self.data_store.get_number_holes())) return False except IndexError as e: self.logger.error(str(e)) self.numHolesEntry.delete(0, tkinter.END) self.numHolesEntry.insert(0, str(self.data_store.get_number_holes())) except Exception as e: self.logger.error(str(e)) messagebox.showerror("Unknown Error", message="Unknown exception trying to convert number of holes to an integer.\nRead value was \"%s\".\nException: %s" % (v, str(e))) self.wallThicknessEntry.delete(0, tkinter.END) self.wallThicknessEntry.insert(0, str(self.data_store.get_wall_thickness())) @debugger def displayFormatCallback(self, event): if self.displayFormatOpt.current() == 0: val = False else: val = True if val != self.data_store.get_disp_frac(): self.data_store.set_disp_frac(val) raise_event("UPDATE_HOLE_EVENT") self.logger.debug("current format set to: %s"%(str(self.data_store.get_disp_frac()))) self.data_store.set_change_flag() else: self.logger.debug("ignore") @debugger def measureUnitsCallback(self, event): if self.measureUnitsOpt.current() == 0: val = False else: val = True if self.data_store.get_units() != val: if self.measureUnitsOpt.current() == 1: self.displayFormatOpt.config(state=tkinter.DISABLED) else: self.displayFormatOpt.config(state="readonly") self.data_store.set_units(val) self.change_units() self.logger.debug("current units set to: %s"%(str(self.data_store.get_units()))) self.data_store.set_change_flag() else: self.logger.debug("ignore") @debugger def bellSelectCallback(self, event): ''' Change the data_store to match the new bell selection ''' val = self.bellNoteCombo.current() if val != self.data_store.get_bell_note_select(): self.data_store.set_bell_note_select(val) self.data_store.set_bell_freq(self.data_store.note_table[val]['frequency']) self.logger.debug("current bell selection set to: %d: %f"%(self.data_store.get_bell_note_select(), self.data_store.get_bell_freq())) raise_event("UPDATE_NOTES_EVENT") self.data_store.set_change_flag() else: self.logger.debug("ignore") self.set_state() @debugger def refreshButtonCommand(self): self.refreshButton.focus_set() self.get_state() #raise_event('UPDATE_LINES_EVENT') raise_event('UPDATE_LOWER_FRAME_EVENT') self.set_state() @debugger def change_units(self): ''' When this is called, the assumption is that the GUI and the data_store have the wrong units. This function takes what ever is in the data_sore and converts if to the units that it finds there. Then it updates the GUI. ''' if self.data_store.get_units(): # true of units are mm self.data_store.set_inside_dia(utility.in_to_mm(self.data_store.get_inside_dia())) self.data_store.set_wall_thickness(utility.in_to_mm(self.data_store.get_wall_thickness())) else: self.data_store.set_inside_dia(utility.mm_to_in(self.data_store.get_inside_dia())) self.data_store.set_wall_thickness(utility.mm_to_in(self.data_store.get_wall_thickness())) self.set_state() # Cause the other frames to update raise_event("CHANGE_UNITS_EVENT") raise_event('CALCULATE_EVENT') self.data_store.set_change_flag() @debugger def setTitleCommand(self, event=None): try: title = self.titleEntry.get() old_title = self.data_store.get_title() if title != old_title: self.data_store.set_title(title) self.logger.debug("title set to: \"%s\""%(str(self.data_store.get_title()))) self.data_store.set_change_flag() else: self.logger.debug("ignore") except IndexError: pass # always ignore
class SetupFormBase(object): ''' This class provides common services for forms in the setup notebook. ''' def __init__(self, master, table, empty_ok=False): self.logger = Logger(self, level=Logger.DEBUG) self.logger.debug("Setup Dialog start constructor") self.master = master self.table = table self.empty_ok = empty_ok self.data = Database.get_instance() self.events = EventHandler.get_instance() self.id_list = self.get_id_list() self.crnt_index = 0 @debugger def get_id_list(self): ''' This method exists so that a form can manage a smaller set of records than every single one. To do that, override this default method in the form class. ''' return self.data.get_id_list(self.table) @debugger def get_id(self): ''' Returns the id of the record in the current form. ''' return self.id_list[self.crnt_index] @debugger def select_button_command(self): self.id_list = self.get_id_list() sel = SelectItem(self.master, self.table) #if not hasattr(sel, 'item_id'): if sel.item_id == -1: self.logger.debug('Select dialog was canceled') elif sel.item_id == 0: self.logger.debug('Select dialog item was not found') else: try: self.logger.debug('Select dialog item selected = %d' % (sel.item_id)) self.crnt_index = self.id_list.index(sel.item_id) self.set_form() except TypeError: mb.showerror( 'ERROR', 'No record was selected. (no records are available?)') self.events.raise_event('select_button') @debugger def new_button_command(self): ''' Clear the form ''' self.clear_form() self.events.raise_event('new_button') @debugger def save_button_command(self): ''' Save the form to the database ''' if not self.id_list is None: self.get_form() self.events.raise_event('save_button') @debugger def del_button_command(self): ''' Delete the item given in the form from the database ''' if not self.id_list is None: val = mb.askokcancel( "Sure?", "Are you sure you want to delete item from %s?" % (self.table)) if val: self.logger.info("Deleting item %d from %s" % (self.id_list[self.crnt_index], self.table)) self.data.delete_row(self.table, self.id_list[self.crnt_index]) self.data.commit() self.id_list = self.get_id_list() if self.crnt_index >= len(self.id_list): self.crnt_index -= 1 self.set_form() self.events.raise_event('del_button') @debugger def next_btn_command(self): ''' Go to the next item in the form table ''' if not self.id_list is None: self.crnt_index += 1 if self.crnt_index >= len(self.id_list): self.crnt_index = len(self.id_list) - 1 self.set_form() self.events.raise_event('next_button') @debugger def prev_btn_command(self): ''' Go to the previous item in the table ''' if not self.id_list is None: self.crnt_index -= 1 if self.crnt_index < 0: self.crnt_index = 0 self.set_form() self.events.raise_event('prev_button') @debugger def clear_form(self): ''' Clear the form. ''' for item in self.form_contents: print(item) item['self'].clear() self.id_list = None self.events.raise_event('clear_form') @debugger def set_form(self): #, row_id): ''' Read the database and place the data in the form. ''' if self.id_list is None: return try: self.id_list = self.get_id_list() row_id = self.id_list[self.crnt_index] except IndexError: if not self.empty_ok: self.logger.info('No records defined for table \'%s\'' % (self.table)) mb.showinfo( 'Records', 'There are no records available for this form: \'%s\".' % (self.table)) self.clear_form() return row = self.data.get_row_by_id(self.table, row_id) if row is None: if not self.empty_ok: self.logger.info('No records defined for table \'%s\'' % (self.table)) mb.showinfo( 'Records', 'There are no records available for this table: \'%s\'.' % (self.table)) self.clear_form() return print(self.form_contents) for item in self.form_contents: if not item['hasid'] is None: # swap in the value that the ID points to rather than the actual ID item['hasid']['id'] = int(row[item['column']]) tmp_row = self.data.get_row_by_id(item['hasid']['table'], item['hasid']['id']) item['self'].write(tmp_row[item['hasid']['column']]) else: item['self'].write(row[item['column']]) self.events.raise_event('set_form') @debugger def get_form(self): ''' Read the form and place the data in the database. ''' if self.id_list is None: return row = {} for item in self.form_contents: if not item['hasid'] is None: # If in the future, forms that require a writable ID in the # form is implemented, then this line will have to change. row[item['column']] = item['hasid']['id'] else: row[item['column']] = item['self'].read() if self.id_list is None: self.data.insert_row(self.table, row) else: row_id = self.id_list[self.crnt_index] self.data.update_row_by_id(self.table, row, row_id) self.id_list = self.get_id_list() self.events.raise_event('get_form')
def move_to_point_and_extract( coords_from_to: list, gps: adapters.GPSUbloxAdapter, vesc_engine: adapters.VescAdapter, smoothie: adapters.SmoothieAdapter, camera: adapters.CameraAdapterIMX219_170, periphery_det: detection.YoloOpenCVDetection, precise_det: detection.YoloOpenCVDetection, client, logger_full: utility.Logger, logger_table: utility.Logger, report_field_names, used_points_history: list, undistorted_zone_radius, working_zone_polygon, working_zone_points_cv, view_zone_polygon, view_zone_points_cv, img_output_dir, nav: navigation.GPSComputing): """ Moves to the given target point and extracts all weeds on the way. :param coords_from_to: :param gps: :param vesc_engine: :param smoothie: :param camera: :param periphery_det: :param precise_det: :param client: :param logger_full: :param logger_table: :param report_field_names: :param used_points_history: :param nav: :param working_zone_polygon: :param undistorted_zone_radius: :param working_zone_points_cv: :param img_output_dir: :return: """ vesc_engine.apply_rpm(config.VESC_RPM_SLOW) slow_mode_time = -float("inf") current_working_mode = working_mode_slow = 1 working_mode_switching = 2 working_mode_fast = 3 close_to_end = True # True if robot is close to one of current movement vector points, False otherwise raw_angles_history = [] stop_helping_point = nav.get_coordinate(coords_from_to[1], coords_from_to[0], 90, 1000) prev_maneuver_time = time.time() prev_pos = gps.get_last_position() # set camera to the Y min res = smoothie.custom_move_to(config.XY_F_MAX, X=config.X_MAX / 2 / config.XY_COEFFICIENT_TO_MM, Y=config.Y_MIN) if res != smoothie.RESPONSE_OK: msg = "INIT: Failed to move camera to Y min, smoothie response:\n" + res logger_full.write(msg + "\n") smoothie.wait_for_all_actions_done() # main navigation control loop while True: # EXTRACTION CONTROL start_t = time.time() frame = camera.get_image() frame_t = time.time() plants_boxes = periphery_det.detect(frame) per_det_t = time.time() debug_save_image( img_output_dir, "(periphery view scan M=" + str(current_working_mode) + ")", frame, plants_boxes, undistorted_zone_radius, working_zone_points_cv if current_working_mode == 1 else view_zone_points_cv) msg = "View frame time: " + str( frame_t - start_t) + "\t\tPer. det. time: " + str(per_det_t - frame_t) logger_full.write(msg + "\n") # slow mode if current_working_mode == working_mode_slow: if any_plant_in_zone(plants_boxes, working_zone_polygon): vesc_engine.stop_moving() time.sleep(0.2) start_work_t = time.time() frame = camera.get_image() frame_t = time.time() plants_boxes = precise_det.detect(frame) pre_det_t = time.time() debug_save_image(img_output_dir, "(precise view scan M=1)", frame, plants_boxes, undistorted_zone_radius, working_zone_points_cv) msg = "Work frame time: " + str( frame_t - start_work_t) + "\t\tPrec. det. time: " + str(pre_det_t - frame_t) logger_full.write(msg + "\n") if any_plant_in_zone(plants_boxes, working_zone_polygon): extract_all_plants(smoothie, camera, precise_det, working_zone_polygon, frame, plants_boxes, undistorted_zone_radius, working_zone_points_cv, img_output_dir) elif not any_plant_in_zone(plants_boxes, view_zone_polygon) and \ time.time() - slow_mode_time > config.SLOW_MODE_MIN_TIME: # set camera to the Y max res = smoothie.custom_move_to( config.XY_F_MAX, X=config.X_MAX / 2 / config.XY_COEFFICIENT_TO_MM, Y=config.Y_MAX / config.XY_COEFFICIENT_TO_MM) if res != smoothie.RESPONSE_OK: msg = "M=" + str( current_working_mode ) + ": " + "Failed to move to Y max, smoothie response:\n" + res logger_full.write(msg + "\n") smoothie.wait_for_all_actions_done() current_working_mode = working_mode_switching vesc_engine.start_moving() # switching to fast mode elif current_working_mode == working_mode_switching: if any_plant_in_zone(plants_boxes, view_zone_polygon): vesc_engine.stop_moving() # set camera to the Y min res = smoothie.custom_move_to(config.XY_F_MAX, X=config.X_MAX / 2 / config.XY_COEFFICIENT_TO_MM, Y=config.Y_MIN) if res != smoothie.RESPONSE_OK: msg = "M=" + str( current_working_mode ) + ": " + "Failed to move to Y min, smoothie response:\n" + res logger_full.write(msg + "\n") smoothie.wait_for_all_actions_done() current_working_mode = working_mode_slow slow_mode_time = time.time() elif smoothie.get_smoothie_current_coordinates(False)[ "Y"] + config.XY_COEFFICIENT_TO_MM * 20 > config.Y_MAX: current_working_mode = working_mode_fast if not close_to_end: vesc_engine.apply_rpm(config.VESC_RPM_FAST) # fast mode else: if any_plant_in_zone(plants_boxes, view_zone_polygon): vesc_engine.stop_moving() # set camera to the Y min res = smoothie.custom_move_to(config.XY_F_MAX, X=config.X_MAX / 2 / config.XY_COEFFICIENT_TO_MM, Y=config.Y_MIN) if res != smoothie.RESPONSE_OK: msg = "M=" + str( current_working_mode ) + ": " + "Failed to move to Y min, smoothie response:\n" + res logger_full.write(msg + "\n") smoothie.wait_for_all_actions_done() current_working_mode = working_mode_slow slow_mode_time = time.time() vesc_engine.apply_rpm(config.VESC_RPM_SLOW) elif close_to_end: vesc_engine.apply_rpm(config.VESC_RPM_SLOW) else: vesc_engine.apply_rpm(config.VESC_RPM_FAST) # NAVIGATION CONTROL nav_start_t = time.time() cur_pos = gps.get_last_position() if str(cur_pos) == str(prev_pos): # msg = "Got the same position, added to history, calculations skipped. Am I stuck?" # print(msg) # logger_full.write(msg + "\n") continue used_points_history.append(cur_pos.copy()) if not client.sendData("{};{}".format(cur_pos[0], cur_pos[1])): msg = "[Client] Connection closed !" print(msg) logger_full.write(msg + "\n") distance = nav.get_distance(cur_pos, coords_from_to[1]) msg = "Distance to B: " + str(distance) # print(msg) logger_full.write(msg + "\n") # check if arrived _, side = nav.get_deviation(coords_from_to[1], stop_helping_point, cur_pos) # if distance <= config.COURSE_DESTINATION_DIFF: # old way if side != 1: # TODO: maybe should use both side and distance checking methods at once vesc_engine.stop_moving() # msg = "Arrived (allowed destination distance difference " + str(config.COURSE_DESTINATION_DIFF) + " mm)" msg = "Arrived to " + str(coords_from_to[1]) # print(msg) logger_full.write(msg + "\n") break # pass by cur points which are very close to prev point to prevent angle errors when robot is staying # (too close points in the same position can produce false huge angles) if nav.get_distance(prev_pos, cur_pos) < config.PREV_CUR_POINT_MIN_DIST: continue # reduce speed if near the target point if config.USE_SPEED_LIMIT: distance_from_start = nav.get_distance(coords_from_to[0], cur_pos) close_to_end = distance < config.DECREASE_SPEED_TRESHOLD or distance_from_start < config.DECREASE_SPEED_TRESHOLD # do maneuvers not more often than specified value cur_time = time.time() if cur_time - prev_maneuver_time < config.MANEUVERS_FREQUENCY: continue prev_maneuver_time = cur_time msg = "Prev: " + str(prev_pos) + " Cur: " + str(cur_pos) + " A: " + str(coords_from_to[0]) \ + " B: " + str(coords_from_to[1]) # print(msg) logger_full.write(msg + "\n") raw_angle = nav.get_angle(prev_pos, cur_pos, cur_pos, coords_from_to[1]) # sum(e) if len(raw_angles_history) >= config.WINDOW: raw_angles_history.pop(0) raw_angles_history.append(raw_angle) sum_angles = sum(raw_angles_history) if sum_angles > config.SUM_ANGLES_HISTORY_MAX: msg = "Sum angles " + str(sum_angles) + " is bigger than max allowed value " + \ str(config.SUM_ANGLES_HISTORY_MAX) + ", setting to " + str(config.SUM_ANGLES_HISTORY_MAX) # print(msg) logger_full.write(msg + "\n") sum_angles = config.SUM_ANGLES_HISTORY_MAX elif sum_angles < -config.SUM_ANGLES_HISTORY_MAX: msg = "Sum angles " + str(sum_angles) + " is less than min allowed value " + \ str(-config.SUM_ANGLES_HISTORY_MAX) + ", setting to " + str(-config.SUM_ANGLES_HISTORY_MAX) # print(msg) logger_full.write(msg + "\n") sum_angles = -config.SUM_ANGLES_HISTORY_MAX angle_kp_ki = raw_angle * config.KP + sum_angles * config.KI target_angle_sm = angle_kp_ki * -config.A_ONE_DEGREE_IN_SMOOTHIE # smoothie -Value == left, Value == right ad_wheels_pos = smoothie.get_adapter_current_coordinates()["A"] sm_wheels_pos = smoothie.get_smoothie_current_coordinates()["A"] # compute order angle (smoothie can't turn for huge values immediately also as cancel movement, # so we need to do nav. actions in steps) order_angle_sm = target_angle_sm - ad_wheels_pos # check for out of update frequency and smoothie execution speed range (for nav wheels) if order_angle_sm > config.MANEUVERS_FREQUENCY * config.A_DEGREES_PER_SECOND * \ config.A_ONE_DEGREE_IN_SMOOTHIE: msg = "Order angle changed from " + str( order_angle_sm) + " to " + str( config.MANEUVERS_FREQUENCY * config.A_DEGREES_PER_SECOND + config.A_ONE_DEGREE_IN_SMOOTHIE ) + " due to exceeding degrees per tick allowed range." # print(msg) logger_full.write(msg + "\n") order_angle_sm = config.MANEUVERS_FREQUENCY * config.A_DEGREES_PER_SECOND * \ config.A_ONE_DEGREE_IN_SMOOTHIE elif order_angle_sm < -(config.MANEUVERS_FREQUENCY * config.A_DEGREES_PER_SECOND * config.A_ONE_DEGREE_IN_SMOOTHIE): msg = "Order angle changed from " + str( order_angle_sm) + " to " + str(-( config.MANEUVERS_FREQUENCY * config.A_DEGREES_PER_SECOND * config.A_ONE_DEGREE_IN_SMOOTHIE )) + " due to exceeding degrees per tick allowed range." # print(msg) logger_full.write(msg + "\n") order_angle_sm = -(config.MANEUVERS_FREQUENCY * config.A_DEGREES_PER_SECOND * config.A_ONE_DEGREE_IN_SMOOTHIE) # convert to global smoothie coordinates order_angle_sm += ad_wheels_pos # checking for out of smoothie supported range if order_angle_sm > config.A_MAX: msg = "Global order angle changed from " + str(order_angle_sm) + " to config.A_MAX = " + \ str(config.A_MAX) + " due to exceeding smoothie allowed values range." # print(msg) logger_full.write(msg + "\n") order_angle_sm = config.A_MAX elif order_angle_sm < config.A_MIN: msg = "Global order angle changed from " + str(order_angle_sm) + " to config.A_MIN = " + \ str(config.A_MIN) + " due to exceeding smoothie allowed values range." # print(msg) logger_full.write(msg + "\n") order_angle_sm = config.A_MIN raw_angle = round(raw_angle, 2) angle_kp_ki = round(angle_kp_ki, 2) order_angle_sm = round(order_angle_sm, 2) sum_angles = round(sum_angles, 2) distance = round(distance, 2) ad_wheels_pos = round(ad_wheels_pos, 2) sm_wheels_pos = round(sm_wheels_pos, 2) gps_quality = cur_pos[2] msg = str(gps_quality).ljust(5) + str(raw_angle).ljust(8) + str( angle_kp_ki).ljust(8) + str(order_angle_sm).ljust(8) + str( sum_angles).ljust(8) + str(distance).ljust(13) + str( ad_wheels_pos).ljust(8) + str(sm_wheels_pos).ljust(9) print(msg) logger_full.write(msg + "\n") # load sensors data to csv s = "," msg = str(gps_quality) + s + str(raw_angle) + s + str(angle_kp_ki) + s + str(order_angle_sm) + s + \ str(sum_angles) + s + str(distance) + s + str(ad_wheels_pos) + s + str(sm_wheels_pos) vesc_data = vesc_engine.get_sensors_data(report_field_names) if vesc_data is not None: msg += s for key in vesc_data: msg += str(vesc_data[key]) + s msg = msg[:-1] logger_table.write(msg + "\n") prev_pos = cur_pos response = smoothie.nav_turn_wheels_to(order_angle_sm, config.A_F_MAX) if response != smoothie.RESPONSE_OK: # TODO: what if response is not ok? msg = "Smoothie response is not ok: " + response print(msg) logger_full.write(msg + "\n") msg = "Nav calc time: " + str(time.time() - nav_start_t) logger_full.write(msg + "\n\n")
class NotesBox(tk.Frame): ''' Implement a notes widget and provide a regular interface to it, same as the other form widgets. ''' def __init__(self, master, table, column, height=20, width=60, *args, **kargs): ''' master = the frame to bind this frame to name = the text of the label table = the name of the database table that is associated with this widget column = the name of the column this widget associates with lw = label width cw = control width ch = control height ''' self.logger = Logger(self, level=Logger.INFO) self.logger.debug("NotesBox enter constructor") super().__init__(master, *args, **kargs) self.column = column self.table = table frame2 = tk.Frame(self, bd=1, relief=tk.RIDGE) frame2.grid(row=0, column=1) self.content = tk.Text(frame2, height=height, width=width, wrap=tk.NONE) self.content.insert(tk.END, '') # see https://www.homeandlearn.uk/tkinter-scrollbars.html self.vsb = tk.Scrollbar(frame2, orient=tk.VERTICAL) self.vsb.config(command=self.content.yview) self.content.config(yscrollcommand=self.vsb.set) self.vsb.pack(side=tk.RIGHT, fill=tk.Y) self.hsb = tk.Scrollbar(frame2, orient=tk.HORIZONTAL) self.hsb.config(command=self.content.xview) self.content.config(xscrollcommand=self.hsb.set) self.hsb.pack(side=tk.BOTTOM, fill=tk.X) self.content.pack(side=tk.LEFT) self.row = { 'table': self.table, 'column': self.column, 'self': self, 'hasid': None } self.logger.debug("NotesBox leave constructor") @debugger def read(self): return self.content.get(1.0, tk.END) @debugger def write(self, val): self.content.delete('1.0', tk.END) if not val is None: self.content.insert(tk.END, val) @debugger def clear(self): self.content.delete('1.0', tk.END) @debugger def get_line(self): ''' Return the form entry to update the form. ''' return self.row
def move_to_point(coords_from_to: list, used_points_history: list, gps: adapters.GPSUbloxAdapter, vesc_engine: adapters.VescAdapter, smoothie: adapters.SmoothieAdapter, logger: utility.Logger, client, nav: navigation.GPSComputing, raw_angles_history: list): """ Moves to given point. :param coords_from_to: :param used_points_history: :param gps: :param vesc_engine: :param smoothie: :param logger: :param client: :param nav: :return: :param raw_angles_history: """ raw_angles_history = [] stop_helping_point = nav.get_coordinate(coords_from_to[1], coords_from_to[0], 90, 1000) prev_maneuver_time = time.time() prev_point = gps.get_fresh_position( ) # TODO: maybe it's ok to get last position instead of waiting for fresh vesc_engine.set_rpm(int(config.VESC_RPM / 2)) vesc_engine.start_moving() # main navigation control loop while True: cur_pos = gps.get_fresh_position() used_points_history.append(cur_pos.copy()) if not client.sendData("{};{}".format(cur_pos[0], cur_pos[1])): msg = "[Client] Connection closed !" print(msg) logger.write(msg + "\n") if str(cur_pos) == str(prev_point): msg = "Got the same position, added to history, calculations skipped" print(msg) logger.write(msg + "\n") continue distance = nav.get_distance(cur_pos, coords_from_to[1]) msg = "Distance to B: " + str(distance) print(msg) logger.write(msg + "\n") # check if arrived _, side = nav.get_deviation(coords_from_to[1], stop_helping_point, cur_pos) # if distance <= config.COURSE_DESTINATION_DIFF: # old way if side != 1: # TODO: maybe should use both side and distance checking methods at once vesc_engine.stop_moving() # msg = "Arrived (allowed destination distance difference " + str(config.COURSE_DESTINATION_DIFF) + " mm)" msg = "Arrived." print(msg) logger.write(msg + "\n") break # reduce speed if near the target point if config.USE_SPEED_LIMIT: distance_from_start = nav.get_distance(coords_from_to[0], cur_pos) if distance < config.DECREASE_SPEED_TRESHOLD or distance_from_start < config.DECREASE_SPEED_TRESHOLD: vesc_engine.apply_rpm(int(config.VESC_RPM / 2)) else: vesc_engine.apply_rpm(config.VESC_RPM) # do maneuvers not more often than specified value cur_time = time.time() if cur_time - prev_maneuver_time < config.MANEUVERS_FREQUENCY: continue prev_maneuver_time = cur_time msg = "Timestamp: " + str(cur_time) print(msg) logger.write(msg + "\n") msg = "Prev: " + str(prev_point) + " Cur: " + str(cur_pos) + " A: " + str(coords_from_to[0]) \ + " B: " + str(coords_from_to[1]) print(msg) logger.write(msg + "\n") raw_angle = nav.get_angle(prev_point, cur_pos, cur_pos, coords_from_to[1]) # sum(e) if len(raw_angles_history) >= config.WINDOW: raw_angles_history.pop(0) raw_angles_history.append(raw_angle) sum_angles = sum(raw_angles_history) if sum_angles > config.SUM_ANGLES_HISTORY_MAX: msg = "Sum angles " + str(sum_angles) + " is bigger than max allowed value " + \ str(config.SUM_ANGLES_HISTORY_MAX) + ", setting to " + str(config.SUM_ANGLES_HISTORY_MAX) print(msg) logger.write(msg) sum_angles = config.SUM_ANGLES_HISTORY_MAX elif sum_angles < -config.SUM_ANGLES_HISTORY_MAX: msg = "Sum angles " + str(sum_angles) + " is less than min allowed value " + \ str(-config.SUM_ANGLES_HISTORY_MAX) + ", setting to " + str(-config.SUM_ANGLES_HISTORY_MAX) print(msg) logger.write(msg) sum_angles = -config.SUM_ANGLES_HISTORY_MAX angle_kp_ki = raw_angle * config.KP + sum_angles * config.KI target_angle_sm = angle_kp_ki * -config.A_ONE_DEGREE_IN_SMOOTHIE # smoothie -Value == left, Value == right ad_wheels_pos = smoothie.get_adapter_current_coordinates()["A"] sm_wheels_pos = smoothie.get_smoothie_current_coordinates()["A"] # compute order angle (smoothie can't turn for huge values immediately also as cancel movement, # so we need to do nav. actions in steps) order_angle_sm = target_angle_sm - ad_wheels_pos # check for out of update frequency and smoothie execution speed (for nav wheels) if order_angle_sm > config.MANEUVERS_FREQUENCY * config.A_DEGREES_PER_SECOND * \ config.A_ONE_DEGREE_IN_SMOOTHIE: msg = "Order angle changed from " + str( order_angle_sm) + " to " + str( config.MANEUVERS_FREQUENCY * config.A_DEGREES_PER_SECOND + config.A_ONE_DEGREE_IN_SMOOTHIE ) + " due to exceeding degrees per tick allowed range." print(msg) logger.write(msg + "\n") order_angle_sm = config.MANEUVERS_FREQUENCY * config.A_DEGREES_PER_SECOND * \ config.A_ONE_DEGREE_IN_SMOOTHIE elif order_angle_sm < -(config.MANEUVERS_FREQUENCY * config.A_DEGREES_PER_SECOND * config.A_ONE_DEGREE_IN_SMOOTHIE): msg = "Order angle changed from " + str( order_angle_sm) + " to " + str(-( config.MANEUVERS_FREQUENCY * config.A_DEGREES_PER_SECOND * config.A_ONE_DEGREE_IN_SMOOTHIE )) + " due to exceeding degrees per tick allowed range." print(msg) logger.write(msg + "\n") order_angle_sm = -(config.MANEUVERS_FREQUENCY * config.A_DEGREES_PER_SECOND * config.A_ONE_DEGREE_IN_SMOOTHIE) # convert to global coordinates order_angle_sm += ad_wheels_pos # checking for out of smoothie supported range if order_angle_sm > config.A_MAX: msg = "Global order angle changed from " + str(order_angle_sm) + " to config.A_MAX = " + \ str(config.A_MAX) + " due to exceeding smoothie allowed values range." print(msg) logger.write(msg + "\n") order_angle_sm = config.A_MAX elif order_angle_sm < config.A_MIN: msg = "Global order angle changed from " + str(order_angle_sm) + " to config.A_MIN = " + \ str(config.A_MIN) + " due to exceeding smoothie allowed values range." print(msg) logger.write(msg + "\n") order_angle_sm = config.A_MIN msg = "Adapter wheels pos (target): " + str(ad_wheels_pos) + " Smoothie wheels pos (current): " \ + str(sm_wheels_pos) print(msg) logger.write(msg + "\n") msg = "KI: " + str(config.KI) + " Sum angles: " + str(sum_angles) + " Sum angles history: " + \ str(raw_angles_history) print(msg) logger.write(msg + "\n") msg = "KP: " + str(config.KP) + " Raw angle: " + str(raw_angle) + " Angle * KP + sum(angles) * KI: " + \ str(angle_kp_ki) + " Smoothie target angle: " + str(target_angle_sm) + \ " Smoothie absolute order angle: " + str(order_angle_sm) print(msg) logger.write(msg + "\n") prev_point = cur_pos response = smoothie.nav_turn_wheels_to(order_angle_sm, config.A_F_MAX) msg = "Smoothie response: " + response print(msg) logger.write(msg) # next tick indent print() logger.write("\n")
class LineBox(tk.Frame): ''' This is a container for the line widgets. It holds a variable number of line boxes and has the ability to write them all to the database when the write method is called. ''' def __init__(self, master, form=None, *args, **kargs): ''' master = The frame to bind the widgets to. name_id = The id of the line containing the customer to associate form = Name of the form to bind the events to. ''' self.logger = Logger(self, level=Logger.DEBUG) self.logger.debug("Line Widget enter constructor") super().__init__(master, bd=1, relief=tk.RIDGE, *args, **kargs) #self.name_id = int(name_id) self.form = form self.events = EventHandler.get_instance() self.data = Database.get_instance() self.line_list = [] self.crnt_index = 0 self.events.register_event('next_button', self.clear) self.events.register_event('prev_button', self.clear) # add button tk.Button(self, text="Add", command=self.add).grid(row=0, column=0) # reset button tk.Button(self, text="Reset", command=self.clear).grid(row=0, column=1) # add one line widget self.add() # self.row = {'table': None, 'column':None, 'self':self, 'hasid':None} self.logger.debug("Line Widget leave constructor") @debugger def add(self): ''' Method that actually adds the line to the widget. ''' line = LineWidget(self, None, self.crnt_index + 1) line.grid(row=self.crnt_index + 1, column=0, columnspan=2, padx=5) self.line_list.append(line) self.crnt_index += 1 # @debugger # def get_line(self): # return self.row @debugger def clear(self): ''' Reset the widget to having one blank line. ''' for item in self.line_list: item.grid_forget() self.line_list = [] self.crnt_index = 0 self.add() @debugger def read(self, sale_id): ''' Read method saves the contects to the database ''' for item in self.line_list: (quan, name) = item.read() if name != '': row = { 'sale_record_ID': sale_id, 'quantity': quan, 'inventory_ID': self.data.get_id_by_name('InventoryItem', name) } self.data.insert_row('ProductList', row) self.data.commit() @debugger def write(self): ''' This function does nothing for this widget ''' self.clear()
def configure(filename="tuf.interposition.json", parent_repository_directory=None, parent_ssl_certificates_directory=None): """ The optional parent_repository_directory parameter is used to specify the containing parent directory of the "repository_directory" specified in a configuration for *all* network locations, because sometimes the absolute location of the "repository_directory" is only known at runtime. If you need to specify a different parent_repository_directory for other network locations, simply call this method again with different parameters. Ditto for the optional parent_ssl_certificates_directory parameter. Example of a TUF interposition configuration JSON object: { "configurations": { "seattle.cs.washington.edu": { "repository_directory": "client/", "repository_mirrors" : { "mirror1": { "url_prefix": "http://seattle-tuf.cs.washington.edu", "metadata_path": "metadata", "targets_path": "targets", "confined_target_dirs": [ "" ] } }, ("target_paths": [ { ".*/(simple/\\w+)/$": "{0}/index.html" }, { ".*/(packages/.+)$": "{0}" } ], "ssl_certificates": "cacert.pem") } } } "target_paths" is optional: If you do not tell TUF to selectively match paths with regular expressions, TUF will work over any path under the given network location. However, if you do specify it, you are then telling TUF how to transform a specified path into another one, and TUF will *not* recognize any unspecified path for the given network location. Unless any "url_prefix" begins with "https://", "ssl_certificates" is optional; it must specify certificates bundled as PEM (RFC 1422). """ INVALID_TUF_CONFIGURATION = "Invalid configuration for {network_location}!" INVALID_TUF_INTERPOSITION_JSON = "Invalid configuration in {filename}!" NO_CONFIGURATIONS = "No configurations found in configuration in {filename}!" try: with open(filename) as tuf_interposition_json: tuf_interpositions = json.load(tuf_interposition_json) configurations = tuf_interpositions.get("configurations", {}) if len(configurations) == 0: raise InvalidConfiguration(NO_CONFIGURATIONS.format(filename=filename)) else: for network_location, configuration in configurations.iteritems(): try: configuration_parser = ConfigurationParser(network_location, configuration, parent_repository_directory=parent_repository_directory, parent_ssl_certificates_directory=parent_ssl_certificates_directory) configuration = configuration_parser.parse() __updater_controller.add(configuration) except: Logger.exception(INVALID_TUF_CONFIGURATION.format(network_location=network_location)) raise except: Logger.exception(INVALID_TUF_INTERPOSITION_JSON.format(filename=filename)) raise
'sgd_huber_loss_combine_sampled', 'sgd_huber_loss_over_sampled', 'sgd_huber_loss_under_sampled', 'sgd_log_loss_basic', 'sgd_log_loss_combine_sampled', 'sgd_log_loss_over_sampled', 'sgd_log_loss_under_sampled', 'xgboost_basic', 'xgboost_combine_sampled', 'xgboost_over_sampled', 'xgboost_under_sampled', ] if __name__ == '__main__': use_project_path() logger = Logger('model/experiment/output/model_scoring_search.txt') scores = [] for predictions in input_predictions: with open('model/experiment/output/%s_predict_proba.p' % predictions, 'rb') as file: frame = pickle.load(file) optimal_threshold, optimal_f1_score = find_optimal_f1_threshold(frame) y_actual = frame.y_actual y_predict_proba = frame.y_predict y_predict = (y_predict_proba[:, 1] >= optimal_threshold).astype(bool) score = [ predictions, optimal_threshold, optimal_f1_score, accuracy_score(y_actual, y_predict),
class BaseDialog(tk.Toplevel): ''' This class provides common services to simple data dialogs. ''' def __init__(self, parent):# , title = None): #init the logger self.logger = Logger(self, level=Logger.DEBUG) self.logger.debug("Base Dialog start constructor") tk.Toplevel.__init__(self, parent) self.transient(parent) self.parent = parent self.result = None # get a copy of the data_store for the children body = tk.Frame(self) self.initial_focus = self.body(body) body.grid(padx=5, pady=5) self.buttonbox() self.grab_set() if not self.initial_focus: self.initial_focus = self self.protocol("WM_DELETE_WINDOW", self.cancel) self.initial_focus.focus_set() #self.wait_window(self) self.logger.debug("Base Dialog leave constructor") # # construction hooks def body(self, master): # create dialog body. return widget that should have # initial focus. this method should be overridden return self def buttonbox(self): # add standard button box. override if you don't want the # standard buttons box = tk.Frame(self) w = tk.Button(box, text="OK", width=10, command=self.ok, default=tk.ACTIVE) w.pack(side=tk.LEFT, padx=5, pady=5) w = tk.Button(box, text="Cancel", width=10, command=self.cancel) w.pack(side=tk.LEFT, padx=5, pady=5) box.grid() # # standard button semantics @debugger def ok(self, event=None): if not self.validate(): self.initial_focus.focus_set() # put focus back return self.withdraw() self.update_idletasks() self.apply() self.cancel() @debugger def cancel(self, event=None): # put focus back to the parent window self.parent.focus_set() self.destroy() # # command hooks def validate(self): return True # override def apply(self): pass # override
class HoleSizeWidgit(tkinter.Frame): def __init__(self, parent, line): ''' This is a specialized widget to track the hole diameter. It displays the value according to the state. It has up and down buttons used to increment or decrement the value. ''' self.logger = Logger(self, Logger.INFO) self.logger.debug("constructor") self.index = line tkinter.Frame.__init__(self, parent) self.data_store = DataStore.get_instance() BITMAP_UP = """ #define up_width 9 #define up_height 5 static unsigned char up_bits[] = { 0x10, 0x00, 0x38, 0x00, 0x7c, 0x00, 0xfe, 0x00, 0xff, 0x01 }; """ BITMAP_DOWN = """ #define down_width 9 #define down_height 5 static unsigned char down_bits[] = { 0xff, 0x01, 0xfe, 0x00, 0x7c, 0x00, 0x38, 0x00, 0x10, 0x00 }; """ self.entry = tkinter.Entry(self, width=25-18) self.entry.grid(row=0, column=1, rowspan=2) self.bitmap_up = tkinter.BitmapImage(data=BITMAP_UP) self.b1 = tkinter.Button(self, text="up", image=self.bitmap_up, command=self.incr_command) self.b1.grid(row=0, column=2) self.b1.config(width=9, height=5) self.bitmap_down = tkinter.BitmapImage(data=BITMAP_DOWN) self.b2 = tkinter.Button(self, image=self.bitmap_down, command=self.decr_command) self.b2.grid(row=1, column=2) self.b2.config(width=9, height=5) register_event("UPDATE_HOLE_EVENT", self.update_val) self.set_state() self.logger.debug("constructor return") @debugger def get_state(self): ''' Put the current value into the data_store. ''' try: if self.data_store.get_disp_frac(): val = utility.fractof(self.entry.get()) else: val = float(self.entry.get()) except ValueError as e: mbox.showerror('ERROR', 'Cannot convert hole to float: \"%s\": %s'%(self.entry.get(), e)) self.data_store.set_hole_size(self.index, val) @debugger def set_state(self): ''' Get the current state from the data_store and place it in the GUI ''' self.update_val() # display the new state @debugger def update_val(self): siz = self.data_store.get_hole_size(self.index) self.entry.delete(0, tkinter.END) if not self.data_store.get_units(): self.logger.debug("updating inch value to %s"%(str(siz))) if self.data_store.get_disp_frac(): self.entry.insert(0, "%s"%(utility.reduce(siz))) else: self.entry.insert(0, "%0.4f"%(siz)) else: self.logger.debug("updating mm value to %s"%(str(siz))) self.entry.insert(0, "%0.4f"%(siz)) # event handlers @debugger def incr_command(self): siz = self.data_store.get_hole_size(self.index) siz = siz + self.data_store.get_hole_inc() if siz > self.data_store.get_hole_max(): siz = self.data_store.get_hole_max() elif siz < self.data_store.get_hole_min(): siz = self.data_store.get_hole_min() self.data_store.set_hole_size(self.index, siz) self.update_val() # update the GUI raise_event("CALCULATE_EVENT") self.data_store.set_change_flag() @debugger def decr_command(self): siz = self.data_store.get_hole_size(self.index) siz = siz - self.data_store.get_hole_inc() if siz < self.data_store.get_hole_min(): siz = self.data_store.get_hole_min() elif siz > self.data_store.get_hole_max(): siz = self.data_store.get_hole_max() self.data_store.set_hole_size(self.index, siz) self.update_val() # update the GUI raise_event("CALCULATE_EVENT") self.data_store.set_change_flag() def print_state(self): self.logger.msg(str(self.get_state()))
def __init__(self, notebook): self.logger = Logger(self, Logger.DEBUG) self.logger.debug(sys._getframe().f_code.co_name) super().__init__(notebook, notebook.PURCHASE_FRAME, 'Business') self.add_title('Purchases Form')
def __init__(self, notebook): self.logger = Logger(self, Logger.DEBUG) self.logger.debug(sys._getframe().f_code.co_name) super().__init__(notebook, notebook.REPORTS_FRAME, 'Business') self.add_title('Reports Form')
def __init__(self, master): self.logger = Logger(self, Logger.INFO) self.logger.debug("constructor") self.master = master self.data_store = DataStore.get_instance() register_event("UPDATE_UPPER_EVENT", self.set_state)
'0110', '0130', '0150', '0141', '0261', '0262', '0271', '0272', '031A', '031B', '0326', '033A', '033B', '041A', '041B', '0450', '0451', '0480', '0481', '051A', '051B', '0550', '0551', '141A', '141B' ] public_violence_excluded_locations = [ 'APARTMENT', 'CHA APARTMENT', 'BUSINESS OFFICE', 'COIN OPERATED MACHINE', 'COLLEGE/UNIVERSITY RESIDENCE HALL', 'FACTORY/MANUFACTURING BUILDING', 'RESIDENCE-GARAGE', 'NURSING HOME/RETIREMENT HOME', 'RESIDENCE', 'WAREHOUSE', 'OTHER' ] if __name__ == '__main__': use_project_path() logger = Logger('Data_Scratch/preprocess.txt') iucr_codes = load_codes('data_scratch/iucr_codes.json') type_codes = load_codes('data_scratch/type_codes.json') description_codes = load_codes('data_scratch/description_codes.json') location_codes = load_codes('data_scratch/location_codes.json') fbi_codes = load_codes('data_scratch/fbi_codes.json') missing_columns = {} for column in column_names: missing_columns[column] = 0 raw_record_count = -1 processed_records = 0 logger.time_log('Starting Pre-Processing.') with open('data_scratch/Crimes_-_2001_to_present.csv') as input_file:
def cleanup(self): """Clean up after certain side effects, such as temporary directories.""" DELETED_TEMPDIR_MESSAGE = "Deleted temporary directory at {tempdir}" shutil.rmtree(self.tempdir) Logger.debug(DELETED_TEMPDIR_MESSAGE.format(tempdir=self.tempdir))