def read_file(self): if set(self.config["files"]) == set(self.bad_files): dialogs.report_no_files() exit(3) current = self.config["files"][self.current_file] file_path = os.path.join(self.folder, current) if current in self.bad_files or not os.path.exists(file_path): if not os.path.exists(file_path): self.datafile = None self.bad_files.append(current) dialogs.notify_read_error(current) self.next_file() self.read_file() return try: self.datafile = DataFile(file_path, self.config["labels"]) self.insert_header() except (UnrecognizedFormatError, BadFileError): self.datafile = None self.bad_files.append(current) dialogs.notify_read_error(current) self.next_file() self.read_file()
def __init__(self, config): self.data = Data() self.config = config self.instrument_class = 'env_sensor' self.name = config['instrument_name'] server_address = (config['interface_config']['host'], config['interface_config']['port']) self.iface = TCPInterface(server_address, self.data) self.iface.start() df_config = { 'base_path': config['datafile_config']['base_path'], 'instrument_class': self.instrument_class, 'instrument_name': self.name, 'write_freq': config['datafile_config']['write_freq'], } self.data_file = DataFile(df_config) self.out_dt = '' self.out_data = {} #self.data = None self.task_list = []
def main(): """ This is the main function of the code it is the starting point of a simulation. """ # Load input parameters from the input file and add the, in allcaps # to the global namespace. global EXTERNAL_FIELD_VECTOR, ELECTRODE parameters = load_input(sys.argv[1], param_descriptors) globals().update(dict((key.upper(), item) for key, item in parameters.iteritems())) if RANDOM_SEED >= 0: seed(RANDOM_SEED) EXTERNAL_FIELD_VECTOR = array([0.0, 0.0, EXTERNAL_FIELD]) ELECTRODE = init_electrode() # init a tree from scratch tr, r0, q0 = init_from_scratch(INITIAL_NODES) dt = TIME_STEP t = r_[0:END_TIME:dt] r, q = r0, q0 dfile = DataFile(OUT_FILE, parameters=parameters) branched = False for i, it in enumerate(t): # with ContextTimer("plotting"): # plot_projections(r, q) # pylab.savefig('tree_%.3d.png' % i) # print 't = %g\ttree_%.3d.png' % (it, i) print "%d/%d t = %g" % (i, len(t), it) branch_prob = BRANCHING_PROBABILITY if SINGLE_BRANCHING_TIME > 0: if it > SINGLE_BRANCHING_TIME: if not branched: branch_prob = inf branched = True if SINGLE_BRANCHING_Z != 0 and not branched: zterm = r[tr.terminals()[0], Z] if zterm < SINGLE_BRANCHING_Z: if not branched: branch_prob = inf branched = True r, q = adapt_step(tr, r, q, dt, p=branch_prob) with ContextTimer("saving %d" % i): phi = solve_phi(r, q) dfile.add_step(it, tr, r, q, phi, error=error, error_dq=error_dq) if END_WITH_RECONNECTION and tr.reconnects(r): print "Finishing due to a reconnection." break
def main(): """ This is the main function of the code it is the starting point of a simulation. """ # Load input parameters from the input file and add the, in allcaps # to the global namespace. global EXTERNAL_FIELD_VECTOR, ELECTRODE parameters = load_input(sys.argv[1], param_descriptors) globals().update( dict((key.upper(), item) for key, item in parameters.iteritems())) if RANDOM_SEED >= 0: seed(RANDOM_SEED) EXTERNAL_FIELD_VECTOR = array([0.0, 0.0, EXTERNAL_FIELD]) ELECTRODE = init_electrode() # init a tree from scratch tr, r0, q0 = init_from_scratch(INITIAL_NODES) dt = TIME_STEP t = r_[0:END_TIME:dt] r, q = r0, q0 dfile = DataFile(OUT_FILE, parameters=parameters) branched = False for i, it in enumerate(t): # with ContextTimer("plotting"): # plot_projections(r, q) # pylab.savefig('tree_%.3d.png' % i) # print 't = %g\ttree_%.3d.png' % (it, i) print "%d/%d t = %g" % (i, len(t), it) branch_prob = BRANCHING_PROBABILITY if SINGLE_BRANCHING_TIME > 0: if it > SINGLE_BRANCHING_TIME: if not branched: branch_prob = inf branched = True if SINGLE_BRANCHING_Z != 0 and not branched: zterm = r[tr.terminals()[0], Z] if zterm < SINGLE_BRANCHING_Z: if not branched: branch_prob = inf branched = True r, q = adapt_step(tr, r, q, dt, p=branch_prob) with ContextTimer("saving %d" % i): phi = solve_phi(r, q) dfile.add_step(it, tr, r, q, phi, error=error, error_dq=error_dq) if END_WITH_RECONNECTION and tr.reconnects(r): print "Finishing due to a reconnection." break
def latest_commit(self): commit = DataFile(self.dir + '.git\\LatestCommit') if not commit['commit']: commit['commit'] = {'sha': '', 'etag': ''} etag = commit['commit']['etag'] sha = commit['commit']['sha'] resp = self.get('git/refs/heads/' + self.branch, etag) if resp.hasdata(): if not resp.is_json(): raise TypeError('Response not in JSON') sha = resp.data['object']['sha'] commit.merge('commit', {'etag': resp.etag, 'sha': sha}) return sha
def latest_commit(self): commit=DataFile(self.dir+'.git\\LatestCommit') if not commit['commit']: commit['commit']={'sha':'','etag':''} etag=commit['commit']['etag'] sha=commit['commit']['sha'] resp=self.get('git/refs/heads/'+self.branch,etag) if resp.hasdata(): if not resp.is_json(): raise TypeError('Response not in JSON') sha=resp.data['object']['sha'] commit.merge('commit',{'etag':resp.etag,'sha':sha}) return sha
def write_stats(ranges, results, prefix="stats"): attrs = list(ranges.keys()) df = DataFile(os.path.join(base_path, '%s_%s.dat' % (prefix, '_'.join(attrs))), ['entries', 'lookups', 'hits', 'total time', 'lookup time', 'scans'] + attr_names("iter time") + attr_names("iter thruput") + attr_names("file size") + attr_names("scan thruput") + attrs, overwrite=True) for config, reps in results: cfg = dict(config) # TODO: avg. over the reps iter_times = Entry("iter time") iter_tps = Entry("iter thruput") file_sizes = Entry("file size") scan_tps = Entry("scan thruput") for data in reps: db_path, entries, lookups, hits, total_time, num_scans, scans_time, lookup_time, iter_time, iter_tp = data.split(", ") iter_times.add(int(iter_time)) iter_tps.add(int(iter_tp)) file_sizes.add(int(os.path.getsize(db_path))) scan_tps.add(float(scans_time)/float(num_scans)) for attr in ranges: df[attr] = cfg[attr] df['entries'] = entries df['lookups'] = lookups df['hits'] = hits df['total time'] = total_time df['lookup time'] = lookup_time df['scans'] = num_scans # df['file size'] = os.path.getsize(db_path) for attr in attr_names("iter time"): df[attr] = getattr(iter_times, attr.split()[-1]) for attr in attr_names("iter thruput"): df[attr] = getattr(iter_tps, attr.split()[-1]) for attr in attr_names("file size"): df[attr] = getattr(file_sizes, attr.split()[-1]) for attr in attr_names("scan thruput"): df[attr] = getattr(scan_tps, attr.split()[-1]) print file_sizes print iter_times print iter_tps print scan_tps df.save() df.close()
def read_file(self): if set(self.files_list) == set(self.bad_files): dialogs.report_no_files() exit(3) current = self.files_list[self.current_file] if current in self.bad_files: self.next_file() self.read_file() return try: self.datafile = DataFile(current, self.config["labels"]) except (UnrecognizedFormatError, BadFileError): self.datafile = None self.bad_files.append(current) dialogs.notify_read_error(os.path.basename(current)) self.next_file() self.read_file()
def __init__(self, name, options): self.name = name self.money = 0 self.lastwinMoney = 0 self.winCount = 0 self.playCount = 0 self.continuousLoss = 0 #投注策略用到 self.continuousWin = 0 #投注策略用到 self.originBetting = options.betting self.currentBetting = options.betting self.gameCount = options.gameCount self.odds = options.odds self.maxBetting = options.maxBetting self.totalNumber = options.totalNumber self.numberBetting = options.numberBetting self.probability = self.probabilityOfWinningEachPeriod() stime = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S-%f') self.datafile = DataFile(r"E:\Git\turkeyGame\var\sample-" + stime + "-" + str(name) + ".xlsx")
def exec_ranges(cmd, args, range_attrs, num_reps=10, pbs_queue=None, pbs_name='storagesim', config=None): # range_attrs is in the form: # range_val: [list of possible values] # # when several range_vals and value lists are given, all possible # permutations are executed base_path = args['base_dir'] if config != None: df = DataFile(os.path.join(config, 'config_ranges.dat'), list(range_attrs.keys()), overwrite=True) outputs = [] for attr_p in permutations(range_attrs): # set the attrs accordingly for this exec names = [] for attr, val in attr_p: args[attr] = val names.append('%s_%s' % (str(attr), str(val))) if config != None: df[attr] = val if config != None: df.save() args['base_dir'] = os.path.join(base_path, '%s' % ('_'.join(names))) if not os.path.exists(args['base_dir']): os.mkdir(args['base_dir']) outputs.append((attr_p, exec_repetitions(cmd, args, num_reps=num_reps, pbs_queue=pbs_queue, pbs_name=pbs_name))) if config != None: df.close() return outputs
def write_stats(ranges, results, prefix="stats"): attrs = list(ranges.keys()) df = DataFile( os.path.join(base_path, '%s_%s.dat' % (prefix, '_'.join(attrs))), ['entries', 'lookups', 'hits', 'total time', 'lookup time', 'scans'] + attr_names("iter time") + attr_names("iter thruput") + attr_names("file size") + attr_names("scan thruput") + attrs, overwrite=True) for config, reps in results: cfg = dict(config) # TODO: avg. over the reps iter_times = Entry("iter time") iter_tps = Entry("iter thruput") file_sizes = Entry("file size") scan_tps = Entry("scan thruput") for data in reps: db_path, entries, lookups, hits, total_time, num_scans, scans_time, lookup_time, iter_time, iter_tp = data.split( ", ") iter_times.add(int(iter_time)) iter_tps.add(int(iter_tp)) file_sizes.add(int(os.path.getsize(db_path))) scan_tps.add(float(scans_time) / float(num_scans)) for attr in ranges: df[attr] = cfg[attr] df['entries'] = entries df['lookups'] = lookups df['hits'] = hits df['total time'] = total_time df['lookup time'] = lookup_time df['scans'] = num_scans # df['file size'] = os.path.getsize(db_path) for attr in attr_names("iter time"): df[attr] = getattr(iter_times, attr.split()[-1]) for attr in attr_names("iter thruput"): df[attr] = getattr(iter_tps, attr.split()[-1]) for attr in attr_names("file size"): df[attr] = getattr(file_sizes, attr.split()[-1]) for attr in attr_names("scan thruput"): df[attr] = getattr(scan_tps, attr.split()[-1]) print file_sizes print iter_times print iter_tps print scan_tps df.save() df.close()
def get_inv_status(path): """ Updates dictionary where invoice number is a key and value is a string that represents actual status of invoice :param path: path to excel file that contains list of invoices to be checked :return: updates REQUESTED_INVOICE_STATUSES dictionary with data query """ for invoice in DataFile.load_invoices(file_path=path): if invoice in GFIS_DATA.keys() and not None: REQUESTED_INVOICE_STATUSES[invoice] = \ f'Scheduled due {GFIS_DATA[invoice][0]}, payment: {GFIS_DATA[invoice][1]}' elif invoice in COMBINED_DATA.keys(): REQUESTED_INVOICE_STATUSES[invoice] = STATUS_CODES[str( COMBINED_DATA[invoice])] elif invoice not in COMBINED_DATA.keys(): REQUESTED_INVOICE_STATUSES[invoice] = 'Missing'
from threading import Thread from datafile import DataFile from crawl import Crawl import sys import os import time import json # 任务队列,从主线程到工作线程 task_queue = Queue(maxsize=100000) # 响应队列,从工作线程到主线程 response_queue = Queue() # 数据存储 data_file = DataFile() # 用户信息获取 crawl = Crawl() # 工作线程数量 threads_num = 100 class MasterThread(Thread): def __init__(self): super(MasterThread, self).__init__() self.count = { 'crawled_count': 0, # 已爬用户数量 'task_count': 0, # 任务数量
def read_and_save_flash(self, filename, with_bootloader): _df = DataFile([]) _df.data = self.read_all_flash(with_bootloader) print 'Read', len(_df.data), 'bytes' _df.save(filename)
s = '' i = 0 for v in lst: vs = hex(v)[2:] i += 1 if len(vs) == 1: vs = '0' + vs s += vs + ' ' if (i % 16) == 0: print s s = '' #fw = DataFile('/Users/trol/Projects/radio/avr-lcd-module-128x128/build/avr-lcd-module-128x128.hex') fw = DataFile( '/Users/trol/Projects/radio/avr-ic-tester-v2/firmware/tester/build/ic-tester-main.hex' ) # read 230400 44.0383300884 us/byte #write 234000 256.21552423 us/byte # 255.434597666 us/byte #l = Loader('/dev/tty.wchusbserial14230', 57600) #l = Loader('/dev/tty.wchusbserial14230', 230400) l = Loader('/dev/tty.wchusbserial14220', 153600) print l.dev.get_about() l.read_and_save_flash('flash_with_loader.hex', True) l.read_and_save_flash('flash_without_loader.hex', False) l.write_flash(fw.data) l.dev.start_app() 1 / 0
from functions import retrieve_gfis_data, data_from_combined, data_from_flow,\ get_inv_status, write_status from datafile import DataFile import glob if __name__ == '__main__': print('################### Basware/GFIS invoice status checking.##########################') print('========================= Created by Dmytro Zimin =============================') print('################################################################################\n\n') print('***Please make sure you have read the instruction manual before using this program***') run = input('To proceed press <y> or any other button to exit\n') if run.lower()[0] == 'y': print('combining basware *.csv files to excel ') for file in glob.glob('basware/*.csv'): basware_file = DataFile(file) basware_file.combine_to_excel('basware', 'basware/combined.xlsx') print('combining flow *.csv files to excel ') for file in glob.glob('flow/*.csv'): flow_file = DataFile(file) flow_file.combine_to_excel('flow', 'flow/flow.xlsx') print('retrieving data...') retrieve_gfis_data('gfis/*.xlsx') data_from_combined('basware/combined.xlsx') data_from_flow('flow/flow.xlsx') get_inv_status('check_invoices.xlsx') print('writing statuses...') write_status('check_invoices.xlsx') print('deleting temporary files...') DataFile.remove_temporary_files('basware/combined.xlsx') DataFile.remove_temporary_files('flow/flow.xlsx') input('Statuses have been added to check_invoices.xlsx! Press any key to exit')
class EnvSensor(): def __init__(self, config): self.data = Data() self.config = config self.instrument_class = 'env_sensor' self.name = config['instrument_name'] server_address = (config['interface_config']['host'], config['interface_config']['port']) self.iface = TCPInterface(server_address, self.data) self.iface.start() df_config = { 'base_path': config['datafile_config']['base_path'], 'instrument_class': self.instrument_class, 'instrument_name': self.name, 'write_freq': config['datafile_config']['write_freq'], } self.data_file = DataFile(df_config) self.out_dt = '' self.out_data = {} #self.data = None self.task_list = [] def start(self): print("Starting EnvSensor") #self.attempt_connect = True task = asyncio.ensure_future(self.read_loop()) self.task_list.append(task) self.is_running = True def stop(self): self.iface.stop() #tasks = asyncio.Task.all_tasks() for t in self.task_list: # print('stop:') # print(t) t.cancel() self.task_list.remove(t) # tasks.remove(t) self.is_running = False self.attempt_connect = False async def read_loop(self): last_read = datetime.now() # print(last_read) while True: # print(self.iface.has_data()) if (self.iface.has_data()): # print('here') buffer = self.iface.read()[0] # print(buffer) # print('here2') dt = buffer['DateTime'] # print(dt) #buf = buffer['Data'] # print(buf) # print(buffer['Data']['RH']['value']) # print(buffer['Data']['RH']) buf = buffer['InstrumentData'] # print(buf) meas_list = buf['METADATA']['MeasurementList'] # print(meas_list) self.out_dt = dt #dat = {} out = '' for meas in meas_list: value = buf['DATA'][meas]['value'] units = buf['DATA'][meas]['units'] self.out_data[meas] = { 'value': buf['DATA'][meas]['value'], 'units': buf['DATA'][meas]['units'], } # print(meas) # print(dat) # print(self.out_data) # print(type(value)) # print(units) out += ' {}={:.2f}{}'.format(meas, value, units) # print(out) #t = buffer['Data']['Temperature']['value'] #rh = buffer['Data']['RH']['value'] # print(rh) #print('[{}] -- {} : T={}C, RH={}%'.format(self.name,dt,t,rh)) #print('[{}] -- {} : {}'.format(self.name,dt,out)) self.data_file.append(buffer) last_read = datetime.now() # print(last_read) #buffer = data.get() #print('buffer : ' + buffer) # print('read_buffer:') # print(buffer) delta = (datetime.now() - last_read).total_seconds() # print(delta) if delta > 5.0: self.out_dt = '' self.out_data = {} # pass await asyncio.sleep(0.1)
from customer import create_customer, print_header from rooms import get_and_print_room_by_no, change_room_status from datafile import DataFile from datetime import datetime NUMBER_OF_RECORDS_PER_PAGE = 10 customerTable = DataFile("customers.dat") def add_customer(): rooms, found, position = get_and_print_room_by_no() if found: room_no = rooms[position].room_no customers = customerTable.get_records() if len(customers) == 0: customer_id = 0 else: customer_id = customers[len(customers) - 1].customer_id + 1 customer = create_customer(customer_id, room_no) confirm = input("Complete the operation? (Y/N) ") if confirm.lower() == 'y': customerTable.add_record(customer) change_room_status(room_no, False) print("Operation Successful") else: print("Operation Canceled") def get_and_print_customer_list_by_name(): customers = customerTable.get_records() results = []
class ProjectData: def __init__(self, project): self.folder = os.path.dirname(project) self.project_file = project self.current_file = 0 self.current_label = 0 self.modified = False self.bad_files = [] self.datafile = None self.config = None self.read_conf() self.read_file() def read(self): if self.modified: self.read_conf() self.modified = False self.read_file() if self.current_label >= len(self.config["labels"]): self.current_label = 0 def read_conf(self): try: with open(self.project_file) as in_file: self.config = json.load(in_file) except IOError: logger.error("Unable to read {}: permission denied".format( self.project_file)) exit(2) def read_file(self): if set(self.config["files"]) == set(self.bad_files): dialogs.report_no_files() exit(3) current = self.config["files"][self.current_file] file_path = os.path.join(self.folder, current) if current in self.bad_files or not os.path.exists(file_path): if not os.path.exists(file_path): self.datafile = None self.bad_files.append(current) dialogs.notify_read_error(current) self.next_file() self.read_file() return try: self.datafile = DataFile(file_path, self.config["labels"]) self.insert_header() except (UnrecognizedFormatError, BadFileError): self.datafile = None self.bad_files.append(current) dialogs.notify_read_error(current) self.next_file() self.read_file() def insert_header(self): header = self.datafile.get_data_header() if str(header) not in self.config.keys(): self.config[str(header)] = { "plot": [[i] for i in self.datafile.get_data_columns()], "normalize": [], "functions": [] } self.modified = True self.save_config() def save_file(self): self.datafile.save() def save_config(self): if self.modified: try: with open(self.project_file, 'w') as out_file: json.dump(self.config, out_file) except IOError: logger.error("Unable to write {}: permission denied".format( self.project_file)) exit(2) self.modified = False def next_label(self): self.current_label = (self.current_label + 1) % len( self.config["labels"]) def prev_label(self): length = len(self.config["labels"]) self.current_label = (self.current_label - 1 + length) % length def get_current_label(self): label = self.config["labels"][self.current_label] color = self.config["colors"][self.current_label] return label, color def get_label_color(self, label): index = self.config["labels"].index(label) return self.config["colors"][index] def get_labels_info(self): return self.config["labels"], self.config["colors"] def set_labels_info(self, names, colors): self.config["labels"] = names self.config["colors"] = colors self.modified = True if self.current_label >= len(names): self.current_label = 0 def get_plot_info(self): header = self.datafile.get_data_header() conf = self.config[str(header)] return conf["plot"], conf["normalize"] def set_plot_info(self, plot_set, normalize): header = self.datafile.get_data_header() conf = self.config[str(header)] conf["plot"] = plot_set conf["normalize"] = normalize self.modified = True def next_file(self): self.current_file = (self.current_file + 1) % len(self.config["files"]) def prev_file(self): self.current_file = (self.current_file - 1 + len( self.config["files"])) % len(self.config["files"]) def get_functions(self): header = self.datafile.get_data_header() return self.config[str(header)]["functions"] def add_function(self, fs): header = self.datafile.get_data_header() new_header = header + [fs.name] self.config[str(new_header)] = self.config[str(header)] self.config[str(new_header)]["functions"].append(fs.name) self.datafile.add_function(fs) self.modified = True def remove_function(self, index): header = self.datafile.get_data_header() conf = self.config[str(header)] f_name = conf["functions"][index] f_col = self.datafile.get_data_columns()[header.index(f_name)] for plot in conf["plot"]: if f_col in plot: plot.remove(f_col) for i, col in enumerate(plot): if col > f_col: plot[i] = col - 1 self.datafile.remove_function(f_name) del conf["functions"][index] self.modified = True
class FilesData: def __init__(self, files): self.files_list = files self.config_list = [] self.current_file = 0 self.current_label = 0 self.modified = False self.bad_files = [] self.datafile = None self.config = None self.init() self.read() def init(self): for file in self.files_list: conf = file + ".json" file_conf = conf if os.path.exists(conf) else None self.config_list.append(file_conf) # noinspection PyTypeChecker def read(self): conf_path = self.config_list[self.current_file] if conf_path: self.read_conf() self.read_file() else: self.config = { "labels": ["Label"], "colors": ["#1f77b4"], } self.read_file() self.config["plot"] = [[i] for i in self.datafile.get_data_columns()] self.config["normalize"] = [] self.config["functions"] = [] if self.current_label >= len(self.config["labels"]): self.current_label = 0 self.modified = False def read_conf(self): conf_path = self.config_list[self.current_file] try: with open(conf_path) as in_file: self.config = json.load(in_file) except IOError: logger.error( "Unable to read {}: permission denied".format(conf_path)) exit(2) def read_file(self): if set(self.files_list) == set(self.bad_files): dialogs.report_no_files() exit(3) current = self.files_list[self.current_file] if current in self.bad_files: self.next_file() self.read_file() return try: self.datafile = DataFile(current, self.config["labels"]) except (UnrecognizedFormatError, BadFileError): self.datafile = None self.bad_files.append(current) dialogs.notify_read_error(os.path.basename(current)) self.next_file() self.read_file() def save_file(self): self.datafile.save() def save_config(self): if self.modified: conf_path = self.files_list[self.current_file] + ".json" try: with open(conf_path, 'w') as out_file: json.dump(self.config, out_file) except IOError: logger.error( "Unable to write {}: permission denied".format(conf_path)) exit(2) self.config_list[self.current_file] = conf_path self.modified = False def next_label(self): self.current_label = (self.current_label + 1) % len( self.config["labels"]) def prev_label(self): length = len(self.config["labels"]) self.current_label = (self.current_label - 1 + length) % length def get_current_label(self): label = self.config["labels"][self.current_label] color = self.config["colors"][self.current_label] return label, color def get_label_color(self, label): index = self.config["labels"].index(label) return self.config["colors"][index] def get_labels_info(self): return self.config["labels"], self.config["colors"] def set_labels_info(self, names, colors): self.config["labels"] = names self.config["colors"] = colors self.modified = True if self.current_label >= len(names): self.current_label = 0 def get_plot_info(self): return self.config["plot"], self.config["normalize"] def set_plot_info(self, plot_set, normalize): self.config["plot"] = plot_set self.config["normalize"] = normalize self.modified = True def next_file(self): self.current_file = (self.current_file + 1) % len(self.files_list) def prev_file(self): self.current_file = (self.current_file - 1 + len(self.files_list)) % len(self.files_list) def get_functions(self): return self.config["functions"] def add_function(self, fs): self.datafile.add_function(fs) self.config["functions"].append(fs.name) self.modified = True def remove_function(self, index): f_name = self.config["functions"][index] header = self.datafile.get_data_header() f_col = self.datafile.get_data_columns()[header.index(f_name)] for plot in self.config["plot"]: if f_col in plot: plot.remove(f_col) for i, col in enumerate(plot): if col > f_col: plot[i] = col - 1 self.datafile.remove_function(f_name) del self.config["functions"][index] self.modified = True
处理在线接单任务 """ def recv_task_online(self,is_fast=True): recv = ReceiveOnline() recv.run(is_fast) if( __name__ == "__main__"): start_time = time.time() #que = queue.Queue() send_thread_list = [] #receive_thread_list = [] cmcc = Cmcc() data_path = './data/data.txt' path_pre = data_path[0:-4] thread_count = 4 data = DataFile(data_path,thread_count) data.run() for i in range(0,thread_count): send_thread = threading.Thread(target=cmcc.send_task,args=(path_pre+str(i)+".txt","./data/Send_Thread"+str(i)+".txt")) send_thread.setDaemon(True) send_thread.start() send_thread_list.append(send_thread) time.sleep(3) #睡120秒再启动快线程 time.sleep(120) thread = threading.Thread(target=cmcc.recv_task_online,args=(True,)) thread.setDaemon(True) thread.start() thread_slow = None while True: """
def retrieve_gfis_data(path_wildcard): """ :param path_wildcard: path to directory that contains *.xlsx files :return: updates the dictionary GFIS_DATA where invoice_number is a key, and schedule_date, parsed_date, payments are values. """ for file in glob.glob(path_wildcard): try: DataFile.remove_row(file) gfis_excel = load_workbook(filename=f'{file}') gfis_sheet = gfis_excel.active except FileNotFoundError: print('File *.xlsx in <gfis> not found') else: invoice_numbers = [ str(invoice[0]) for invoice in gfis_sheet.iter_rows(min_row=MIN_ROW, min_col=GFIS_INVOICE_COL, max_col=GFIS_INVOICE_COL, values_only=True) ] schedule_dates = [ datetime.strftime(schedule[0], '%Y-%m-%d') if schedule[0] else 'no data' for schedule in gfis_sheet.iter_rows(min_row=MIN_ROW, min_col=GFIS_SCHEDULE_COL, max_col=GFIS_SCHEDULE_COL, values_only=True) ] spread_payment_dates = [ payment_date[0] for payment_date in gfis_sheet.iter_rows( min_row=MIN_ROW, min_col=last_column(file), max_col=last_column(file), values_only=True) ] parsed_payments_dates = [ datetime.strftime(date, '%Y-%m-%d') if date is not None else 'NOT PAID' for date in spread_payment_dates ] payments = [ payment[0] for payment in gfis_sheet.iter_rows(min_row=MIN_ROW, min_col=GFIS_PAYMENT_COL, max_col=GFIS_PAYMENT_COL, values_only=True) ] for invoice, schedule_date, payment_date, payment in zip( invoice_numbers, schedule_dates, parsed_payments_dates, payments): if invoice not in GFIS_DATA.keys(): GFIS_DATA[invoice] = schedule_date, payment_date, payment elif payment > GFIS_DATA.get(invoice)[2]: GFIS_DATA[invoice] = schedule_date, payment_date, payment
from crawl import Crawl from datafile import DataFile __author__ = """\ /\/\ ___ _ __ __ _ _ __ / \ / _ \| '__/ _` | '_ \ / /\/\ \ (_) | | | (_| | | | | \/ \/\___/|_| \__,_|_| |_|""" # 任务队列,从主线程到工作线程 task_queue = Queue(maxsize=100000) # 响应队列,从工作线程到主线程 response_queue = Queue() # 数据文件操作接口 df = DataFile() # 用户信息获取接口 crawl = Crawl() # 工作线程的数量 threads_numbers = 20 class MasterThread(Thread): """ 主线程 Attributes: count: 状态信息,用于实时显示爬虫状态 crawled_set: 已爬取用户集合,用于去除重复用户 task_set: 待爬取用户集合,元素与任务队列保持一致,用于去除重复用户
def load(self, directory): assert (os.path.isdir(directory)) self.reset() self.directory_ = directory dataFile = DataFile() print "-- loading data from '" + self.getName() + "'" # load coverage events filename = os.path.join(self.directory_, COVERAGE_EVENTS_FILE) dataFile.load(filename) self.coverageEvents_ = dataFile.getDataAs("fl") # load time events filename = os.path.join(self.directory_, TIME_EVENTS_FILE) dataFile.load(filename) self.timeEvents_ = dataFile.getDataAs("fl") # load experiment config filename = os.path.join(self.directory_, EXPERIMENT_CONFIG_FILE) dataFile.load(filename) self.robotCount_ = int(dataFile.data_[0][0]) self.worldType_ = dataFile.data_[0][1] # load final coverage filename = os.path.join(self.directory_, FINAL_COVERAGE_FILE) dataFile.load(filename) self.finalCoverage_ = dataFile.getDataAs("fl") # load meanTileTimeBetweenVisits filename = os.path.join(self.directory_, MEAN_TILE_TIME_BEWTEEN_VISITS_FILE) dataFile.load(filename) self.meanTileTimeBetweenVisits_ = dataFile.getDataAs("iil") # load tileVisits filename = os.path.join(self.directory_, TILE_VISITS_FILE) dataFile.load(filename) self.tileVisits_ = dataFile.getDataAs("iii") # load visitsEvents filename = os.path.join(self.directory_, VISITS_EVENTS_FILE) dataFile.load(filename) self.visitsEvents_ = dataFile.getDataAs("fff") # load timeBetweenVisitsEvents filename = os.path.join(self.directory_, TIME_BETWEEN_VISITS_EVENTS_FILE) dataFile.load(filename) self.timeBetweenVisitsEvents_ = dataFile.getDataAs("fll")
import os import time from queue import Queue from threading import Thread from bs4 import BeautifulSoup import requests from datafile import DataFile from login import Login from crawler import Crawler task_queue = Queue() response_queue = Queue() data = DataFile() thread_numbers = 3 class MasterThread(Thread): def __init__(self): Thread.__init__(self) crawled_list = data.load_crawed_list() self.crawled_set = set(crawled_list) un_crawled_user_list = data.load_uncrawled_user_list(self.crawled_set) self.task_set = set() for user_url in un_crawled_user_list: task_queue.put(user_url) self.task_set.add(user_url) self.temp_user_infos = []
def load(self, directory): assert(os.path.isdir(directory)) self.reset() self.directory_ = directory dataFile = DataFile() print "-- loading data from '" + self.getName() + "'" # load coverage events filename = os.path.join(self.directory_, COVERAGE_EVENTS_FILE) dataFile.load(filename) self.coverageEvents_= dataFile.getDataAs("fl") # load time events filename = os.path.join(self.directory_, TIME_EVENTS_FILE) dataFile.load(filename) self.timeEvents_= dataFile.getDataAs("fl") # load experiment config filename = os.path.join(self.directory_, EXPERIMENT_CONFIG_FILE) dataFile.load(filename) self.robotCount_ = int(dataFile.data_[0][0]); self.worldType_ = dataFile.data_[0][1]; # load final coverage filename = os.path.join(self.directory_, FINAL_COVERAGE_FILE) dataFile.load(filename) self.finalCoverage_ = dataFile.getDataAs("fl") # load meanTileTimeBetweenVisits filename = os.path.join(self.directory_, MEAN_TILE_TIME_BEWTEEN_VISITS_FILE) dataFile.load(filename) self.meanTileTimeBetweenVisits_ = dataFile.getDataAs("iil") # load tileVisits filename = os.path.join(self.directory_, TILE_VISITS_FILE) dataFile.load(filename) self.tileVisits_ = dataFile.getDataAs("iii") # load visitsEvents filename = os.path.join(self.directory_, VISITS_EVENTS_FILE) dataFile.load(filename) self.visitsEvents_ = dataFile.getDataAs("fff") # load timeBetweenVisitsEvents filename = os.path.join(self.directory_, TIME_BETWEEN_VISITS_EVENTS_FILE) dataFile.load(filename) self.timeBetweenVisitsEvents_ = dataFile.getDataAs("fll")
def save(self): print "saving summary of '" + self.getName() + "'..." summaryDir = self.getSummaryDir() if os.path.isdir(summaryDir): shutil.rmtree(summaryDir, True, False) mkdirRec(summaryDir) dataFile = DataFile() filename = "" for worldType in AVAILABLE_WORLDS: print "-- saving data of '" + worldType + "'" worldDir = os.path.join(summaryDir,worldType) mkdirRec(worldDir) for robotCount in AVAILABLE_ROBOT_COUNTS: if self.meanCoverageEvents_[worldType][robotCount].hasData(): filename = self.getSaveFilename(worldDir, COVERAGE_EVENTS_FILE, robotCount) dataFile.reset() dataFile.addComment("# [coverage meanTimestamp(usec)]") data = self.meanCoverageEvents_[worldType][robotCount].getMean() dataFile.setDataAs(data[:2], "fl") dataFile.save(filename) if self.meanTimeEvents_[worldType][robotCount].hasData(): filename = self.getSaveFilename(worldDir, TIME_EVENTS_FILE, robotCount) dataFile.reset() dataFile.addComment("# [coverage meanTimestamp(usec)]") data = self.meanTimeEvents_[worldType][robotCount].getMean() dataFile.setDataAs(data[:2], "fl") dataFile.save(filename) if self.meanTileTimeBetweenVisits_[worldType][robotCount].hasData(): filename = self.getSaveFilename(worldDir, MEAN_TILE_TIME_BEWTEEN_VISITS_FILE, robotCount) dataFile.reset() dataFile.addComment("# [x y meanTimeBetweenVisits(usec)]") data = self.meanTileTimeBetweenVisits_[worldType][robotCount].getMeanGrid() dataFile.setDataAs(data[:3], "iil") dataFile.save(filename) if self.meanTileVisits_[worldType][robotCount].hasData(): filename = self.getSaveFilename(worldDir, TILE_VISITS_FILE, robotCount) dataFile.reset() dataFile.addComment("# [x y meanVisitCount]") data = self.meanTileVisits_[worldType][robotCount].getMeanGrid() dataFile.setDataAs(data[:3], "iif") dataFile.save(filename) if self.meanFinalCoverage_[worldType][robotCount].hasData(): filename = self.getSaveFilename(worldDir, FINAL_COVERAGE_FILE, robotCount) dataFile.reset() dataFile.addComment("# [coverage timeStamp(usec)]") data = self.meanFinalCoverage_[worldType][robotCount].getMean() dataFile.setDataAs(data[:2], "fl") dataFile.save(filename)
from datafile import DataFile from room import create_room, print_header NUMBER_OF_RECORDS_PER_PAGE = 10 roomTable = DataFile("rooms.dat") def add_room(): rooms = roomTable.get_records() if len(rooms) == 0: room_id = 0 else: room_id = rooms[len(rooms) - 1].room_id + 1 room = create_room(room_id) roomTable.add_record(room) print("Operation Successful") def get_and_print_room_by_no(): rooms = roomTable.get_records() found = False position = -1 if len(rooms) == 0: print("No Record found") else: room_no = int(input("Enter the room no: ")) for room in rooms: position += 1 if room_no == room.room_no: found = True
def _parse_data_file(self, binary_control_record, data_fp): tf = DataFile(self.config) tf.from_binary(binary_control_record, data_fp) return tf
def run_files(filelist): fig_s = plt.figure('pos_deltas') ax_pos = fig_s.add_subplot(111) fig_s.suptitle('delta_lat_lon') ax_pos.set(ylabel='dlat (deg)') ax_pos.set(xlabel='dlon (deg)') ax_pos.grid() ax_pos.axis('equal') ax_pos.set_ylim(-5e-5, 5e-5) ax_pos.set_xlim(-5e-5, 5e-5) fig_en = plt.figure('enu_scatter') ax_en = fig_en.add_subplot(111) fig_en.suptitle('enu_scatter') ax_en.set(ylabel='dNorth (m)') ax_en.set(xlabel='dEast (m)') ax_en.grid() ax_en.axis('equal') ax_en.set_ylim(-10, 10) ax_en.set_xlim(-10, 10) fig_p = plt.figure('delta_pos_enu') ax_e = fig_p.add_subplot(311) fig_p.suptitle('delta_pos_enu') ax_e.set(ylabel='dE (m)') ax_e.set(xlabel='Time (s)') ax_e.grid() ax_e.set_ylim(-10, 10) ax_n = fig_p.add_subplot(312) ax_n.set(ylabel='dN (m)') ax_n.set(xlabel='Time (s)') ax_n.grid() ax_n.set_ylim(-10, 10) ax_u = fig_p.add_subplot(313) ax_u.set(ylabel='dU (m)') ax_u.set(xlabel='Time (s)') ax_u.grid() ax_u.set_ylim(-10, 10) for f in filelist: print('---- WORKING ' + f + ' ----') fname = f.split('.')[0] df = DataFile(f, data_dir='data/hw1/') df.setup() lat_mean = np.mean(df.file_df['latitude {deg}']) * u.degree lon_mean = np.mean(df.file_df['longitude {deg}']) * u.degree h_mean = np.mean(df.file_df['h_ellipsiod {m}']) * u.meter sats_mean = np.mean(df.file_df['num_sats']) dlats = df.file_df['latitude {deg}'] - lat_mean.value dlons = df.file_df['longitude {deg}'] - lon_mean.value dhs = df.file_df['h_ellipsiod {m}'] - h_mean.value dhs_std = np.std(dhs) lat_std = np.std(df.file_df['latitude {deg}']) lon_std = np.std(df.file_df['longitude {deg}']) CEP_latlon = 0.62 * lat_std + 0.56 * lon_std res = ax_pos.scatter(dlons, dlats, s=10, label=fname) color = res.get_facecolor()[0] circle = plt.Circle((0, 0), CEP_latlon, fill=False, color=color) ax_pos.add_artist(circle) print('-- Means -- lat = {}, lon = {}, h_mean = {}, num_sats = {}'. format(lat_mean, lon_mean, h_mean, sats_mean)) # compute ECEF position vector using mean LAT, LON, H r_ecef_mean = wgs84.geodetic_point_in_ecef(lat_mean, lon_mean, h_mean) rs_ecef = wgs84.geodetic_point_in_ecef_arr( df.file_df['latitude {deg}'].to_numpy() * u.degree, df.file_df['longitude {deg}'].to_numpy() * u.degree, df.file_df['h_ellipsiod {m}'].to_numpy() * u.meter) x_ecef_std = np.std(rs_ecef[:, 0]) y_ecef_std = np.std(rs_ecef[:, 1]) z_ecef_std = np.std(rs_ecef[:, 2]) SEP = 0.51 * (x_ecef_std + y_ecef_std + z_ecef_std) print('-- SEP -- {} {}'.format(SEP, u.meter.to_string())) print('-- dhs_std -- {} {}'.format(dhs_std, u.meter.to_string())) print('-- CEP_latlon -- {} {}'.format(CEP_latlon, u.degree.to_string())) print('ECEF_from_MEAN_GEO = ', r_ecef_mean) R_ecef_enu = wgs84.ecef_to_enu(lat_mean, lon_mean) rs_enu = np.zeros((len(rs_ecef), 3)) for ii in range(len(rs_ecef)): r_ecef = rs_ecef[ii, :] r_ecef_row = r_ecef_mean.flatten() dr_ecef = r_ecef - r_ecef_row.value dr_enu = np.matmul(R_ecef_enu, dr_ecef) rs_enu[ii, :] = dr_enu ax_e.plot(df.time_s, rs_enu[:, 0], label='SEP={:.2f} {}'.format(SEP, u.meter.to_string())) ax_n.plot(df.time_s, rs_enu[:, 1], label=f) ax_u.plot(df.time_s, rs_enu[:, 2], label=fname) ax_en.scatter(rs_enu[:, 0], rs_enu[:, 1], s=10, label=fname) # newline print("") ax_pos.legend() ax_e.legend() ax_u.legend() ax_en.legend()
class Turkey: '这是一只好斗的小火鸡' def __init__(self, name, options): self.name = name self.money = 0 self.lastwinMoney = 0 self.winCount = 0 self.playCount = 0 self.continuousLoss = 0 #投注策略用到 self.continuousWin = 0 #投注策略用到 self.originBetting = options.betting self.currentBetting = options.betting self.gameCount = options.gameCount self.odds = options.odds self.maxBetting = options.maxBetting self.totalNumber = options.totalNumber self.numberBetting = options.numberBetting self.probability = self.probabilityOfWinningEachPeriod() stime = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S-%f') self.datafile = DataFile(r"E:\Git\turkeyGame\var\sample-" + stime + "-" + str(name) + ".xlsx") #子类继承 def probabilityOfWinningEachPeriod(self): '每次中奖的概率:可由子类重写' return self.numberBetting / self.totalNumber def isWin(self): '本期是否中奖,随机获取' return random.randint(0, 100) < (self.probability * 100) #策略接口 def setStrategy(self, strategy): self.strategy = strategy #止盈止损接口 def setPlayer(self, player): self.player = player def earnMeneyOnePeriod(self): '本期是否赚钱。中则加钱,不中则扣钱' if self.isWin(): self.winCount = self.winCount + 1 self.continuousWin = self.continuousWin + 1 self.continuousLoss = 0 makeMoney = self.currentBetting * self.odds - self.currentBetting * self.numberBetting else: self.continuousLoss = self.continuousLoss + 1 self.continuousWin = 0 makeMoney = -(self.currentBetting * self.numberBetting) self.lastwinMoney = makeMoney self.playCount = self.playCount + 1 self.money = self.money + makeMoney def saveStateEachPeriod(self): '记录每局输赢' self.datafile.write2xlsx(numberPeriod=self.playCount, money=self.money, lastwinMoney=self.lastwinMoney, betting=self.currentBetting, probability=self.probability, cost=self.currentBetting * self.numberBetting) def startStrategy(self): '启用策略,当某几期之后,亏钱时,增大赌注,默认是翻倍;' self.strategy.doStrategy() def playGameEachPeriod(self): self.earnMeneyOnePeriod() self.saveStateEachPeriod() self.startStrategy() def cleanGame(self): self.datafile.save2xlsx() self.datafile.close def startGame(self): self.player.play() self.cleanGame() return self.money