def read_file(self):
if set(self.config["files"]) == set(self.bad_files):
dialogs.report_no_files()
exit(3)
current = self.config["files"][self.current_file]
file_path = os.path.join(self.folder, current)
if current in self.bad_files or not os.path.exists(file_path):
if not os.path.exists(file_path):
self.datafile = None
self.bad_files.append(current)
dialogs.notify_read_error(current)
self.next_file()
self.read_file()
return
try:
self.datafile = DataFile(file_path, self.config["labels"])
self.insert_header()
except (UnrecognizedFormatError, BadFileError):
self.datafile = None
self.bad_files.append(current)
dialogs.notify_read_error(current)
self.next_file()
self.read_file()
def __init__(self, config):
self.data = Data()
self.config = config
self.instrument_class = 'env_sensor'
self.name = config['instrument_name']
server_address = (config['interface_config']['host'],
config['interface_config']['port'])
self.iface = TCPInterface(server_address, self.data)
self.iface.start()
df_config = {
'base_path': config['datafile_config']['base_path'],
'instrument_class': self.instrument_class,
'instrument_name': self.name,
'write_freq': config['datafile_config']['write_freq'],
}
self.data_file = DataFile(df_config)
self.out_dt = ''
self.out_data = {}
#self.data = None
self.task_list = []
def main():
""" This is the main function of the code it is the starting point of
a simulation. """
# Load input parameters from the input file and add the, in allcaps
# to the global namespace.
global EXTERNAL_FIELD_VECTOR, ELECTRODE
parameters = load_input(sys.argv[1], param_descriptors)
globals().update(dict((key.upper(), item)
for key, item in parameters.iteritems()))
if RANDOM_SEED >= 0:
seed(RANDOM_SEED)
EXTERNAL_FIELD_VECTOR = array([0.0, 0.0, EXTERNAL_FIELD])
ELECTRODE = init_electrode()
# init a tree from scratch
tr, r0, q0 = init_from_scratch(INITIAL_NODES)
dt = TIME_STEP
t = r_[0:END_TIME:dt]
r, q = r0, q0
dfile = DataFile(OUT_FILE, parameters=parameters)
branched = False
for i, it in enumerate(t):
# with ContextTimer("plotting"):
# plot_projections(r, q)
# pylab.savefig('tree_%.3d.png' % i)
# print 't = %g\ttree_%.3d.png' % (it, i)
print "%d/%d t = %g" % (i, len(t), it)
branch_prob = BRANCHING_PROBABILITY
if SINGLE_BRANCHING_TIME > 0:
if it > SINGLE_BRANCHING_TIME:
if not branched:
branch_prob = inf
branched = True
if SINGLE_BRANCHING_Z != 0 and not branched:
zterm = r[tr.terminals()[0], Z]
if zterm < SINGLE_BRANCHING_Z:
if not branched:
branch_prob = inf
branched = True
r, q = adapt_step(tr, r, q, dt, p=branch_prob)
with ContextTimer("saving %d" % i):
phi = solve_phi(r, q)
dfile.add_step(it, tr, r, q, phi,
error=error, error_dq=error_dq)
if END_WITH_RECONNECTION and tr.reconnects(r):
print "Finishing due to a reconnection."
break
def main():
""" This is the main function of the code it is the starting point of
a simulation. """
# Load input parameters from the input file and add the, in allcaps
# to the global namespace.
global EXTERNAL_FIELD_VECTOR, ELECTRODE
parameters = load_input(sys.argv[1], param_descriptors)
globals().update(
dict((key.upper(), item) for key, item in parameters.iteritems()))
if RANDOM_SEED >= 0:
seed(RANDOM_SEED)
EXTERNAL_FIELD_VECTOR = array([0.0, 0.0, EXTERNAL_FIELD])
ELECTRODE = init_electrode()
# init a tree from scratch
tr, r0, q0 = init_from_scratch(INITIAL_NODES)
dt = TIME_STEP
t = r_[0:END_TIME:dt]
r, q = r0, q0
dfile = DataFile(OUT_FILE, parameters=parameters)
branched = False
for i, it in enumerate(t):
# with ContextTimer("plotting"):
# plot_projections(r, q)
# pylab.savefig('tree_%.3d.png' % i)
# print 't = %g\ttree_%.3d.png' % (it, i)
print "%d/%d t = %g" % (i, len(t), it)
branch_prob = BRANCHING_PROBABILITY
if SINGLE_BRANCHING_TIME > 0:
if it > SINGLE_BRANCHING_TIME:
if not branched:
branch_prob = inf
branched = True
if SINGLE_BRANCHING_Z != 0 and not branched:
zterm = r[tr.terminals()[0], Z]
if zterm < SINGLE_BRANCHING_Z:
if not branched:
branch_prob = inf
branched = True
r, q = adapt_step(tr, r, q, dt, p=branch_prob)
with ContextTimer("saving %d" % i):
phi = solve_phi(r, q)
dfile.add_step(it, tr, r, q, phi, error=error, error_dq=error_dq)
if END_WITH_RECONNECTION and tr.reconnects(r):
print "Finishing due to a reconnection."
break
def latest_commit(self):
commit = DataFile(self.dir + '.git\\LatestCommit')
if not commit['commit']:
commit['commit'] = {'sha': '', 'etag': ''}
etag = commit['commit']['etag']
sha = commit['commit']['sha']
resp = self.get('git/refs/heads/' + self.branch, etag)
if resp.hasdata():
if not resp.is_json():
raise TypeError('Response not in JSON')
sha = resp.data['object']['sha']
commit.merge('commit', {'etag': resp.etag, 'sha': sha})
return sha
def latest_commit(self):
commit=DataFile(self.dir+'.git\\LatestCommit')
if not commit['commit']:
commit['commit']={'sha':'','etag':''}
etag=commit['commit']['etag']
sha=commit['commit']['sha']
resp=self.get('git/refs/heads/'+self.branch,etag)
if resp.hasdata():
if not resp.is_json():
raise TypeError('Response not in JSON')
sha=resp.data['object']['sha']
commit.merge('commit',{'etag':resp.etag,'sha':sha})
return sha
def write_stats(ranges, results, prefix="stats"):
attrs = list(ranges.keys())
df = DataFile(os.path.join(base_path, '%s_%s.dat' % (prefix, '_'.join(attrs))), ['entries', 'lookups', 'hits', 'total time', 'lookup time', 'scans'] + attr_names("iter time") + attr_names("iter thruput") + attr_names("file size") + attr_names("scan thruput") + attrs, overwrite=True)
for config, reps in results:
cfg = dict(config)
# TODO: avg. over the reps
iter_times = Entry("iter time")
iter_tps = Entry("iter thruput")
file_sizes = Entry("file size")
scan_tps = Entry("scan thruput")
for data in reps:
db_path, entries, lookups, hits, total_time, num_scans, scans_time, lookup_time, iter_time, iter_tp = data.split(", ")
iter_times.add(int(iter_time))
iter_tps.add(int(iter_tp))
file_sizes.add(int(os.path.getsize(db_path)))
scan_tps.add(float(scans_time)/float(num_scans))
for attr in ranges:
df[attr] = cfg[attr]
df['entries'] = entries
df['lookups'] = lookups
df['hits'] = hits
df['total time'] = total_time
df['lookup time'] = lookup_time
df['scans'] = num_scans
# df['file size'] = os.path.getsize(db_path)
for attr in attr_names("iter time"):
df[attr] = getattr(iter_times, attr.split()[-1])
for attr in attr_names("iter thruput"):
df[attr] = getattr(iter_tps, attr.split()[-1])
for attr in attr_names("file size"):
df[attr] = getattr(file_sizes, attr.split()[-1])
for attr in attr_names("scan thruput"):
df[attr] = getattr(scan_tps, attr.split()[-1])
print file_sizes
print iter_times
print iter_tps
print scan_tps
df.save()
df.close()
def read_file(self):
if set(self.files_list) == set(self.bad_files):
dialogs.report_no_files()
exit(3)
current = self.files_list[self.current_file]
if current in self.bad_files:
self.next_file()
self.read_file()
return
try:
self.datafile = DataFile(current, self.config["labels"])
except (UnrecognizedFormatError, BadFileError):
self.datafile = None
self.bad_files.append(current)
dialogs.notify_read_error(os.path.basename(current))
self.next_file()
self.read_file()
def __init__(self, name, options):
self.name = name
self.money = 0
self.lastwinMoney = 0
self.winCount = 0
self.playCount = 0
self.continuousLoss = 0 #投注策略用到
self.continuousWin = 0 #投注策略用到
self.originBetting = options.betting
self.currentBetting = options.betting
self.gameCount = options.gameCount
self.odds = options.odds
self.maxBetting = options.maxBetting
self.totalNumber = options.totalNumber
self.numberBetting = options.numberBetting
self.probability = self.probabilityOfWinningEachPeriod()
stime = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S-%f')
self.datafile = DataFile(r"E:\Git\turkeyGame\var\sample-" + stime +
"-" + str(name) + ".xlsx")
def exec_ranges(cmd, args, range_attrs, num_reps=10, pbs_queue=None, pbs_name='storagesim', config=None):
# range_attrs is in the form:
# range_val: [list of possible values]
#
# when several range_vals and value lists are given, all possible
# permutations are executed
base_path = args['base_dir']
if config != None:
df = DataFile(os.path.join(config, 'config_ranges.dat'), list(range_attrs.keys()), overwrite=True)
outputs = []
for attr_p in permutations(range_attrs):
# set the attrs accordingly for this exec
names = []
for attr, val in attr_p:
args[attr] = val
names.append('%s_%s' % (str(attr), str(val)))
if config != None:
df[attr] = val
if config != None:
df.save()
args['base_dir'] = os.path.join(base_path, '%s' % ('_'.join(names)))
if not os.path.exists(args['base_dir']):
os.mkdir(args['base_dir'])
outputs.append((attr_p, exec_repetitions(cmd, args, num_reps=num_reps, pbs_queue=pbs_queue, pbs_name=pbs_name)))
if config != None:
df.close()
return outputs
def write_stats(ranges, results, prefix="stats"):
attrs = list(ranges.keys())
df = DataFile(
os.path.join(base_path, '%s_%s.dat' % (prefix, '_'.join(attrs))),
['entries', 'lookups', 'hits', 'total time', 'lookup time', 'scans'] +
attr_names("iter time") + attr_names("iter thruput") +
attr_names("file size") + attr_names("scan thruput") + attrs,
overwrite=True)
for config, reps in results:
cfg = dict(config)
# TODO: avg. over the reps
iter_times = Entry("iter time")
iter_tps = Entry("iter thruput")
file_sizes = Entry("file size")
scan_tps = Entry("scan thruput")
for data in reps:
db_path, entries, lookups, hits, total_time, num_scans, scans_time, lookup_time, iter_time, iter_tp = data.split(
", ")
iter_times.add(int(iter_time))
iter_tps.add(int(iter_tp))
file_sizes.add(int(os.path.getsize(db_path)))
scan_tps.add(float(scans_time) / float(num_scans))
for attr in ranges:
df[attr] = cfg[attr]
df['entries'] = entries
df['lookups'] = lookups
df['hits'] = hits
df['total time'] = total_time
df['lookup time'] = lookup_time
df['scans'] = num_scans
# df['file size'] = os.path.getsize(db_path)
for attr in attr_names("iter time"):
df[attr] = getattr(iter_times, attr.split()[-1])
for attr in attr_names("iter thruput"):
df[attr] = getattr(iter_tps, attr.split()[-1])
for attr in attr_names("file size"):
df[attr] = getattr(file_sizes, attr.split()[-1])
for attr in attr_names("scan thruput"):
df[attr] = getattr(scan_tps, attr.split()[-1])
print file_sizes
print iter_times
print iter_tps
print scan_tps
df.save()
df.close()
def get_inv_status(path):
"""
Updates dictionary where invoice number is a key and value is a string that represents actual status
of invoice
:param path: path to excel file that contains list of invoices to be checked
:return: updates REQUESTED_INVOICE_STATUSES dictionary with data query
"""
for invoice in DataFile.load_invoices(file_path=path):
if invoice in GFIS_DATA.keys() and not None:
REQUESTED_INVOICE_STATUSES[invoice] = \
f'Scheduled due {GFIS_DATA[invoice][0]}, payment: {GFIS_DATA[invoice][1]}'
elif invoice in COMBINED_DATA.keys():
REQUESTED_INVOICE_STATUSES[invoice] = STATUS_CODES[str(
COMBINED_DATA[invoice])]
elif invoice not in COMBINED_DATA.keys():
REQUESTED_INVOICE_STATUSES[invoice] = 'Missing'
def exec_ranges(cmd,
args,
range_attrs,
num_reps=10,
pbs_queue=None,
pbs_name='storagesim',
config=None):
# range_attrs is in the form:
# range_val: [list of possible values]
#
# when several range_vals and value lists are given, all possible
# permutations are executed
base_path = args['base_dir']
if config != None:
df = DataFile(os.path.join(config, 'config_ranges.dat'),
list(range_attrs.keys()),
overwrite=True)
outputs = []
for attr_p in permutations(range_attrs):
# set the attrs accordingly for this exec
names = []
for attr, val in attr_p:
args[attr] = val
names.append('%s_%s' % (str(attr), str(val)))
if config != None:
df[attr] = val
if config != None:
df.save()
args['base_dir'] = os.path.join(base_path, '%s' % ('_'.join(names)))
if not os.path.exists(args['base_dir']):
os.mkdir(args['base_dir'])
outputs.append((attr_p,
exec_repetitions(cmd,
args,
num_reps=num_reps,
pbs_queue=pbs_queue,
pbs_name=pbs_name)))
if config != None:
df.close()
return outputs
from threading import Thread
from datafile import DataFile
from crawl import Crawl
import sys
import os
import time
import json
# 任务队列，从主线程到工作线程
task_queue = Queue(maxsize=100000)
# 响应队列，从工作线程到主线程
response_queue = Queue()
# 数据存储
data_file = DataFile()
# 用户信息获取
crawl = Crawl()
# 工作线程数量
threads_num = 100
class MasterThread(Thread):
def __init__(self):
super(MasterThread, self).__init__()
self.count = {
'crawled_count': 0, # 已爬用户数量
'task_count': 0, # 任务数量
def read_and_save_flash(self, filename, with_bootloader):
_df = DataFile([])
_df.data = self.read_all_flash(with_bootloader)
print 'Read', len(_df.data), 'bytes'
_df.save(filename)
s = ''
i = 0
for v in lst:
vs = hex(v)[2:]
i += 1
if len(vs) == 1:
vs = '0' + vs
s += vs + ' '
if (i % 16) == 0:
print s
s = ''
#fw = DataFile('/Users/trol/Projects/radio/avr-lcd-module-128x128/build/avr-lcd-module-128x128.hex')
fw = DataFile(
'/Users/trol/Projects/radio/avr-ic-tester-v2/firmware/tester/build/ic-tester-main.hex'
)
# read 230400 44.0383300884 us/byte
#write 234000 256.21552423 us/byte
# 255.434597666 us/byte
#l = Loader('/dev/tty.wchusbserial14230', 57600)
#l = Loader('/dev/tty.wchusbserial14230', 230400)
l = Loader('/dev/tty.wchusbserial14220', 153600)
print l.dev.get_about()
l.read_and_save_flash('flash_with_loader.hex', True)
l.read_and_save_flash('flash_without_loader.hex', False)
l.write_flash(fw.data)
l.dev.start_app()
1 / 0
from functions import retrieve_gfis_data, data_from_combined, data_from_flow,\
get_inv_status, write_status
from datafile import DataFile
import glob
if __name__ == '__main__':
print('################### Basware/GFIS invoice status checking.##########################')
print('========================= Created by Dmytro Zimin =============================')
print('################################################################################\n\n')
print('***Please make sure you have read the instruction manual before using this program***')
run = input('To proceed press <y> or any other button to exit\n')
if run.lower()[0] == 'y':
print('combining basware *.csv files to excel ')
for file in glob.glob('basware/*.csv'):
basware_file = DataFile(file)
basware_file.combine_to_excel('basware', 'basware/combined.xlsx')
print('combining flow *.csv files to excel ')
for file in glob.glob('flow/*.csv'):
flow_file = DataFile(file)
flow_file.combine_to_excel('flow', 'flow/flow.xlsx')
print('retrieving data...')
retrieve_gfis_data('gfis/*.xlsx')
data_from_combined('basware/combined.xlsx')
data_from_flow('flow/flow.xlsx')
get_inv_status('check_invoices.xlsx')
print('writing statuses...')
write_status('check_invoices.xlsx')
print('deleting temporary files...')
DataFile.remove_temporary_files('basware/combined.xlsx')
DataFile.remove_temporary_files('flow/flow.xlsx')
input('Statuses have been added to check_invoices.xlsx! Press any key to exit')
class EnvSensor():
def __init__(self, config):
self.data = Data()
self.config = config
self.instrument_class = 'env_sensor'
self.name = config['instrument_name']
server_address = (config['interface_config']['host'],
config['interface_config']['port'])
self.iface = TCPInterface(server_address, self.data)
self.iface.start()
df_config = {
'base_path': config['datafile_config']['base_path'],
'instrument_class': self.instrument_class,
'instrument_name': self.name,
'write_freq': config['datafile_config']['write_freq'],
}
self.data_file = DataFile(df_config)
self.out_dt = ''
self.out_data = {}
#self.data = None
self.task_list = []
def start(self):
print("Starting EnvSensor")
#self.attempt_connect = True
task = asyncio.ensure_future(self.read_loop())
self.task_list.append(task)
self.is_running = True
def stop(self):
self.iface.stop()
#tasks = asyncio.Task.all_tasks()
for t in self.task_list:
# print('stop:')
# print(t)
t.cancel()
self.task_list.remove(t)
# tasks.remove(t)
self.is_running = False
self.attempt_connect = False
async def read_loop(self):
last_read = datetime.now()
# print(last_read)
while True:
# print(self.iface.has_data())
if (self.iface.has_data()):
# print('here')
buffer = self.iface.read()[0]
# print(buffer)
# print('here2')
dt = buffer['DateTime']
# print(dt)
#buf = buffer['Data']
# print(buf)
# print(buffer['Data']['RH']['value'])
# print(buffer['Data']['RH'])
buf = buffer['InstrumentData']
# print(buf)
meas_list = buf['METADATA']['MeasurementList']
# print(meas_list)
self.out_dt = dt
#dat = {}
out = ''
for meas in meas_list:
value = buf['DATA'][meas]['value']
units = buf['DATA'][meas]['units']
self.out_data[meas] = {
'value': buf['DATA'][meas]['value'],
'units': buf['DATA'][meas]['units'],
}
# print(meas)
# print(dat)
# print(self.out_data)
# print(type(value))
# print(units)
out += ' {}={:.2f}{}'.format(meas, value, units)
# print(out)
#t = buffer['Data']['Temperature']['value']
#rh = buffer['Data']['RH']['value']
# print(rh)
#print('[{}] -- {} : T={}C, RH={}%'.format(self.name,dt,t,rh))
#print('[{}] -- {} : {}'.format(self.name,dt,out))
self.data_file.append(buffer)
last_read = datetime.now()
# print(last_read)
#buffer = data.get()
#print('buffer : ' + buffer)
# print('read_buffer:')
# print(buffer)
delta = (datetime.now() - last_read).total_seconds()
# print(delta)
if delta > 5.0:
self.out_dt = ''
self.out_data = {}
# pass
await asyncio.sleep(0.1)
from customer import create_customer, print_header
from rooms import get_and_print_room_by_no, change_room_status
from datafile import DataFile
from datetime import datetime
NUMBER_OF_RECORDS_PER_PAGE = 10
customerTable = DataFile("customers.dat")
def add_customer():
rooms, found, position = get_and_print_room_by_no()
if found:
room_no = rooms[position].room_no
customers = customerTable.get_records()
if len(customers) == 0:
customer_id = 0
else:
customer_id = customers[len(customers) - 1].customer_id + 1
customer = create_customer(customer_id, room_no)
confirm = input("Complete the operation? (Y/N) ")
if confirm.lower() == 'y':
customerTable.add_record(customer)
change_room_status(room_no, False)
print("Operation Successful")
else:
print("Operation Canceled")
def get_and_print_customer_list_by_name():
customers = customerTable.get_records()
results = []
class ProjectData:
def __init__(self, project):
self.folder = os.path.dirname(project)
self.project_file = project
self.current_file = 0
self.current_label = 0
self.modified = False
self.bad_files = []
self.datafile = None
self.config = None
self.read_conf()
self.read_file()
def read(self):
if self.modified:
self.read_conf()
self.modified = False
self.read_file()
if self.current_label >= len(self.config["labels"]):
self.current_label = 0
def read_conf(self):
try:
with open(self.project_file) as in_file:
self.config = json.load(in_file)
except IOError:
logger.error("Unable to read {}: permission denied".format(
self.project_file))
exit(2)
def read_file(self):
if set(self.config["files"]) == set(self.bad_files):
dialogs.report_no_files()
exit(3)
current = self.config["files"][self.current_file]
file_path = os.path.join(self.folder, current)
if current in self.bad_files or not os.path.exists(file_path):
if not os.path.exists(file_path):
self.datafile = None
self.bad_files.append(current)
dialogs.notify_read_error(current)
self.next_file()
self.read_file()
return
try:
self.datafile = DataFile(file_path, self.config["labels"])
self.insert_header()
except (UnrecognizedFormatError, BadFileError):
self.datafile = None
self.bad_files.append(current)
dialogs.notify_read_error(current)
self.next_file()
self.read_file()
def insert_header(self):
header = self.datafile.get_data_header()
if str(header) not in self.config.keys():
self.config[str(header)] = {
"plot": [[i] for i in self.datafile.get_data_columns()],
"normalize": [],
"functions": []
}
self.modified = True
self.save_config()
def save_file(self):
self.datafile.save()
def save_config(self):
if self.modified:
try:
with open(self.project_file, 'w') as out_file:
json.dump(self.config, out_file)
except IOError:
logger.error("Unable to write {}: permission denied".format(
self.project_file))
exit(2)
self.modified = False
def next_label(self):
self.current_label = (self.current_label + 1) % len(
self.config["labels"])
def prev_label(self):
length = len(self.config["labels"])
self.current_label = (self.current_label - 1 + length) % length
def get_current_label(self):
label = self.config["labels"][self.current_label]
color = self.config["colors"][self.current_label]
return label, color
def get_label_color(self, label):
index = self.config["labels"].index(label)
return self.config["colors"][index]
def get_labels_info(self):
return self.config["labels"], self.config["colors"]
def set_labels_info(self, names, colors):
self.config["labels"] = names
self.config["colors"] = colors
self.modified = True
if self.current_label >= len(names):
self.current_label = 0
def get_plot_info(self):
header = self.datafile.get_data_header()
conf = self.config[str(header)]
return conf["plot"], conf["normalize"]
def set_plot_info(self, plot_set, normalize):
header = self.datafile.get_data_header()
conf = self.config[str(header)]
conf["plot"] = plot_set
conf["normalize"] = normalize
self.modified = True
def next_file(self):
self.current_file = (self.current_file + 1) % len(self.config["files"])
def prev_file(self):
self.current_file = (self.current_file - 1 + len(
self.config["files"])) % len(self.config["files"])
def get_functions(self):
header = self.datafile.get_data_header()
return self.config[str(header)]["functions"]
def add_function(self, fs):
header = self.datafile.get_data_header()
new_header = header + [fs.name]
self.config[str(new_header)] = self.config[str(header)]
self.config[str(new_header)]["functions"].append(fs.name)
self.datafile.add_function(fs)
self.modified = True
def remove_function(self, index):
header = self.datafile.get_data_header()
conf = self.config[str(header)]
f_name = conf["functions"][index]
f_col = self.datafile.get_data_columns()[header.index(f_name)]
for plot in conf["plot"]:
if f_col in plot:
plot.remove(f_col)
for i, col in enumerate(plot):
if col > f_col:
plot[i] = col - 1
self.datafile.remove_function(f_name)
del conf["functions"][index]
self.modified = True
class FilesData:
def __init__(self, files):
self.files_list = files
self.config_list = []
self.current_file = 0
self.current_label = 0
self.modified = False
self.bad_files = []
self.datafile = None
self.config = None
self.init()
self.read()
def init(self):
for file in self.files_list:
conf = file + ".json"
file_conf = conf if os.path.exists(conf) else None
self.config_list.append(file_conf)
# noinspection PyTypeChecker
def read(self):
conf_path = self.config_list[self.current_file]
if conf_path:
self.read_conf()
self.read_file()
else:
self.config = {
"labels": ["Label"],
"colors": ["#1f77b4"],
}
self.read_file()
self.config["plot"] = [[i]
for i in self.datafile.get_data_columns()]
self.config["normalize"] = []
self.config["functions"] = []
if self.current_label >= len(self.config["labels"]):
self.current_label = 0
self.modified = False
def read_conf(self):
conf_path = self.config_list[self.current_file]
try:
with open(conf_path) as in_file:
self.config = json.load(in_file)
except IOError:
logger.error(
"Unable to read {}: permission denied".format(conf_path))
exit(2)
def read_file(self):
if set(self.files_list) == set(self.bad_files):
dialogs.report_no_files()
exit(3)
current = self.files_list[self.current_file]
if current in self.bad_files:
self.next_file()
self.read_file()
return
try:
self.datafile = DataFile(current, self.config["labels"])
except (UnrecognizedFormatError, BadFileError):
self.datafile = None
self.bad_files.append(current)
dialogs.notify_read_error(os.path.basename(current))
self.next_file()
self.read_file()
def save_file(self):
self.datafile.save()
def save_config(self):
if self.modified:
conf_path = self.files_list[self.current_file] + ".json"
try:
with open(conf_path, 'w') as out_file:
json.dump(self.config, out_file)
except IOError:
logger.error(
"Unable to write {}: permission denied".format(conf_path))
exit(2)
self.config_list[self.current_file] = conf_path
self.modified = False
def next_label(self):
self.current_label = (self.current_label + 1) % len(
self.config["labels"])
def prev_label(self):
length = len(self.config["labels"])
self.current_label = (self.current_label - 1 + length) % length
def get_current_label(self):
label = self.config["labels"][self.current_label]
color = self.config["colors"][self.current_label]
return label, color
def get_label_color(self, label):
index = self.config["labels"].index(label)
return self.config["colors"][index]
def get_labels_info(self):
return self.config["labels"], self.config["colors"]
def set_labels_info(self, names, colors):
self.config["labels"] = names
self.config["colors"] = colors
self.modified = True
if self.current_label >= len(names):
self.current_label = 0
def get_plot_info(self):
return self.config["plot"], self.config["normalize"]
def set_plot_info(self, plot_set, normalize):
self.config["plot"] = plot_set
self.config["normalize"] = normalize
self.modified = True
def next_file(self):
self.current_file = (self.current_file + 1) % len(self.files_list)
def prev_file(self):
self.current_file = (self.current_file - 1 +
len(self.files_list)) % len(self.files_list)
def get_functions(self):
return self.config["functions"]
def add_function(self, fs):
self.datafile.add_function(fs)
self.config["functions"].append(fs.name)
self.modified = True
def remove_function(self, index):
f_name = self.config["functions"][index]
header = self.datafile.get_data_header()
f_col = self.datafile.get_data_columns()[header.index(f_name)]
for plot in self.config["plot"]:
if f_col in plot:
plot.remove(f_col)
for i, col in enumerate(plot):
if col > f_col:
plot[i] = col - 1
self.datafile.remove_function(f_name)
del self.config["functions"][index]
self.modified = True
处理在线接单任务
"""
def recv_task_online(self,is_fast=True):
recv = ReceiveOnline()
recv.run(is_fast)
if( __name__ == "__main__"):
start_time = time.time()
#que = queue.Queue()
send_thread_list = []
#receive_thread_list = []
cmcc = Cmcc()
data_path = './data/data.txt'
path_pre = data_path[0:-4]
thread_count = 4
data = DataFile(data_path,thread_count)
data.run()
for i in range(0,thread_count):
send_thread = threading.Thread(target=cmcc.send_task,args=(path_pre+str(i)+".txt","./data/Send_Thread"+str(i)+".txt"))
send_thread.setDaemon(True)
send_thread.start()
send_thread_list.append(send_thread)
time.sleep(3)
#睡120秒再启动快线程
time.sleep(120)
thread = threading.Thread(target=cmcc.recv_task_online,args=(True,))
thread.setDaemon(True)
thread.start()
thread_slow = None
while True:
"""
def retrieve_gfis_data(path_wildcard):
"""
:param path_wildcard: path to directory that contains *.xlsx files
:return: updates the dictionary GFIS_DATA where invoice_number is a key, and
schedule_date, parsed_date, payments are values.
"""
for file in glob.glob(path_wildcard):
try:
DataFile.remove_row(file)
gfis_excel = load_workbook(filename=f'{file}')
gfis_sheet = gfis_excel.active
except FileNotFoundError:
print('File *.xlsx in <gfis> not found')
else:
invoice_numbers = [
str(invoice[0])
for invoice in gfis_sheet.iter_rows(min_row=MIN_ROW,
min_col=GFIS_INVOICE_COL,
max_col=GFIS_INVOICE_COL,
values_only=True)
]
schedule_dates = [
datetime.strftime(schedule[0], '%Y-%m-%d')
if schedule[0] else 'no data'
for schedule in gfis_sheet.iter_rows(min_row=MIN_ROW,
min_col=GFIS_SCHEDULE_COL,
max_col=GFIS_SCHEDULE_COL,
values_only=True)
]
spread_payment_dates = [
payment_date[0] for payment_date in gfis_sheet.iter_rows(
min_row=MIN_ROW,
min_col=last_column(file),
max_col=last_column(file),
values_only=True)
]
parsed_payments_dates = [
datetime.strftime(date, '%Y-%m-%d')
if date is not None else 'NOT PAID'
for date in spread_payment_dates
]
payments = [
payment[0]
for payment in gfis_sheet.iter_rows(min_row=MIN_ROW,
min_col=GFIS_PAYMENT_COL,
max_col=GFIS_PAYMENT_COL,
values_only=True)
]
for invoice, schedule_date, payment_date, payment in zip(
invoice_numbers, schedule_dates, parsed_payments_dates,
payments):
if invoice not in GFIS_DATA.keys():
GFIS_DATA[invoice] = schedule_date, payment_date, payment
elif payment > GFIS_DATA.get(invoice)[2]:
GFIS_DATA[invoice] = schedule_date, payment_date, payment
from crawl import Crawl
from datafile import DataFile
__author__ = """\
/\/\ ___ _ __ __ _ _ __
/ \ / _ \| '__/ _` | '_ \
/ /\/\ \ (_) | | | (_| | | | |
\/ \/\___/|_| \__,_|_| |_|"""
# 任务队列，从主线程到工作线程
task_queue = Queue(maxsize=100000)
# 响应队列，从工作线程到主线程
response_queue = Queue()
# 数据文件操作接口
df = DataFile()
# 用户信息获取接口
crawl = Crawl()
# 工作线程的数量
threads_numbers = 20
class MasterThread(Thread):
"""
主线程
Attributes:
count: 状态信息，用于实时显示爬虫状态
crawled_set: 已爬取用户集合，用于去除重复用户
task_set: 待爬取用户集合，元素与任务队列保持一致，用于去除重复用户
def load(self, directory):
assert (os.path.isdir(directory))
self.reset()
self.directory_ = directory
dataFile = DataFile()
print "-- loading data from '" + self.getName() + "'"
# load coverage events
filename = os.path.join(self.directory_, COVERAGE_EVENTS_FILE)
dataFile.load(filename)
self.coverageEvents_ = dataFile.getDataAs("fl")
# load time events
filename = os.path.join(self.directory_, TIME_EVENTS_FILE)
dataFile.load(filename)
self.timeEvents_ = dataFile.getDataAs("fl")
# load experiment config
filename = os.path.join(self.directory_, EXPERIMENT_CONFIG_FILE)
dataFile.load(filename)
self.robotCount_ = int(dataFile.data_[0][0])
self.worldType_ = dataFile.data_[0][1]
# load final coverage
filename = os.path.join(self.directory_, FINAL_COVERAGE_FILE)
dataFile.load(filename)
self.finalCoverage_ = dataFile.getDataAs("fl")
# load meanTileTimeBetweenVisits
filename = os.path.join(self.directory_,
MEAN_TILE_TIME_BEWTEEN_VISITS_FILE)
dataFile.load(filename)
self.meanTileTimeBetweenVisits_ = dataFile.getDataAs("iil")
# load tileVisits
filename = os.path.join(self.directory_, TILE_VISITS_FILE)
dataFile.load(filename)
self.tileVisits_ = dataFile.getDataAs("iii")
# load visitsEvents
filename = os.path.join(self.directory_, VISITS_EVENTS_FILE)
dataFile.load(filename)
self.visitsEvents_ = dataFile.getDataAs("fff")
# load timeBetweenVisitsEvents
filename = os.path.join(self.directory_,
TIME_BETWEEN_VISITS_EVENTS_FILE)
dataFile.load(filename)
self.timeBetweenVisitsEvents_ = dataFile.getDataAs("fll")
import os
import time
from queue import Queue
from threading import Thread
from bs4 import BeautifulSoup
import requests
from datafile import DataFile
from login import Login
from crawler import Crawler
task_queue = Queue()
response_queue = Queue()
data = DataFile()
thread_numbers = 3
class MasterThread(Thread):
def __init__(self):
Thread.__init__(self)
crawled_list = data.load_crawed_list()
self.crawled_set = set(crawled_list)
un_crawled_user_list = data.load_uncrawled_user_list(self.crawled_set)
self.task_set = set()
for user_url in un_crawled_user_list:
task_queue.put(user_url)
self.task_set.add(user_url)
self.temp_user_infos = []
def load(self, directory):
assert(os.path.isdir(directory))
self.reset()
self.directory_ = directory
dataFile = DataFile()
print "-- loading data from '" + self.getName() + "'"
# load coverage events
filename = os.path.join(self.directory_, COVERAGE_EVENTS_FILE)
dataFile.load(filename)
self.coverageEvents_= dataFile.getDataAs("fl")
# load time events
filename = os.path.join(self.directory_, TIME_EVENTS_FILE)
dataFile.load(filename)
self.timeEvents_= dataFile.getDataAs("fl")
# load experiment config
filename = os.path.join(self.directory_, EXPERIMENT_CONFIG_FILE)
dataFile.load(filename)
self.robotCount_ = int(dataFile.data_[0][0]);
self.worldType_ = dataFile.data_[0][1];
# load final coverage
filename = os.path.join(self.directory_, FINAL_COVERAGE_FILE)
dataFile.load(filename)
self.finalCoverage_ = dataFile.getDataAs("fl")
# load meanTileTimeBetweenVisits
filename = os.path.join(self.directory_, MEAN_TILE_TIME_BEWTEEN_VISITS_FILE)
dataFile.load(filename)
self.meanTileTimeBetweenVisits_ = dataFile.getDataAs("iil")
# load tileVisits
filename = os.path.join(self.directory_, TILE_VISITS_FILE)
dataFile.load(filename)
self.tileVisits_ = dataFile.getDataAs("iii")
# load visitsEvents
filename = os.path.join(self.directory_, VISITS_EVENTS_FILE)
dataFile.load(filename)
self.visitsEvents_ = dataFile.getDataAs("fff")
# load timeBetweenVisitsEvents
filename = os.path.join(self.directory_, TIME_BETWEEN_VISITS_EVENTS_FILE)
dataFile.load(filename)
self.timeBetweenVisitsEvents_ = dataFile.getDataAs("fll")
def save(self):
print "saving summary of '" + self.getName() + "'..."
summaryDir = self.getSummaryDir()
if os.path.isdir(summaryDir):
shutil.rmtree(summaryDir, True, False)
mkdirRec(summaryDir)
dataFile = DataFile()
filename = ""
for worldType in AVAILABLE_WORLDS:
print "-- saving data of '" + worldType + "'"
worldDir = os.path.join(summaryDir,worldType)
mkdirRec(worldDir)
for robotCount in AVAILABLE_ROBOT_COUNTS:
if self.meanCoverageEvents_[worldType][robotCount].hasData():
filename = self.getSaveFilename(worldDir, COVERAGE_EVENTS_FILE, robotCount)
dataFile.reset()
dataFile.addComment("# [coverage meanTimestamp(usec)]")
data = self.meanCoverageEvents_[worldType][robotCount].getMean()
dataFile.setDataAs(data[:2], "fl")
dataFile.save(filename)
if self.meanTimeEvents_[worldType][robotCount].hasData():
filename = self.getSaveFilename(worldDir, TIME_EVENTS_FILE, robotCount)
dataFile.reset()
dataFile.addComment("# [coverage meanTimestamp(usec)]")
data = self.meanTimeEvents_[worldType][robotCount].getMean()
dataFile.setDataAs(data[:2], "fl")
dataFile.save(filename)
if self.meanTileTimeBetweenVisits_[worldType][robotCount].hasData():
filename = self.getSaveFilename(worldDir, MEAN_TILE_TIME_BEWTEEN_VISITS_FILE, robotCount)
dataFile.reset()
dataFile.addComment("# [x y meanTimeBetweenVisits(usec)]")
data = self.meanTileTimeBetweenVisits_[worldType][robotCount].getMeanGrid()
dataFile.setDataAs(data[:3], "iil")
dataFile.save(filename)
if self.meanTileVisits_[worldType][robotCount].hasData():
filename = self.getSaveFilename(worldDir, TILE_VISITS_FILE, robotCount)
dataFile.reset()
dataFile.addComment("# [x y meanVisitCount]")
data = self.meanTileVisits_[worldType][robotCount].getMeanGrid()
dataFile.setDataAs(data[:3], "iif")
dataFile.save(filename)
if self.meanFinalCoverage_[worldType][robotCount].hasData():
filename = self.getSaveFilename(worldDir, FINAL_COVERAGE_FILE, robotCount)
dataFile.reset()
dataFile.addComment("# [coverage timeStamp(usec)]")
data = self.meanFinalCoverage_[worldType][robotCount].getMean()
dataFile.setDataAs(data[:2], "fl")
dataFile.save(filename)
from datafile import DataFile
from room import create_room, print_header
NUMBER_OF_RECORDS_PER_PAGE = 10
roomTable = DataFile("rooms.dat")
def add_room():
rooms = roomTable.get_records()
if len(rooms) == 0:
room_id = 0
else:
room_id = rooms[len(rooms) - 1].room_id + 1
room = create_room(room_id)
roomTable.add_record(room)
print("Operation Successful")
def get_and_print_room_by_no():
rooms = roomTable.get_records()
found = False
position = -1
if len(rooms) == 0:
print("No Record found")
else:
room_no = int(input("Enter the room no: "))
for room in rooms:
position += 1
if room_no == room.room_no:
found = True
def _parse_data_file(self, binary_control_record, data_fp):
tf = DataFile(self.config)
tf.from_binary(binary_control_record, data_fp)
return tf
def run_files(filelist):
fig_s = plt.figure('pos_deltas')
ax_pos = fig_s.add_subplot(111)
fig_s.suptitle('delta_lat_lon')
ax_pos.set(ylabel='dlat (deg)')
ax_pos.set(xlabel='dlon (deg)')
ax_pos.grid()
ax_pos.axis('equal')
ax_pos.set_ylim(-5e-5, 5e-5)
ax_pos.set_xlim(-5e-5, 5e-5)
fig_en = plt.figure('enu_scatter')
ax_en = fig_en.add_subplot(111)
fig_en.suptitle('enu_scatter')
ax_en.set(ylabel='dNorth (m)')
ax_en.set(xlabel='dEast (m)')
ax_en.grid()
ax_en.axis('equal')
ax_en.set_ylim(-10, 10)
ax_en.set_xlim(-10, 10)
fig_p = plt.figure('delta_pos_enu')
ax_e = fig_p.add_subplot(311)
fig_p.suptitle('delta_pos_enu')
ax_e.set(ylabel='dE (m)')
ax_e.set(xlabel='Time (s)')
ax_e.grid()
ax_e.set_ylim(-10, 10)
ax_n = fig_p.add_subplot(312)
ax_n.set(ylabel='dN (m)')
ax_n.set(xlabel='Time (s)')
ax_n.grid()
ax_n.set_ylim(-10, 10)
ax_u = fig_p.add_subplot(313)
ax_u.set(ylabel='dU (m)')
ax_u.set(xlabel='Time (s)')
ax_u.grid()
ax_u.set_ylim(-10, 10)
for f in filelist:
print('---- WORKING ' + f + ' ----')
fname = f.split('.')[0]
df = DataFile(f, data_dir='data/hw1/')
df.setup()
lat_mean = np.mean(df.file_df['latitude {deg}']) * u.degree
lon_mean = np.mean(df.file_df['longitude {deg}']) * u.degree
h_mean = np.mean(df.file_df['h_ellipsiod {m}']) * u.meter
sats_mean = np.mean(df.file_df['num_sats'])
dlats = df.file_df['latitude {deg}'] - lat_mean.value
dlons = df.file_df['longitude {deg}'] - lon_mean.value
dhs = df.file_df['h_ellipsiod {m}'] - h_mean.value
dhs_std = np.std(dhs)
lat_std = np.std(df.file_df['latitude {deg}'])
lon_std = np.std(df.file_df['longitude {deg}'])
CEP_latlon = 0.62 * lat_std + 0.56 * lon_std
res = ax_pos.scatter(dlons, dlats, s=10, label=fname)
color = res.get_facecolor()[0]
circle = plt.Circle((0, 0), CEP_latlon, fill=False, color=color)
ax_pos.add_artist(circle)
print('-- Means -- lat = {}, lon = {}, h_mean = {}, num_sats = {}'.
format(lat_mean, lon_mean, h_mean, sats_mean))
# compute ECEF position vector using mean LAT, LON, H
r_ecef_mean = wgs84.geodetic_point_in_ecef(lat_mean, lon_mean, h_mean)
rs_ecef = wgs84.geodetic_point_in_ecef_arr(
df.file_df['latitude {deg}'].to_numpy() * u.degree,
df.file_df['longitude {deg}'].to_numpy() * u.degree,
df.file_df['h_ellipsiod {m}'].to_numpy() * u.meter)
x_ecef_std = np.std(rs_ecef[:, 0])
y_ecef_std = np.std(rs_ecef[:, 1])
z_ecef_std = np.std(rs_ecef[:, 2])
SEP = 0.51 * (x_ecef_std + y_ecef_std + z_ecef_std)
print('-- SEP -- {} {}'.format(SEP, u.meter.to_string()))
print('-- dhs_std -- {} {}'.format(dhs_std, u.meter.to_string()))
print('-- CEP_latlon -- {} {}'.format(CEP_latlon,
u.degree.to_string()))
print('ECEF_from_MEAN_GEO = ', r_ecef_mean)
R_ecef_enu = wgs84.ecef_to_enu(lat_mean, lon_mean)
rs_enu = np.zeros((len(rs_ecef), 3))
for ii in range(len(rs_ecef)):
r_ecef = rs_ecef[ii, :]
r_ecef_row = r_ecef_mean.flatten()
dr_ecef = r_ecef - r_ecef_row.value
dr_enu = np.matmul(R_ecef_enu, dr_ecef)
rs_enu[ii, :] = dr_enu
ax_e.plot(df.time_s,
rs_enu[:, 0],
label='SEP={:.2f} {}'.format(SEP, u.meter.to_string()))
ax_n.plot(df.time_s, rs_enu[:, 1], label=f)
ax_u.plot(df.time_s, rs_enu[:, 2], label=fname)
ax_en.scatter(rs_enu[:, 0], rs_enu[:, 1], s=10, label=fname)
# newline
print("")
ax_pos.legend()
ax_e.legend()
ax_u.legend()
ax_en.legend()
class Turkey:
'这是一只好斗的小火鸡'
def __init__(self, name, options):
self.name = name
self.money = 0
self.lastwinMoney = 0
self.winCount = 0
self.playCount = 0
self.continuousLoss = 0 #投注策略用到
self.continuousWin = 0 #投注策略用到
self.originBetting = options.betting
self.currentBetting = options.betting
self.gameCount = options.gameCount
self.odds = options.odds
self.maxBetting = options.maxBetting
self.totalNumber = options.totalNumber
self.numberBetting = options.numberBetting
self.probability = self.probabilityOfWinningEachPeriod()
stime = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S-%f')
self.datafile = DataFile(r"E:\Git\turkeyGame\var\sample-" + stime +
"-" + str(name) + ".xlsx")
#子类继承
def probabilityOfWinningEachPeriod(self):
'每次中奖的概率：可由子类重写'
return self.numberBetting / self.totalNumber
def isWin(self):
'本期是否中奖,随机获取'
return random.randint(0, 100) < (self.probability * 100)
#策略接口
def setStrategy(self, strategy):
self.strategy = strategy
#止盈止损接口
def setPlayer(self, player):
self.player = player
def earnMeneyOnePeriod(self):
'本期是否赚钱。中则加钱，不中则扣钱'
if self.isWin():
self.winCount = self.winCount + 1
self.continuousWin = self.continuousWin + 1
self.continuousLoss = 0
makeMoney = self.currentBetting * self.odds - self.currentBetting * self.numberBetting
else:
self.continuousLoss = self.continuousLoss + 1
self.continuousWin = 0
makeMoney = -(self.currentBetting * self.numberBetting)
self.lastwinMoney = makeMoney
self.playCount = self.playCount + 1
self.money = self.money + makeMoney
def saveStateEachPeriod(self):
'记录每局输赢'
self.datafile.write2xlsx(numberPeriod=self.playCount,
money=self.money,
lastwinMoney=self.lastwinMoney,
betting=self.currentBetting,
probability=self.probability,
cost=self.currentBetting * self.numberBetting)
def startStrategy(self):
'启用策略，当某几期之后，亏钱时，增大赌注，默认是翻倍;'
self.strategy.doStrategy()
def playGameEachPeriod(self):
self.earnMeneyOnePeriod()
self.saveStateEachPeriod()
self.startStrategy()
def cleanGame(self):
self.datafile.save2xlsx()
self.datafile.close
def startGame(self):
self.player.play()
self.cleanGame()
return self.money
