def __init__(self, config):
self.data = Data()
self.config = config
self.instrument_class = 'env_sensor'
self.name = config['instrument_name']
server_address = (config['interface_config']['host'],
config['interface_config']['port'])
self.iface = TCPInterface(server_address, self.data)
self.iface.start()
df_config = {
'base_path': config['datafile_config']['base_path'],
'instrument_class': self.instrument_class,
'instrument_name': self.name,
'write_freq': config['datafile_config']['write_freq'],
}
self.data_file = DataFile(df_config)
self.out_dt = ''
self.out_data = {}
#self.data = None
self.task_list = []
def read_file(self):
if set(self.config["files"]) == set(self.bad_files):
dialogs.report_no_files()
exit(3)
current = self.config["files"][self.current_file]
file_path = os.path.join(self.folder, current)
if current in self.bad_files or not os.path.exists(file_path):
if not os.path.exists(file_path):
self.datafile = None
self.bad_files.append(current)
dialogs.notify_read_error(current)
self.next_file()
self.read_file()
return
try:
self.datafile = DataFile(file_path, self.config["labels"])
self.insert_header()
except (UnrecognizedFormatError, BadFileError):
self.datafile = None
self.bad_files.append(current)
dialogs.notify_read_error(current)
self.next_file()
self.read_file()
def main():
""" This is the main function of the code it is the starting point of
a simulation. """
# Load input parameters from the input file and add the, in allcaps
# to the global namespace.
global EXTERNAL_FIELD_VECTOR, ELECTRODE
parameters = load_input(sys.argv[1], param_descriptors)
globals().update(
dict((key.upper(), item) for key, item in parameters.iteritems()))
if RANDOM_SEED >= 0:
seed(RANDOM_SEED)
EXTERNAL_FIELD_VECTOR = array([0.0, 0.0, EXTERNAL_FIELD])
ELECTRODE = init_electrode()
# init a tree from scratch
tr, r0, q0 = init_from_scratch(INITIAL_NODES)
dt = TIME_STEP
t = r_[0:END_TIME:dt]
r, q = r0, q0
dfile = DataFile(OUT_FILE, parameters=parameters)
branched = False
for i, it in enumerate(t):
# with ContextTimer("plotting"):
# plot_projections(r, q)
# pylab.savefig('tree_%.3d.png' % i)
# print 't = %g\ttree_%.3d.png' % (it, i)
print "%d/%d t = %g" % (i, len(t), it)
branch_prob = BRANCHING_PROBABILITY
if SINGLE_BRANCHING_TIME > 0:
if it > SINGLE_BRANCHING_TIME:
if not branched:
branch_prob = inf
branched = True
if SINGLE_BRANCHING_Z != 0 and not branched:
zterm = r[tr.terminals()[0], Z]
if zterm < SINGLE_BRANCHING_Z:
if not branched:
branch_prob = inf
branched = True
r, q = adapt_step(tr, r, q, dt, p=branch_prob)
with ContextTimer("saving %d" % i):
phi = solve_phi(r, q)
dfile.add_step(it, tr, r, q, phi, error=error, error_dq=error_dq)
if END_WITH_RECONNECTION and tr.reconnects(r):
print "Finishing due to a reconnection."
break
def write_stats(ranges, results, prefix="stats"):
attrs = list(ranges.keys())
df = DataFile(
os.path.join(base_path, '%s_%s.dat' % (prefix, '_'.join(attrs))),
['entries', 'lookups', 'hits', 'total time', 'lookup time', 'scans'] +
attr_names("iter time") + attr_names("iter thruput") +
attr_names("file size") + attr_names("scan thruput") + attrs,
overwrite=True)
for config, reps in results:
cfg = dict(config)
# TODO: avg. over the reps
iter_times = Entry("iter time")
iter_tps = Entry("iter thruput")
file_sizes = Entry("file size")
scan_tps = Entry("scan thruput")
for data in reps:
db_path, entries, lookups, hits, total_time, num_scans, scans_time, lookup_time, iter_time, iter_tp = data.split(
", ")
iter_times.add(int(iter_time))
iter_tps.add(int(iter_tp))
file_sizes.add(int(os.path.getsize(db_path)))
scan_tps.add(float(scans_time) / float(num_scans))
for attr in ranges:
df[attr] = cfg[attr]
df['entries'] = entries
df['lookups'] = lookups
df['hits'] = hits
df['total time'] = total_time
df['lookup time'] = lookup_time
df['scans'] = num_scans
# df['file size'] = os.path.getsize(db_path)
for attr in attr_names("iter time"):
df[attr] = getattr(iter_times, attr.split()[-1])
for attr in attr_names("iter thruput"):
df[attr] = getattr(iter_tps, attr.split()[-1])
for attr in attr_names("file size"):
df[attr] = getattr(file_sizes, attr.split()[-1])
for attr in attr_names("scan thruput"):
df[attr] = getattr(scan_tps, attr.split()[-1])
print file_sizes
print iter_times
print iter_tps
print scan_tps
df.save()
df.close()
def save(self):
print "saving summary of '" + self.getName() + "'..."
summaryDir = self.getSummaryDir()
if os.path.isdir(summaryDir):
shutil.rmtree(summaryDir, True, False)
mkdirRec(summaryDir)
dataFile = DataFile()
filename = ""
for worldType in AVAILABLE_WORLDS:
print "-- saving data of '" + worldType + "'"
worldDir = os.path.join(summaryDir,worldType)
mkdirRec(worldDir)
for robotCount in AVAILABLE_ROBOT_COUNTS:
if self.meanCoverageEvents_[worldType][robotCount].hasData():
filename = self.getSaveFilename(worldDir, COVERAGE_EVENTS_FILE, robotCount)
dataFile.reset()
dataFile.addComment("# [coverage meanTimestamp(usec)]")
data = self.meanCoverageEvents_[worldType][robotCount].getMean()
dataFile.setDataAs(data[:2], "fl")
dataFile.save(filename)
if self.meanTimeEvents_[worldType][robotCount].hasData():
filename = self.getSaveFilename(worldDir, TIME_EVENTS_FILE, robotCount)
dataFile.reset()
dataFile.addComment("# [coverage meanTimestamp(usec)]")
data = self.meanTimeEvents_[worldType][robotCount].getMean()
dataFile.setDataAs(data[:2], "fl")
dataFile.save(filename)
if self.meanTileTimeBetweenVisits_[worldType][robotCount].hasData():
filename = self.getSaveFilename(worldDir, MEAN_TILE_TIME_BEWTEEN_VISITS_FILE, robotCount)
dataFile.reset()
dataFile.addComment("# [x y meanTimeBetweenVisits(usec)]")
data = self.meanTileTimeBetweenVisits_[worldType][robotCount].getMeanGrid()
dataFile.setDataAs(data[:3], "iil")
dataFile.save(filename)
if self.meanTileVisits_[worldType][robotCount].hasData():
filename = self.getSaveFilename(worldDir, TILE_VISITS_FILE, robotCount)
dataFile.reset()
dataFile.addComment("# [x y meanVisitCount]")
data = self.meanTileVisits_[worldType][robotCount].getMeanGrid()
dataFile.setDataAs(data[:3], "iif")
dataFile.save(filename)
if self.meanFinalCoverage_[worldType][robotCount].hasData():
filename = self.getSaveFilename(worldDir, FINAL_COVERAGE_FILE, robotCount)
dataFile.reset()
dataFile.addComment("# [coverage timeStamp(usec)]")
data = self.meanFinalCoverage_[worldType][robotCount].getMean()
dataFile.setDataAs(data[:2], "fl")
dataFile.save(filename)
def load(self, directory):
assert (os.path.isdir(directory))
self.reset()
self.directory_ = directory
dataFile = DataFile()
print "-- loading data from '" + self.getName() + "'"
# load coverage events
filename = os.path.join(self.directory_, COVERAGE_EVENTS_FILE)
dataFile.load(filename)
self.coverageEvents_ = dataFile.getDataAs("fl")
# load time events
filename = os.path.join(self.directory_, TIME_EVENTS_FILE)
dataFile.load(filename)
self.timeEvents_ = dataFile.getDataAs("fl")
# load experiment config
filename = os.path.join(self.directory_, EXPERIMENT_CONFIG_FILE)
dataFile.load(filename)
self.robotCount_ = int(dataFile.data_[0][0])
self.worldType_ = dataFile.data_[0][1]
# load final coverage
filename = os.path.join(self.directory_, FINAL_COVERAGE_FILE)
dataFile.load(filename)
self.finalCoverage_ = dataFile.getDataAs("fl")
# load meanTileTimeBetweenVisits
filename = os.path.join(self.directory_,
MEAN_TILE_TIME_BEWTEEN_VISITS_FILE)
dataFile.load(filename)
self.meanTileTimeBetweenVisits_ = dataFile.getDataAs("iil")
# load tileVisits
filename = os.path.join(self.directory_, TILE_VISITS_FILE)
dataFile.load(filename)
self.tileVisits_ = dataFile.getDataAs("iii")
# load visitsEvents
filename = os.path.join(self.directory_, VISITS_EVENTS_FILE)
dataFile.load(filename)
self.visitsEvents_ = dataFile.getDataAs("fff")
# load timeBetweenVisitsEvents
filename = os.path.join(self.directory_,
TIME_BETWEEN_VISITS_EVENTS_FILE)
dataFile.load(filename)
self.timeBetweenVisitsEvents_ = dataFile.getDataAs("fll")
def latest_commit(self):
commit = DataFile(self.dir + '.git\\LatestCommit')
if not commit['commit']:
commit['commit'] = {'sha': '', 'etag': ''}
etag = commit['commit']['etag']
sha = commit['commit']['sha']
resp = self.get('git/refs/heads/' + self.branch, etag)
if resp.hasdata():
if not resp.is_json():
raise TypeError('Response not in JSON')
sha = resp.data['object']['sha']
commit.merge('commit', {'etag': resp.etag, 'sha': sha})
return sha
def exec_ranges(cmd,
args,
range_attrs,
num_reps=10,
pbs_queue=None,
pbs_name='storagesim',
config=None):
# range_attrs is in the form:
# range_val: [list of possible values]
#
# when several range_vals and value lists are given, all possible
# permutations are executed
base_path = args['base_dir']
if config != None:
df = DataFile(os.path.join(config, 'config_ranges.dat'),
list(range_attrs.keys()),
overwrite=True)
outputs = []
for attr_p in permutations(range_attrs):
# set the attrs accordingly for this exec
names = []
for attr, val in attr_p:
args[attr] = val
names.append('%s_%s' % (str(attr), str(val)))
if config != None:
df[attr] = val
if config != None:
df.save()
args['base_dir'] = os.path.join(base_path, '%s' % ('_'.join(names)))
if not os.path.exists(args['base_dir']):
os.mkdir(args['base_dir'])
outputs.append((attr_p,
exec_repetitions(cmd,
args,
num_reps=num_reps,
pbs_queue=pbs_queue,
pbs_name=pbs_name)))
if config != None:
df.close()
return outputs
def read_file(self):
if set(self.files_list) == set(self.bad_files):
dialogs.report_no_files()
exit(3)
current = self.files_list[self.current_file]
if current in self.bad_files:
self.next_file()
self.read_file()
return
try:
self.datafile = DataFile(current, self.config["labels"])
except (UnrecognizedFormatError, BadFileError):
self.datafile = None
self.bad_files.append(current)
dialogs.notify_read_error(os.path.basename(current))
self.next_file()
self.read_file()
def __init__(self, name, options):
self.name = name
self.money = 0
self.lastwinMoney = 0
self.winCount = 0
self.playCount = 0
self.continuousLoss = 0 #投注策略用到
self.continuousWin = 0 #投注策略用到
self.originBetting = options.betting
self.currentBetting = options.betting
self.gameCount = options.gameCount
self.odds = options.odds
self.maxBetting = options.maxBetting
self.totalNumber = options.totalNumber
self.numberBetting = options.numberBetting
self.probability = self.probabilityOfWinningEachPeriod()
stime = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S-%f')
self.datafile = DataFile(r"E:\Git\turkeyGame\var\sample-" + stime +
"-" + str(name) + ".xlsx")
def run_files(filelist):
fig_s = plt.figure('pos_deltas')
ax_pos = fig_s.add_subplot(111)
fig_s.suptitle('delta_lat_lon')
ax_pos.set(ylabel='dlat (deg)')
ax_pos.set(xlabel='dlon (deg)')
ax_pos.grid()
ax_pos.axis('equal')
ax_pos.set_ylim(-5e-5, 5e-5)
ax_pos.set_xlim(-5e-5, 5e-5)
fig_en = plt.figure('enu_scatter')
ax_en = fig_en.add_subplot(111)
fig_en.suptitle('enu_scatter')
ax_en.set(ylabel='dNorth (m)')
ax_en.set(xlabel='dEast (m)')
ax_en.grid()
ax_en.axis('equal')
ax_en.set_ylim(-10, 10)
ax_en.set_xlim(-10, 10)
fig_p = plt.figure('delta_pos_enu')
ax_e = fig_p.add_subplot(311)
fig_p.suptitle('delta_pos_enu')
ax_e.set(ylabel='dE (m)')
ax_e.set(xlabel='Time (s)')
ax_e.grid()
ax_e.set_ylim(-10, 10)
ax_n = fig_p.add_subplot(312)
ax_n.set(ylabel='dN (m)')
ax_n.set(xlabel='Time (s)')
ax_n.grid()
ax_n.set_ylim(-10, 10)
ax_u = fig_p.add_subplot(313)
ax_u.set(ylabel='dU (m)')
ax_u.set(xlabel='Time (s)')
ax_u.grid()
ax_u.set_ylim(-10, 10)
for f in filelist:
print('---- WORKING ' + f + ' ----')
fname = f.split('.')[0]
df = DataFile(f, data_dir='data/hw1/')
df.setup()
lat_mean = np.mean(df.file_df['latitude {deg}']) * u.degree
lon_mean = np.mean(df.file_df['longitude {deg}']) * u.degree
h_mean = np.mean(df.file_df['h_ellipsiod {m}']) * u.meter
sats_mean = np.mean(df.file_df['num_sats'])
dlats = df.file_df['latitude {deg}'] - lat_mean.value
dlons = df.file_df['longitude {deg}'] - lon_mean.value
dhs = df.file_df['h_ellipsiod {m}'] - h_mean.value
dhs_std = np.std(dhs)
lat_std = np.std(df.file_df['latitude {deg}'])
lon_std = np.std(df.file_df['longitude {deg}'])
CEP_latlon = 0.62 * lat_std + 0.56 * lon_std
res = ax_pos.scatter(dlons, dlats, s=10, label=fname)
color = res.get_facecolor()[0]
circle = plt.Circle((0, 0), CEP_latlon, fill=False, color=color)
ax_pos.add_artist(circle)
print('-- Means -- lat = {}, lon = {}, h_mean = {}, num_sats = {}'.
format(lat_mean, lon_mean, h_mean, sats_mean))
# compute ECEF position vector using mean LAT, LON, H
r_ecef_mean = wgs84.geodetic_point_in_ecef(lat_mean, lon_mean, h_mean)
rs_ecef = wgs84.geodetic_point_in_ecef_arr(
df.file_df['latitude {deg}'].to_numpy() * u.degree,
df.file_df['longitude {deg}'].to_numpy() * u.degree,
df.file_df['h_ellipsiod {m}'].to_numpy() * u.meter)
x_ecef_std = np.std(rs_ecef[:, 0])
y_ecef_std = np.std(rs_ecef[:, 1])
z_ecef_std = np.std(rs_ecef[:, 2])
SEP = 0.51 * (x_ecef_std + y_ecef_std + z_ecef_std)
print('-- SEP -- {} {}'.format(SEP, u.meter.to_string()))
print('-- dhs_std -- {} {}'.format(dhs_std, u.meter.to_string()))
print('-- CEP_latlon -- {} {}'.format(CEP_latlon,
u.degree.to_string()))
print('ECEF_from_MEAN_GEO = ', r_ecef_mean)
R_ecef_enu = wgs84.ecef_to_enu(lat_mean, lon_mean)
rs_enu = np.zeros((len(rs_ecef), 3))
for ii in range(len(rs_ecef)):
r_ecef = rs_ecef[ii, :]
r_ecef_row = r_ecef_mean.flatten()
dr_ecef = r_ecef - r_ecef_row.value
dr_enu = np.matmul(R_ecef_enu, dr_ecef)
rs_enu[ii, :] = dr_enu
ax_e.plot(df.time_s,
rs_enu[:, 0],
label='SEP={:.2f} {}'.format(SEP, u.meter.to_string()))
ax_n.plot(df.time_s, rs_enu[:, 1], label=f)
ax_u.plot(df.time_s, rs_enu[:, 2], label=fname)
ax_en.scatter(rs_enu[:, 0], rs_enu[:, 1], s=10, label=fname)
# newline
print("")
ax_pos.legend()
ax_e.legend()
ax_u.legend()
ax_en.legend()
from datafile import DataFile
from room import create_room, print_header
NUMBER_OF_RECORDS_PER_PAGE = 10
roomTable = DataFile("rooms.dat")
def add_room():
rooms = roomTable.get_records()
if len(rooms) == 0:
room_id = 0
else:
room_id = rooms[len(rooms) - 1].room_id + 1
room = create_room(room_id)
roomTable.add_record(room)
print("Operation Successful")
def get_and_print_room_by_no():
rooms = roomTable.get_records()
found = False
position = -1
if len(rooms) == 0:
print("No Record found")
else:
room_no = int(input("Enter the room no: "))
for room in rooms:
position += 1
if room_no == room.room_no:
found = True
处理在线接单任务
"""
def recv_task_online(self,is_fast=True):
recv = ReceiveOnline()
recv.run(is_fast)
if( __name__ == "__main__"):
start_time = time.time()
#que = queue.Queue()
send_thread_list = []
#receive_thread_list = []
cmcc = Cmcc()
data_path = './data/data.txt'
path_pre = data_path[0:-4]
thread_count = 4
data = DataFile(data_path,thread_count)
data.run()
for i in range(0,thread_count):
send_thread = threading.Thread(target=cmcc.send_task,args=(path_pre+str(i)+".txt","./data/Send_Thread"+str(i)+".txt"))
send_thread.setDaemon(True)
send_thread.start()
send_thread_list.append(send_thread)
time.sleep(3)
#睡120秒再启动快线程
time.sleep(120)
thread = threading.Thread(target=cmcc.recv_task_online,args=(True,))
thread.setDaemon(True)
thread.start()
thread_slow = None
while True:
"""
def read_and_save_flash(self, filename, with_bootloader):
_df = DataFile([])
_df.data = self.read_all_flash(with_bootloader)
print 'Read', len(_df.data), 'bytes'
_df.save(filename)
from threading import Thread
from datafile import DataFile
from crawl import Crawl
import sys
import os
import time
import json
# 任务队列，从主线程到工作线程
task_queue = Queue(maxsize=100000)
# 响应队列，从工作线程到主线程
response_queue = Queue()
# 数据存储
data_file = DataFile()
# 用户信息获取
crawl = Crawl()
# 工作线程数量
threads_num = 100
class MasterThread(Thread):
def __init__(self):
super(MasterThread, self).__init__()
self.count = {
'crawled_count': 0, # 已爬用户数量
'task_count': 0, # 任务数量
s = ''
i = 0
for v in lst:
vs = hex(v)[2:]
i += 1
if len(vs) == 1:
vs = '0' + vs
s += vs + ' '
if (i % 16) == 0:
print s
s = ''
#fw = DataFile('/Users/trol/Projects/radio/avr-lcd-module-128x128/build/avr-lcd-module-128x128.hex')
fw = DataFile(
'/Users/trol/Projects/radio/avr-ic-tester-v2/firmware/tester/build/ic-tester-main.hex'
)
# read 230400 44.0383300884 us/byte
#write 234000 256.21552423 us/byte
# 255.434597666 us/byte
#l = Loader('/dev/tty.wchusbserial14230', 57600)
#l = Loader('/dev/tty.wchusbserial14230', 230400)
l = Loader('/dev/tty.wchusbserial14220', 153600)
print l.dev.get_about()
l.read_and_save_flash('flash_with_loader.hex', True)
l.read_and_save_flash('flash_without_loader.hex', False)
l.write_flash(fw.data)
l.dev.start_app()
1 / 0
from functions import retrieve_gfis_data, data_from_combined, data_from_flow,\
get_inv_status, write_status
from datafile import DataFile
import glob
if __name__ == '__main__':
print('################### Basware/GFIS invoice status checking.##########################')
print('========================= Created by Dmytro Zimin =============================')
print('################################################################################\n\n')
print('***Please make sure you have read the instruction manual before using this program***')
run = input('To proceed press <y> or any other button to exit\n')
if run.lower()[0] == 'y':
print('combining basware *.csv files to excel ')
for file in glob.glob('basware/*.csv'):
basware_file = DataFile(file)
basware_file.combine_to_excel('basware', 'basware/combined.xlsx')
print('combining flow *.csv files to excel ')
for file in glob.glob('flow/*.csv'):
flow_file = DataFile(file)
flow_file.combine_to_excel('flow', 'flow/flow.xlsx')
print('retrieving data...')
retrieve_gfis_data('gfis/*.xlsx')
data_from_combined('basware/combined.xlsx')
data_from_flow('flow/flow.xlsx')
get_inv_status('check_invoices.xlsx')
print('writing statuses...')
write_status('check_invoices.xlsx')
print('deleting temporary files...')
DataFile.remove_temporary_files('basware/combined.xlsx')
DataFile.remove_temporary_files('flow/flow.xlsx')
input('Statuses have been added to check_invoices.xlsx! Press any key to exit')
from customer import create_customer, print_header
from rooms import get_and_print_room_by_no, change_room_status
from datafile import DataFile
from datetime import datetime
NUMBER_OF_RECORDS_PER_PAGE = 10
customerTable = DataFile("customers.dat")
def add_customer():
rooms, found, position = get_and_print_room_by_no()
if found:
room_no = rooms[position].room_no
customers = customerTable.get_records()
if len(customers) == 0:
customer_id = 0
else:
customer_id = customers[len(customers) - 1].customer_id + 1
customer = create_customer(customer_id, room_no)
confirm = input("Complete the operation? (Y/N) ")
if confirm.lower() == 'y':
customerTable.add_record(customer)
change_room_status(room_no, False)
print("Operation Successful")
else:
print("Operation Canceled")
def get_and_print_customer_list_by_name():
customers = customerTable.get_records()
results = []
from crawl import Crawl
from datafile import DataFile
__author__ = """\
/\/\ ___ _ __ __ _ _ __
/ \ / _ \| '__/ _` | '_ \
/ /\/\ \ (_) | | | (_| | | | |
\/ \/\___/|_| \__,_|_| |_|"""
# 任务队列，从主线程到工作线程
task_queue = Queue(maxsize=100000)
# 响应队列，从工作线程到主线程
response_queue = Queue()
# 数据文件操作接口
df = DataFile()
# 用户信息获取接口
crawl = Crawl()
# 工作线程的数量
threads_numbers = 20
class MasterThread(Thread):
"""
主线程
Attributes:
count: 状态信息，用于实时显示爬虫状态
crawled_set: 已爬取用户集合，用于去除重复用户
task_set: 待爬取用户集合，元素与任务队列保持一致，用于去除重复用户
import os
import time
from queue import Queue
from threading import Thread
from bs4 import BeautifulSoup
import requests
from datafile import DataFile
from login import Login
from crawler import Crawler
task_queue = Queue()
response_queue = Queue()
data = DataFile()
thread_numbers = 3
class MasterThread(Thread):
def __init__(self):
Thread.__init__(self)
crawled_list = data.load_crawed_list()
self.crawled_set = set(crawled_list)
un_crawled_user_list = data.load_uncrawled_user_list(self.crawled_set)
self.task_set = set()
for user_url in un_crawled_user_list:
task_queue.put(user_url)
self.task_set.add(user_url)
self.temp_user_infos = []
