def percentage_for_shape(self, shape):
if shape.state == 'fresh':
val = min(self.epoch - shape.state_stable_since,
conf()['shape_confirm_frames'])
return val / conf()['shape_confirm_frames']
else:
return 0
def config( self ):
"""
Create a config file with default values
"""
parser = argparse.ArgumentParser( description = 'Create a config file with default values',
formatter_class = argparse.ArgumentDefaultsHelpFormatter,
usage = '%(prog)s config' )
self.addConfigParserArgs( parser )
self.addLogParserArgs( parser )
args = self.parseAndMergeArgs( parser )
conf.conf( ).initConf( args.config )
print( 'Generated default configuration file: %s' % args.config )
def config(self):
"""
Create a config file with default values
"""
parser = argparse.ArgumentParser(
description='Create a config file with default values',
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
usage='%(prog)s config')
self.addConfigParserArgs(parser)
self.addLogParserArgs(parser)
args = self.parseAndMergeArgs(parser)
conf.conf().initConf(args.config)
print('Generated default configuration file: %s' % args.config)
def __init__(self):
cf = conf()
logging.basicConfig(
filename=cf.getConfObject("Logging", "infolog"),
format='%(asctime)s %(message)s',
level=logging.INFO)
logging.info('Logging initializes')
def __init__(self):
# load network configuration
self.conf_ = conf()
self.conf_.start()
# get train data
pre_proc = pre_processor()
train_x, train_y, val_x, val_y = pre_proc.get_data()
self.nn = nn(self.conf_)
# Build Computation Graph
self.nn.define_tensors()
self.nn.forward_prop()
self.nn.compute_loss()
self.nn.compute_acc()
self.nn.back_prop()
# initialize tensors
self.sess = tf.Session()
self.sess.run(tf.global_variables_initializer())
self.sess.run(tf.local_variables_initializer())
# start training
self.start(train_x, train_y, val_x, val_y)
def _config(self):
log = getLogger(__name__)
try:
configuration = conf()
self._conf = configuration.getdict('CELERY')
except RegistryError, e:
log.error(e)
def __init__(self, port):
util_root = '/works/demon_11st/utils'
sys.path.insert(0, util_root)
from demon_utils import demon_utils
app.demon_utils = demon_utils('/storage/enroll')
vsm_root = '/works/demon_11st/vsm'
sys.path.insert(0, vsm_root)
from vsm import vsm
#import pdb; pdb.set_trace()
image_sentence_filename = '/storage/attribute/PBrain_all.csv.image_sentence.txt'
app.vsm = vsm(image_sentence_filename)
agent_detector_root = '/works/demon_11st/agent/detection'
sys.path.insert(0, agent_detector_root)
import _init_paths
from conf import conf
from agent_detector import agent_detector
yaml_file = '/storage/product/detection/11st_All/cfg/faster_rcnn_end2end_test.yml'
conf = conf(yaml_file, 0)
app.agent_detector = agent_detector()
#import pdb; pdb.set_trace()
agent_attribute_root = '/works/demon_11st/agent/attribute'
sys.path.insert(0, agent_attribute_root)
from agent_attribute import agent_attribute
attribute_demon_host_ip = host_ip
attribute_demon_port= 8080
app.agent_attribute = agent_attribute( \
attribute_demon_host_ip, attribute_demon_port)
app.result_dic = init_result_dic()
web_server.__init__(self, app, port)
def nfs():
print "testing remote server..."
con = conf.conf()
ip = con.getStoreStyle()[1]
path = con.getStorePath()
# nettest(ip)
getRemoteFile(ip, path)
def __init__(self): # load network configuration self.conf_ = conf() self.conf_.start() # get test data pre_proc = pre_processor() test_x, test_y = pre_proc.get_data(testing=True) self.nn = nn(self.conf_) self.sess = tf.Session() # Build Computation Graph meta_filepath = "../models/sess-" + str(self.conf_.epochs) + ".meta" ckpt_filepath = "../models/sess-" + str(self.conf_.epochs) self.nn.restore_tensors(self.sess, meta_filepath, ckpt_filepath) self.nn.forward_prop(testing=True) self.nn.compute_loss() self.nn.compute_acc() # Initialize local variables only (not global variables) self.sess.run(tf.local_variables_initializer()) # start testing self.start(test_x, test_y)
def __init__(self):
self.conf_ = conf()
pre_processor_ = pre_processor()
train_x, train_y, val_x, val_y = pre_processor_.get_data()
# Build Computation Graph
self.cnn = cnn()
self.cnn.forward_prop()
self.cnn.compute_loss()
self.cnn.compute_acc()
self.cnn.back_prop()
# initialize tensors
self.sess = tf.Session()
self.sess.run(tf.global_variables_initializer())
self.sess.run(tf.local_variables_initializer())
logging.info("training...")
self.saver = tf.train.Saver(max_to_keep=1)
for epoch in range(self.cnn.epochs):
iters = int(len(train_x) / self.cnn.batch_size)
for iter_ in range(iters):
# extract batch data
batch_x = train_x[iter_ * self.cnn.batch_size:(iter_ + 1) *
self.cnn.batch_size, :]
batch_y = train_y[iter_ * self.cnn.batch_size:(iter_ + 1) *
self.cnn.batch_size, :]
# train
self.sess.run(self.cnn.tf_train,
feed_dict={
self.cnn.tf_X: batch_x,
self.cnn.tf_y: batch_y
})
# train loss
_, loss_train, acc_train = self.cnn.analyze_epoch(
train_x, train_y, self.sess)
# val loss
_, loss_val, acc_val = self.cnn.analyze_epoch(
val_x, val_y, self.sess)
# save model
_ = self.saver.save(self.sess,
"../models/sess",
global_step=int(epoch + 1))
# log the performance at each epoch
logging.info(
"epoch: {0}, loss_train: {1}, acc_train: {2}, loss_val: {3}, acc_val: {4}"
.format(epoch + 1, loss_train, acc_train, loss_val, acc_val))
self.sess.close()
def __init__(self):
self.db_cnf = configparser.ConfigParser()
self.cf = conf.conf()
self.db_cnf.read(self.cf.get_db_conf(), encoding="utf-8")
self.db = pymysql.connect(self.db_cnf.get('db', 'db_host'),
self.db_cnf.get('db', 'db_user'),
self.db_cnf.get('db', 'db_pwd'),
self.db_cnf.get('db', 'db_database'))
def __init__(self):
self.conf = conf()
self.db = {}
self.metric = {}
class free:
pass
self.conn = free()
self.get_dbinfo()
self.connect()
def __init__(self):
self.dbg = pydbg()
self.dbg.start_time = time.time()
self.cf = conf.conf()
#self.crasher = fuzzutil.crasher(self.cf)
self.InitPage = 'http://' + self.cf.host + ':' + self.cf.port + '/'
print '[maim] Server: ' + self.cf.host + ':' + self.cf.port
print '[main] Taregt: ' + self.cf.target
print '[main] Image: ' + self.cf.image
os.system('mkdir logs 2> null')
print '[main] LogPath: ./logs/'
def propagate_touchet_event(self, event, data):
try:
r = requests.get(conf()['event_sink_url'],
params={
'event': event,
**data
})
r.raise_for_status()
except requests.exceptions.RequestException as e:
print("An error occured while attempting to connect to NodeRed:")
print(e)
def createconnection(self):
## initalize a connection and return it
configobj = conf()
config = {
'user': configobj.dbusername,
'password': configobj.dbpass,
'host': configobj.dbhost,
'database': configobj.dbase,
'buffered': True
}
return mysql.connector.connect(**config) ## return the database object
def __init__(self):
self.dbg = pydbg()
self.dbg.start_time = time.time()
self.cf = conf.conf()
#self.crasher = fuzzutil.crasher(self.cf)
self.InitPage = 'http://'+self.cf.host+':'+self.cf.port+'/'
print '[maim] Server: ' + self.cf.host+':'+self.cf.port
print '[main] Taregt: ' + self.cf.target
print '[main] Image: ' + self.cf.image
os.system('mkdir logs 2> null')
print '[main] LogPath: ./logs/'
def test():
c=conf()
a=analysis(c.getVmlinux())
get_vmcore(c.getStoreStyle()[0])
# vmcore='/var/crash/127.0.0.1-2014.07.13-18\:23\:21/vmcore'
# vmlinux='/usr/lib/debug/lib/modules/3.9.5-301.fc19.i686.PAE/vmlinux'
vmlinux = c.getVmlinux()
an = analysis(vmlinux)
an.analysis()
an.log_wash()
async def grid_get(request: Request, citytag: str = ''):
client_ip = request.client.host
log_print(f'Request from {client_ip} for {citytag} grid')
# init conf
try:
cfg_file = os.path.join(os.environ['WORKSPACE'], 'slides', 'vars',
'conf', 'conf.json')
with open(cfg_file) as cin:
cfg = json.load(cin)
cw = conf(cfg)
except Exception as e:
log_print('conf generation failed : {}'.format(e))
raise HTTPException(
status_code=500,
detail='grid geojson conf init failed : {}'.format(e))
if citytag not in cw.cparams:
raise HTTPException(status_code=401,
detail='malformed url citytag {} not in {}'.format(
citytag, cw.cparams.keys()))
wdir = cw.wdir
try:
os.chdir(wdir)
except:
os.mkdir(wdir)
os.chdir(wdir)
with open(cw.cparams[citytag]) as tin:
simconf = json.load(tin)
confs = json.dumps(simconf)
with open(wdir + '/wsconf_grid_{}.json'.format(citytag), 'w') as outc:
json.dump(simconf, outc, indent=2)
#print(confs, flush=True)
s = simulation(confs)
if s.is_valid():
base = wdir + '/grid_{}'.format(citytag)
geojf = base + '.geojson'
if not os.path.exists(geojf):
s.dump_grid_geojson(geojf)
with open(geojf) as gin:
geoj = json.load(gin)
ret = {'message': 'geojson ok', 'type': 'grid', 'geojson': geoj}
else:
raise HTTPException(
status_code=501,
detail='grid geojson creation for \'{}\' failed'.format(citytag))
return ret
class Head(object):
configuration = conf("Settings.ini")
jaw = Servo(
channel = int(configuration.get_setting("Jaw","channel")),
frequency = int(configuration.get_setting("Jaw","frequency")),
minPulseLength = int(configuration.get_setting("Jaw","minPulseLength")),
maxPulseLength = int(configuration.get_setting("Jaw","maxPulseLength"))
)
left_antenna = Servo(
channel = int(configuration.get_setting("LeftAntenna","channel")),
frequency = int(configuration.get_setting("LeftAntenna","frequency")),
minPulseLength = int(configuration.get_setting("LeftAntenna","minPulseLength")),
maxPulseLength = int(configuration.get_setting("LeftAntenna","maxPulseLength"))
)
right_antenna = Servo(
channel = int(configuration.get_setting("RightAntenna","channel")),
frequency = int(configuration.get_setting("RightAntenna","frequency")),
minPulseLength = int(configuration.get_setting("RightAntenna","minPulseLength")),
maxPulseLength = int(configuration.get_setting("RightAntenna","maxPulseLength"))
)
left_brow_Vertical = Servo(
channel = int(configuration.get_setting("LeftBrowVertical","channel")),
frequency = int(configuration.get_setting("LeftBrowVertical","frequency")),
minPulseLength = int(configuration.get_setting("LeftBrowVertical","minPulseLength")),
maxPulseLength = int(configuration.get_setting("LeftBrowVertical","maxPulseLength"))
)
right_brow_vertical = Servo(
channel = int(configuration.get_setting("RightBrowVertical","channel")),
frequency = int(configuration.get_setting("RightBrowVertical","frequency")),
minPulseLength = int(configuration.get_setting("RightBrowVertical","minPulseLength")),
maxPulseLength = int(configuration.get_setting("RightBrowVertical","maxPulseLength"))
)
left_brow_horizontal = Servo(
channel = int(configuration.get_setting("LeftBrowHorizontal","channel")),
frequency = int(configuration.get_setting("LeftBrowHorizontal","frequency")),
minPulseLength = int(configuration.get_setting("LeftBrowHorizontal","minPulseLength")),
maxPulseLength = int(configuration.get_setting("LeftBrowHorizontal","maxPulseLength"))
)
right_brow_horizontal = Servo(
channel = int(configuration.get_setting("RightBrowHorizontal","channel")),
frequency = int(configuration.get_setting("RightBrowHorizontal","frequency")),
minPulseLength = int(configuration.get_setting("RightBrowHorizontal","minPulseLength")),
maxPulseLength = int(configuration.get_setting("RightBrowHorizontal","maxPulseLength"))
)
def menu(client):
while True:
print(Fore.WHITE, end="")
print(Back.GREEN, end="")
print("Live Stream Debugger 0.0.1 - Ericarthurc")
print(Style.RESET_ALL, end="")
print(Fore.GREEN, end="")
print("Main Menu:")
print("1. Update stream keys")
print("2. Restart streaming processes")
print("3. Exit program")
print(Style.RESET_ALL, end="")
val = input("Select option [1, 2, 3]: ")
if val == '1':
youtube = input("Input YouTube stream key: ")
facebook = input("Input Facebook stream key: ")
sftp = client.open_sftp()
sftp.chdir(path=r'/usr/local/nginx/conf')
client.exec_command(
'cp /usr/local/nginx/conf/nginx.conf /usr/local/nginx/conf/nginx.conf.BACKUP'
)
file = sftp.file('nginx.conf', 'w', 1)
file.write(conf(youtube, facebook))
file.flush()
print(Fore.CYAN, end="")
print_slow('Stream keys written to configuration file!', 0.06)
client.exec_command('/usr/local/nginx/sbin/nginx -s stop')
client.exec_command('/usr/local/nginx/sbin/nginx')
print('')
print_slow('Processes restarted!', 0.06)
print(Style.RESET_ALL, end="")
time.sleep(1)
os.system('cls')
elif val == '2':
client.exec_command('systemctl restart stunnel4')
client.exec_command('systemctl restart nginx')
client.exec_command('/usr/local/nginx/sbin/nginx -s stop')
client.exec_command('/usr/local/nginx/sbin/nginx')
print(Fore.CYAN, end="")
print_slow('Processes restarted!', 0.06)
print(Style.RESET_ALL, end="")
time.sleep(1)
os.system('cls')
elif val == '3':
print(Fore.RED, end="")
print_slow('Program closing...', 0.04)
print(Style.RESET_ALL, end="")
time.sleep(1)
sys.exit(0)
def Main():
logging.basicConfig(level=logging.INFO,
format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s',
datefmt='%a, %d %b %Y %H:%M:%S',
filename='robot.log',
filemode='w')
console = logging.StreamHandler()
console.setLevel(logging.INFO)
formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')
console.setFormatter(formatter)
logging.getLogger('').addHandler(console)
logging.info("SYSTEM START")
RobotCon = conf.conf()
RobotCon.LoadConf()
ser = SerialOpt.SerOpt(RobotCon.comName,RobotCon.comSpeed)
def __init__(self):
super().__init__(scrapy.Spider)
self.db_cnf = configparser.ConfigParser()
self.cf = conf.conf()
self.db_cnf.read(self.cf.get_db_conf(), encoding="utf-8")
self.db = pymysql.connect(self.db_cnf.get('db', 'db_host'),
self.db_cnf.get('db', 'db_user'),
self.db_cnf.get('db', 'db_pwd'),
self.db_cnf.get('db', 'db_database'))
self.cursor = self.db.cursor(cursor=pymysql.cursors.DictCursor)
# 设置所有需要爬去的商品ID
# self.start_urls = self.get_start_urls() #设置入口url
self.goods_id = self.getGoodsId()
# 设置商品类型
if len(self.goods_id) > 0:
self.get_goods_type(self.goods_id)
self.start_urls = [self.get_url()] #设置入口url
def __init__(self):
"""
Do argument setup / description, and execute subcommand
"""
cmdStr = ''
for cmd, desc in sorted(self.commands.items()):
cmdStr += ' %s%s\n' % (cmd.ljust(10, ' '), desc)
parser = argparse.ArgumentParser(prog=self.PROG_NAME,
description='',
usage='''%(prog)s <command> [<args>]
The most commonly used %(prog)s commands are:
''' + cmdStr)
parser.add_argument('command', help='Subcommand to run')
args = parser.parse_args(sys.argv[1:2])
if (not hasattr(self, args.command)) or (args.command
not in self.commands.keys()):
print('Unrecognized command')
parser.print_help()
exit(1)
# setup our default member vars
self.user = user.user()
self.processedUrls = {}
self.historian = historian.historian(self)
self.configParser = conf.conf().genDefault()
self.txtFile = False
self.skipHandling = False
self.browsers = {}
self.urllist = []
self.blacklist = []
self.whitelist = []
self.stats = {
'tick': datetime.now(),
'tock': None,
'visited': 0,
'skipped': 0,
'handled': 0
}
# use CLI command == function name, use it
getattr(self, args.command)()
def getRemoteFile(ip, remotedir):
"""
used to get back the file stored in nfs server
"""
print "retrieving remote server..."
c = conf.conf()
sto = c.getStorePath()
mountnfs(ip, remotedir)
cmd = "ls -l /mnt/ftest/" + sto + "|grep d[r-][w-][x-] |sort -n |tail -1|awk '{print $9}'"
status = commands.getstatusoutput(cmd)
if status[0] != 0:
raise commandFailException(cmd, status)
remVmcore = "/mnt/ftest/" + sto + "/" + status[1] + "/vmcore"
copyVmcore(remVmcore)
print "get remote file done"
def cnt_intersects_with_hand(self, cnt, include_secondary_contours=True):
if self.hand_cnt is None:
return False
# Create two empty frames, draw the contours and perform bitwise and
hand_img = np.zeros((realsensecam().H, realsensecam().W, 1), np.uint8)
cnt_img = hand_img.copy()
contours_to_check = [self.hand_cnt]
if include_secondary_contours:
contours_to_check += self.secondary_hand_cnts
cv2.drawContours(hand_img, contours_to_check, -1, 255, -1)
cv2.drawContours(hand_img, contours_to_check, -1, 255,
conf()['hand_shape_intersection_border'])
cv2.drawContours(cnt_img, [cnt], 0, 255, cv2.FILLED)
anded = np.bitwise_and(hand_img, cnt_img)
# If there are non-zero pixels left after "and"ing, there is an intersection
return np.any(anded)
def fungo_test_wrapper(name='cellcycle_FUN'):
X_train, X_test, train_ids, test_ids, id2doc, nodes = read_fungo(name)
X_train, X_test = np.array(X_train), np.array(X_test)
id2doc_train = id2doc
args = conf()
# id2doc_train = filter_ancestors(id2doc, nodes)
tree = Tree(args,
train_ids,
test_ids,
id2doc=id2doc_train,
id2doc_a=id2doc,
nodes=nodes,
rootname='Top')
mlb = MultiLabelBinarizer(classes=tree.class_idx)
Y_train = mlb.fit_transform(
[tree.id2doc_ancestors[docid]['class_idx'] for docid in train_ids])
Y_test = mlb.transform(
[tree.id2doc_ancestors[docid]['class_idx'] for docid in test_ids])
return X_train, Y_train, X_test, Y_test
def __init__( self ):
"""
Do argument setup / description, and execute subcommand
"""
cmdStr = ''
for cmd, desc in sorted( self.commands.items() ):
cmdStr += ' %s%s\n' % ( cmd.ljust( 10, ' ' ), desc )
parser = argparse.ArgumentParser( prog = self.PROG_NAME, description = '', usage = '''%(prog)s <command> [<args>]
The most commonly used %(prog)s commands are:
''' + cmdStr )
parser.add_argument( 'command', help = 'Subcommand to run' )
args = parser.parse_args( sys.argv[ 1:2 ] )
if ( not hasattr( self, args.command ) ) or ( args.command not in self.commands.keys() ):
print( 'Unrecognized command' )
parser.print_help( )
exit( 1 )
# setup our default member vars
self.user = user.user( )
self.processedUrls = { }
self.historian = historian.historian( self )
self.configParser = conf.conf( ).genDefault( )
self.txtFile = False
self.skipHandling = False
self.browsers = { }
self.urllist = [ ]
self.blacklist = [ ]
self.whitelist = [ ]
self.stats = {
'tick': datetime.now( ),
'tock': None,
'visited': 0,
'skipped': 0,
'handled': 0
}
# use CLI command == function name, use it
getattr( self, args.command )( )
def detect_shapes(self):
# Get mask by filtering by color saturation in HSV color space
hsv = cv2.cvtColor(realsensecam().bgr, cv2.COLOR_BGR2HSV)
mask = cv2.inRange(
hsv, np.array([0, conf()['shape_saturation_threshold'], 0]),
np.array([255, 255, 255]))
self.most_recent_mask = mask
# Find contours in mask in order to isolate individual shapes
contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_SIMPLE)
detected_shapes = []
for cnt in contours:
if cv2.contourArea(cnt) < 750:
continue
if handdetector().cnt_intersects_with_hand(cnt):
continue
s = Shape(cnt)
if s is not None:
detected_shapes.append(s)
return detected_shapes
def __init__(self, port):
util_root = '/works/demon_11st/utils'
sys.path.insert(0, util_root)
from exifutil import exifutil
app.exifutils = exifutil()
#import pdb; pdb.set_trace()
agent_root = '/works/demon_11st/agent/detection'
sys.path.insert(0, agent_root)
import _init_paths
from conf import conf
from agent import agent
yaml_file = '/storage/product/detection/11st_All/cfg/faster_rcnn_end2end_test.yml'
conf = conf(yaml_file, 0)
app.agent = agent()
from korean_url_handler import korean_url_handler
app.korean_url_handler = korean_url_handler()
# start web server
web_server.__init__(self, app, port)
def __action_menu_open(self):
self.action_menu_active = True
self.action_menu_armed = False
self.action_menu_progress = 0
# Contact NodeRed and load available actions
try:
r = requests.get(conf()['actions_url'])
matrix = list(np.ndindex(3, 6))[3:]
self.action_menu.clear_items()
self.action_menu.add_item(0, 1, 'Exit action menu',
self.__action_menu_close)
self.action_menu.add_item(0, 2, '(none)', self.__set_shape_action,
"")
for pos, action in enumerate(r.json()):
if pos == len(matrix):
print(
"Cannot display all actions in menu, not enough space!"
)
self.action_menu.add_item(*tuple(matrix[pos]), action,
self.__set_shape_action, action)
except requests.exceptions.RequestException as e:
print("An error occured while attempting to connect to NodeRed:")
print(e)
def __init__(self, logger=None):
super().__init__()
if conf()['compile_pyx_on_startup']:
os.system("python setup.py build_ext --inplace")
self.frame = 0
self.logger = logger
# Initialize all needed modules
shapedetector()
shapetracker()
touchedshapetracker()
handdetector()
handtracker()
visualizer()
ui()
shapepicker()
shaperegionpicker()
# Put the wires together
handtracker().subscribe('hand_exit', shapetracker().clear_lost_shapes)
handtracker().subscribe('finger_up',
touchedshapetracker().on_finger_up)
handtracker().subscribe('finger_pressing', ui().on_finger_pressing)
handtracker().subscribe('finger_up', ui().on_finger_up)
handtracker().subscribe('finger_down', ui().on_finger_down)
handtracker().subscribe('finger_pressing',
shapepicker().on_finger_pressing)
handtracker().subscribe('finger_up', shapepicker().on_finger_up)
handtracker().subscribe('hand_exit', handdetector().on_hand_exit)
handtracker().subscribe('finger_pressing',
shaperegionpicker().on_finger_pressing)
handtracker().subscribe('finger_up', shaperegionpicker().on_finger_up)
handtracker().subscribe('hand_exit', shaperegionpicker().on_hand_exit)
handtracker().subscribe('hand_exit',
shapepositionpicker().on_hand_exit)
def test_reading(self):
cf = conf()
self.assertEqual(cf.getConfObject("Logging","errorlog"), '../logs/error.log2')
def test_singleton_conf_creation(self): cf = conf(confFile="testconf.ini") cf2 = conf(confFile="nottest.ini") self.assertEqual(cf.url, cf2.url)
def local():
c = conf.conf()
path = c.getStorePath()
getFile(path)
def __init__(self):
log = getLogger(__name__)
try:
self.conf = conf().getdict("DATABASE")
except RegistryError, e:
log.error(e)
from datetime import datetime
import numpy as np
import torch
from sklearn.preprocessing import MultiLabelBinarizer
from tensorboardX import SummaryWriter
from torch.autograd import Variable
from torch.utils.data import DataLoader
from tqdm import tqdm
from Logger_morning import myLogger
from conf import conf, print_config
from feature_dataset import featureDataset, my_collate
from raw_model import Linear_Model
from readData_fungo import read_fungo
args = conf()
if args.load_model is None or args.isTrain:
comment = f'_{args.dataset}_{args.base_model}_{args.mode}_{args.remark}'
current_time = datetime.now().strftime('%b%d_%H-%M-%S')
log_dir = os.path.join('runs2', current_time + comment)
else:
log_dir = ''.join(args.load_model[:args.load_model.rfind('/')])
print(f'reuse dir: {log_dir}')
logger = myLogger(name='exp', log_path=log_dir + '.log')
# incompatible with logger...
writer = SummaryWriter(log_dir=log_dir)
writer.add_text('Parameters', str(vars(args)))
print_config(args, logger)
logger.setLevel(args.log_level)
from HMCN import HMCN
def __init__(self):
self.proxy_cf = configparser.ConfigParser()
self.cf = conf.conf()
self.proxy_cf.read(self.cf.get_proxy_conf(), encoding="utf-8")
async def sim_post(body: body_sim, request: Request, citytag: str = 'null'):
client_ip = request.client.host
log_print('Request from {} city {}'.format(client_ip, citytag))
start_date = body.start_date
stop_date = body.stop_date
sampling_dt = body.sampling_dt
log_print('Parameters {} - {} sampling {} city {}'.format(
start_date, stop_date, sampling_dt, citytag))
# init conf
try:
cfg_file = os.path.join(os.environ['WORKSPACE'], 'slides', 'vars',
'conf', 'conf.json')
with open(cfg_file) as cin:
cfg = json.load(cin)
cw = conf(cfg, logger)
except Exception as e:
log_print('conf init failed : {}'.format(e))
raise HTTPException(status_code=500,
detail='conf init failed : {}'.format(e))
# sanity check
if citytag not in cw.cparams:
raise HTTPException(
status_code=400,
detail='city \'{}\' not available. Current cities : {}'.format(
citytag, list(cw.cparams.keys())))
# generate config json
try:
simconf = cw.generate(start_date, stop_date, citytag)
except Exception as e:
log_print('config generation failed : {}'.format(e))
raise HTTPException(status_code=500,
detail='conf generation failed : {}'.format(e))
# set up environment and move execution to working dir
wdir = cw.wdir
try:
os.chdir(wdir)
except:
os.mkdir(wdir)
os.chdir(wdir)
# override sim parameters
simconf['start_date'] = start_date
simconf['stop_date'] = stop_date
simconf['sampling_dt'] = sampling_dt
simconf['state_basename'] = wdir + '/r_{}'.format(citytag)
simconf['enable_stats'] = True
#simconf['explore_node'] = [0]
confs = json.dumps(simconf)
with open(wdir + '/wsconf_sim_{}.json'.format(citytag), 'w') as outc:
json.dump(simconf, outc, indent=2)
#print(confs, flush=True)
# run simulation
s = simulation(confs)
log_print('sim info : {}'.format(s.sim_info()))
if s.is_valid():
tsim = datetime.now()
s.run()
log_print('{} simulation done in {}'.format(citytag,
datetime.now() - tsim))
pof = s.poly_outfile()
log_print('Polyline counters output file : {}'.format(pof))
dfp = pd.read_csv(pof, sep=';')
pp = {
t: {i: int(x)
for i, x in enumerate(v[1:]) if x != 0}
for t, v in zip(dfp.timestamp, dfp.values)
}
ret = {'message': 'simulation OK', 'city': citytag, 'data': pp}
else:
raise HTTPException(status_code=501, detail='simulation init failed')
return ret
def main():
try:
# Set Global Watchdog timeout in Seconds
MOD.watchdogEnable(300)
#######################################
## Initialization
#######################################
rtnList = [-1,-1] #[return status,return data]
# return status:
# -1: Exception occurred
# 0: No errors occurred, no return data
# 1: No errors occurred, return data
# 2: Error occurred
#Initialize the serial interface @ 115200, we must do this or nothing comes out the serial port
rtnList = mySER.init("115200",'8N1')
if (rtnList[0] == -1):
return
mySER.sendUART("Beginning the serial bridge program. \r\n\r\n")
# Set Global Watchdog timeout in Seconds
#MOD.watchdogEnable(300)
#Get configuration from demo.conf file, transpose into new local myApp class
myApp = conf.conf('/sys','demo.conf')
if myApp.CONF_STATUS != 0:
mySER.sendUART("Demo configuration error: " + str(myApp.CONF_STATUS))
return rtnList
mySER.sendUART("Configuration Loaded.\r\n")
rtnList = myIO.init()
if (rtnList[0] == -1) or (rtnList[0] == -2) or rtnList[1] == "ERROR": raise UserWarning
mySER.sendUART("\r\nInitializing Network Setup. \r\n")
#Set Network specific settings, wait for SIM card to be ready
rtnList = NETWORK.initNetwork(myApp.ENS)
if (rtnList[0] == -1) or (rtnList[0] == -2): raise UserWarning
#Initialize SOCKET communications
rtnList = SOCKET.init('1',myApp.APN)
if (rtnList[0] == -1) or (rtnList[0] == -2): raise UserWarning
mySER.sendUART("Network Setup Initialized. \r\n\r\n")
#Update Unit information
rtnList = ATC.getUnitInfo()
if (rtnList[0] == -1): raise UserWarning
# Loop forever, without this loop the script would run once and exit script mode. On reboot or power-on the script would run once more
while (1):
MOD.watchdogReset()
#######################################
## Registration
#######################################
RegCheck = 0 #Initialize check
DataCheck = 0 #Initialize Check
#What is the signal strength?
rtnList = ATC.sendAtCmd('AT+CSQ',ATC.properties.CMD_TERMINATOR,5)
if (rtnList[0] == -1) or (rtnList[0] == -2): raise UserWarning
mySER.sendUART("Signal Strength (AT+CSQ): " + rtnList[1] + "\r\n")
#Wait until module is registered to GSM Network
mySER.sendUART("Checking Modem Registration. \r\n")
rtnList = NETWORK.isRegistered(180) #Wait 180 seconds for module to obtain GSM registration
if (rtnList[0] == -1) or (rtnList[0] == -2): raise UserWarning
if (rtnList[0] == 0): #Registered successfully
RegCheck = 1
mySER.sendUART("Modem Registered. \r\n\r\n")
#Wait until module is ready for data
mySER.sendUART("Checking Data Availability. \r\n")
rtnList = NETWORK.isDataAttached(myApp.CGMM, 180) #Wait 180 seconds for module to obtain GSM registration
if (rtnList[0] == -1) or (rtnList[0] == -2): raise UserWarning
if (rtnList[0] == 0): #Registered successfully
DataCheck = 1
mySER.sendUART("Modem ready for data. \r\n\r\n")
#check for exit prompt, let's use command EXITSCR
rtnList = mySER.readUART()
if (rtnList[1].find('EXITSCR') != -1):
print '\r\nExit Command Found\r\n'
return
#######################################
## Socket Connection
#######################################
# Loop while we're registered/ready
while (RegCheck == 1 and DataCheck == 1):
#check for exit prompt, let's use command EXITSCR
rtnList = mySER.readUART()
if (rtnList[1].find('EXITSCR') != -1):
print '\r\nExit Command Found\r\n'
return
rtnList = NETWORK.isRegistered(10) #Fast reg check
if (rtnList[0] == -1) or (rtnList[0] == -2): #unavailable, drill down discretely and show the status
RegCheck = 0
rtnList = ATC.sendAtCmd('AT+CREG?',ATC.properties.CMD_TERMINATOR,5)
mySER.sendUART("Registration Unavailable, Status: " + rtnList[1] + "\r\n")
break #exit loop
rtnList = NETWORK.isDataAttached(myApp.CGMM, 10) #Fast data check
if (rtnList[0] == -1) or (rtnList[0] == -2): #unavailable, drill down discretely and show the status
DataCheck = 0
rtnList = ATC.sendAtCmd('AT+CGREG?',ATC.properties.CMD_TERMINATOR,5)
mySER.sendUART("Data Unavailable, Status: " + rtnList[1] + "\r\n")
break #exit loop
#Open socket in online mode
mySER.sendUART("Opening socket to server: " + myApp.IP + ":" + myApp.PORT + "\r\n")
rtnList = SOCKET.openSocket(myApp.IP,myApp.PORT,1,myApp.USERNAME,myApp.PASSWORD,myApp.PROTOCOL,0)
if (rtnList[0] == -1) or (rtnList[0] == -2): raise UserWarning
#Check for open connection, this catches both online and command mode operations
if (rtnList[1] == "OK" or rtnList[1] == "CONNECT"):
mySER.sendUART("Socket opened successfully.\r\n")
else:
mySER.sendUART("Connection failed to open, trying again. \r\n")
#Check for open socket
DCD = MDM.getDCD()
if (DCD == 1):
#Set DCD on USIF0
SER.setDCD(1)
#Forward CONNECT Message to Serial Port
mySER.sendUART('\r\nConnected to Server\r\n')
myIO.USER_LED('ON')
#######################################
## Data exchange loop
#######################################
##Loop while Socket connected. This loop operates on the notion that we are in online mode, so we are not using send_CM
while(1):
#Pet the watchdog
MOD.watchdogReset()
#Check for DCD active on MDM
DCD = MDM.getDCD()
if (DCD == 1):
#Outbound
#Get data from serial port, use function return
rtnList = mySER.readUART()
if (len(rtnList[1])!=0):
#Forward data to open socket if it's not empty and not the exit script command
#check for exit prompt, let's use command EXITSCR
if (rtnList[1].find('EXITSCR') != -1): #exit found
print '\r\nExit Command Found\r\n'
print 'Exiting data mode\r\n'
rtnList = SOCKET.exitDataMode()
print 'Closing the socket\r\n'
rtnList = SOCKET.closeSocket(1)
return
print rtnList[1] #Debug
res = MDM.send(rtnList[1],1)
#Inbound
#Get data from open socket connection, straight MDM read
res = MDM.read()
if (len(res)!=0):
#Forward socket data to serial port
print res #Debug
mySER.sendUART(res)
#When socket is closed exit loop via this path
#Will guarantee that '\r\nNO CARRIER\r\n' is sent every time
if (DCD == 0):
#Clear DCD on USIF0
SER.setDCD(0)
myIO.USER_LED('OFF')
#Get any remaining data from closed socket connection, straight MDM read
res = MDM.read()
if (len(res)!=0):
#Forward socket data to serial port
mySER.sendUART(res)
break #exit loop to registration checks
except UserWarning:
print 'Controlled Script exit'
except:
print sys.exc_info()
rtnList[0] = -1
return
from xml.etree.ElementTree import Element, SubElement, dump, ElementTree, fromstring, tostring
import pytz
import requests
from conf import conf
from transit_util import Stop, Vehicle
regex = '(?<=<)\\w+:'
mot_siri = 'http://siri.motrealtime.co.il:8081/Siri/SiriServices'
icarus_siri_proxy = 'http://siri.motrealtime.icarusnet.me/Siri/SiriServices'
url = icarus_siri_proxy
username = DB = conf()['SIRI']['Username']
# username = '******'
ns0 = 'http://schemas.xmlsoap.org/soap/envelope/'
ns1 = 'http://new.webservice.namespace'
ns3 = 'http://www.ifopt.org.uk/acsb'
ns4 = 'http://www.ifopt.org.uk/ifopt'
ns_siri = 'http://www.siri.org.uk/siri'
ns_siriWS = 'http://new.webservice.namespace'
nsS = 'http://schemas.xmlsoap.org/soap/envelope/'
prefix = '<?xml version="1.0" ?>'
def xpath(tags):
path = './'
for ns, tag in tags:
path += ('{' + ns + '}' if ns is not None else '') + tag + '/'
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import time
import datetime
import csv
from selenium.webdriver.support.ui import Select
import conf
import myconf
conf = conf.conf()
myconf = myconf.myconf()
driver = myconf.driver
def clickThenWait(selector_type, selector, wait_time):
if selector_type == "id":
element = driver.find_element_by_id(selector)
if selector_type == "name":
element = driver.find_element_by_name(selector)
if selector_type == "xpath":
element = driver.find_element_by_xpath(selector)
if selector_type == "text":
element = driver.find_element_by_link_text(selector)
driver.execute_script('arguments[0].scrollIntoView({block: "center",});',
element)
element.click()
time.sleep(wait_time)
#TODOクラス化する
def goSiteManagementPage():
import os
import conf
import logging
import sys
import shutil
from model import DataModel
LOG = logging.getLogger(__name__)
CONF = conf.conf()
db_full_path = os.path.join(conf.project_file, CONF["multifile"]["fold"])
class MultiFile(DataModel):
def __init__(self, **kwargs):
super(MultiFile, self).__init__(**kwargs)
self.path = None
if not os.path.isdir(db_full_path):
os.mkdir(db_full_path, mode=0o777)
def key_path(self, key):
path = os.path.join(self.path, key + ".txt")
return path
def user_path(self):
self.data = self.model_data()
user_id = self.data['user_id']
if not user_id:
logging.error("Not exist userID,the data is %s" % self.data)
return False
user_path = os.path.join(db_full_path, user_id)
return user_path
def test_autosetting(self): cf = conf() self.assertEqual(cf.url, 'http://www.postimees.ee')
def get_flights_from_api(airport, json_data=None):
"""Get flights from the FlightRequests API and immediately cache them in the database"""
time = datetime.datetime.now()
flight_stats_conf = conf()["flightStats"]
# Can't use Request's parameter handling because most of these parameters are built into the URL path,
# i.e. not passed as part of the querystring
url = flight_stats_conf["urls"]["arrivalsByAirport"].format(
airport=airport,
year=time.year,
month=time.month,
day=time.day,
hour=time.hour,
app_id=flight_stats_conf["appId"],
app_key=flight_stats_conf["appKey"])
response = requests.get(url)
if response.status_code != 200:
print("Failed to query flightstats for status of airport " + airport,
file=sys.stderr)
return
data = response.json()
# TODO: Probably change this so we're not needlessly constructing dictionaries from data
# already provided by the server. Doing it this way started out as a way to make the db
# less reliant on how the FlightStats api provides data to us, but now I'm not so sure
# it's a good idea.
# Data acquired; extract flights
ret = list(
map(
lambda flight: {
"flightId":
int(flight["flightId"]),
"arrivalId":
str(flight["arrivalAirportFsCode"]),
"departureId":
str(flight["departureAirportFsCode"]),
"airlineCode":
str(flight["carrierFsCode"]),
"flightNumber":
int(flight["flightNumber"]),
"departure_local":
parser.parse(flight["departureDate"]["dateLocal"]),
"arrival_local":
parser.parse(flight["arrivalDate"]["dateLocal"])
}, data["flightStatuses"]))
# In order to shove things into the db we need to extract relevant airports & airlines
# for better readability on the front end
if "airports" in data["appendix"].keys():
airports = list(
map(
lambda raw_airport: {
"name": raw_airport["name"],
"city": raw_airport["city"],
"country": raw_airport["countryName"],
"fs": raw_airport["fs"],
"iata": raw_airport["iata"]
}, data["appendix"]["airports"]))
db.save_airports(airports)
if "airlines" in data["appendix"].keys():
airlines = list(
map(
lambda airline: {
"name": airline["name"],
"fs": airline["fs"],
"iata": airline["iata"]
}, data["appendix"]["airlines"]))
db.save_airlines(airlines)
db.save_flights(ret)
return ret
from sqlalchemy import create_engine
from sqlalchemy.ext.automap import automap_base
from sqlalchemy.orm import sessionmaker
from conf import conf
from transit_util import Stop
lat_bounds = {
'high': 29.596243,
'low': 29.515590,
}
lon_bounds = {'high': 34.998930, 'low': 34.900423}
print("Creating Engine...", end="")
engine = None
try:
DB = conf()['db']
engine = create_engine('mysql+pymysql://{}:{}@{}/{}'.format(
DB['user'], DB['password'], DB['host'], DB['database']))
automap = automap_base()
automap.prepare(engine, reflect=True)
except Exception as e:
print("Failed:", e)
exit(-1)
print("Done")
def gtfs_import():
schedule = Schedule("gtfs.sqlite")
append_feed(schedule, "israel-gtfs.zip")
def main():
try:
rtnList = [-1, -1] # [return status,return data]
# return status:
# -1: Exception occurred
# 0: No errors occurred, no return data
# 1: No errors occurred, return data
# 2: Error occurred
# Initialize the serial interface @ 115200, we must do this or nothing comes out the serial port
rtnList = mySER.init("115200", "8N1")
if rtnList[0] == -1:
return
mySER.sendUART("Beginning the T2 SMS Query Program. \r\n\r\n")
# Set Global Watchdog timeout in Seconds
# MOD.watchdogEnable(300)
# Get configuration from demoT2.conf file, transpose into new local myApp class
myApp = conf.conf("/sys", "demoT2.conf")
if myApp.CONF_STATUS != 0:
mySER.sendUART("DemoT2 configuration error: " + str(myApp.CONF_STATUS))
return rtnList
mySER.sendUART("Configuration Loaded.\r\n")
rtnList = myIO.init()
if (rtnList[0] == -1) or (rtnList[0] == -2) or rtnList[1] == "ERROR":
raise UserWarning
mySER.sendUART("\r\nInitializing Network Setup. \r\n")
# Set Network specific settings, wait for SIM card to be ready
rtnList = NETWORK.initNetwork(myApp.ENS)
if (rtnList[0] == -1) or (rtnList[0] == -2) or rtnList[1] == "ERROR":
raise UserWarning
# Initialize SOCKET communications
rtnList = SOCKET.init("1", myApp.APN)
if (rtnList[0] == -1) or (rtnList[0] == -2) or rtnList[1] == "ERROR":
raise UserWarning
mySER.sendUART("Network Setup Initialized. \r\n\r\n")
mySER.sendUART("Initializing GPS \r\n")
# Turn the GPS ON
rtnList[1] = GPS.getPowerOnOff()
while rtnList[1] != 1:
GPS.powerOnOff(1)
rtnList[1] = GPS.getPowerOnOff()
mySER.sendUART("GPS Status: " + str(rtnList[1]) + "\r\n")
mySER.sendUART("GPS Initialized. \r\n\r\n")
mySER.sendUART("Initializing SMS. \r\n")
# Setup SMS
rtnList = mySMS.configSMS()
if rtnList[0] == 0:
mySER.sendUART("SMS Initialized. \r\n\r\n")
else:
return
# Update Unit information
rtnList = ATC.getUnitInfo()
if (rtnList[0] == -1) or rtnList[1] == "ERROR":
raise UserWarning
# Loop forever, without this loop the script would run once and exit script mode. On reboot or power-on the script would run once more
while 1:
# MOD.watchdogReset()
RegCheck = 0 # Initialize check
mySER.sendUART("Checking Modem Registration. \r\n")
# Wait until module is registered to GSM Network
rtnList = NETWORK.isRegistered(180) # Wait 180 seconds for module to obtain GSM registration
if (rtnList[0] == -1) or (rtnList[0] == -2) or rtnList[1] == "ERROR":
raise UserWarning
if rtnList[0] == 0:
RegCheck = 1
mySER.sendUART("Modem Registered. Waiting for SMS. \r\n\r\n")
# Loop forever, without this loop the script would run once and exit script mode. On reboot or power-on the script would run once more
while RegCheck == 1:
# MOD.watchdogReset()
# Update NMEA Data
# The following are available through the included GPS module:
# GPS.getActualPosition(), returns all fields like AT$GPSACP would
# GPS.getLastGGA()
# GPS.getLastGLL()
# GPS.getLastGSA()
# GPS.getLastGSV()
# GPS.getLastRMC()
# GPS.getLastVTG()
# GPS.getPosition(), this gives LAT/LONG in numeric format
# For the purposes of this demo, RMC will be used
# The returned value gives a \r\n at the end so we must strip it for proper usage.
# sleep for 1 second to let the GPS catch up otherwise we eventually get a response that .rstrip errors on
time.sleep(1)
GPSPositionTemp = ""
GPSPositionTemp = GPS.getLastRMC()
GPSPositionTemp = GPSPositionTemp.rstrip()
rtnList = mySMS.CheckNewSMS()
# If a new SMS is found and is valid, pass it to the command logic
if rtnList[0] == 1:
# We have received a new SMS, let's find what it wants us to do.
mySER.sendUART("SMS Received.\r\n")
mySER.sendUART("SMS Data : " + str(rtnList[1]) + "\r\n")
rtnList = mySMS.SMSCommand(rtnList[1])
# 0: Pass, action carried out and SMS sent back
# -1: Exception
# -2: Pass, Unrecognized change command or AT command received though
# -3: Unrecognized SMS
# -4: Error sending an SMS to the originating P/N
if rtnList[0] == -1:
return
rtnList = NETWORK.isRegistered(10) # Check Registration on the fly
if (rtnList[0] == -1) or (rtnList[0] == -2):
RegCheck = 0
mySER.sendUART("\r\nRegistration not available, checking status.\r\n")
# What is the signal strength?
rtnList = ATC.sendAtCmd("AT+CSQ", ATC.properties.CMD_TERMINATOR, 0, 5)
mySER.sendUART("Signal Strength (AT+CSQ): " + rtnList[1] + "\r\n")
# Still registered?
rtnList = ATC.sendAtCmd("AT+CREG?", ATC.properties.CMD_TERMINATOR, 0, 5)
mySER.sendUART("Registration Check (AT+CREG?): " + rtnList[1] + "\r\n")
break
except UserWarning:
print "Controlled Script exit"
except:
print sys.exc_info()
rtnList[0] = -1
return
def __init__(self): self.__cfg = conf.conf() self.__pageValues = self.__cfg.getIndex()
import shell_cmd as shell
import time
import threading
import Queue
def shell_cmd(camd):
subprocess.PIPE
#cmd= 'ping -c %s -i %s %s' % (3,0.01, "172.16.205."+str(i)+"\n")
P=subprocess.Popen(cmd,stdin = subprocess.PIPE,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE,
shell = True)
P.stdin.close()
P.wait()
return P.stdout.readlines()
cf = conf.conf('floatingip.conf')
def external:
cmd = 'neutron net-list --router:external=True'
def floatingip_create(user,num):
count = 1
while (count < num+1):
cmd_neutron = 'neutron floatingip-create %s' % (external)
neutrn = shell.shell_cmd(cmd_neutron)
if (user == 'admin'):
shell.shell_cmd('bash')
rc = cf.admin_rc()
cmd_source = 'source %s' %(rc)
source = shell.shell_cmd(cmd_source)
cmd_neutron
import requests
import json
from requests.auth import HTTPBasicAuth
from conf import conf
#get connection info
conf=conf()
def get_all_ports_statics():
#configure Json
headers = {'Content-type': 'application/json'}
flowUrl = '/controller/nb/v2/statistics/default/port'
url = conf['controllerIp'] + flowUrl
#get flow static
result=requests.get(url,auth=conf['auth'],headers=headers)
#print json result
if result.status_code>210:
print 'Json Error'
#decode json data
data=result.json()
#return portStatic
return data
if not tm: tm= time.strftime("%H:%M:%S")
db.connect()
tags = soup('h1')
with open(output, 'a', encoding="utf-8", newline="") as csvfile:
writer = csv.writer(csvfile, delimiter=';')
for tag in tags:
counter["total"]+=1
try:
heading = self.removeComments("".join([string.strip() for string in tag.strings]))
writer.writerow([heading, str(counter["total"]), dt, tm])
print (dt)
db.insertHeading(heading, str(counter["total"]), dt, tm)
except UnicodeEncodeError:
counter["errors"]+=1
print ("jama")
db.close()
return counter
if __name__ == '__main__':
cf = conf()
url=cf.url
db = database(cf)
html = urlopen(url).read()
pmparser = postimeesParser()
counter = pmparser.parse(html, db, cf=cf)
print (str(counter["total"]) + " korda")
print (str(counter["errors"]) + " probleemi")